Examples of org.kitesdk.data.spi.DatasetRepository

org.kitesdk.data.spi.DatasetRepository

A logical repository (storage system) of {@link Dataset}s.

Implementations of {@code DatasetRepository} are storage systems that containzero or more {@link Dataset}s. A repository acts as a factory, as well as a registry, of datasets. You can {@link #create(String,String,DatasetDescriptor)} anew {@link Dataset} with a name and schema, or retrieve a handle to anexisting dataset, by name, by way of the {@link #load(String,String)} method. Whilenot expressly forbidden, most repositories are expected to support only a single concrete {@link Dataset} implementation.

No guarantees are made as to the durability, reliability, or availability of the underlying storage. That is, a {@code DatasetRepository} could be ondisk, in memory, or some combination. See the implementation class for details about the guarantees it provides.

Implementations of {@link DatasetRepository} are immutable.
@see Dataset @see DatasetDescriptor

        "The entity type can't be null, use Object.class to have the type"
            + " determined by the schema.");


    Pair<DatasetRepository, Map<String, String>> pair =
        Registration.lookupDatasetUri(URI.create(uri.getRawSchemeSpecificPart()));
    DatasetRepository repo = pair.first();
    Map<String, String> uriOptions = pair.second();


    return (D) repo.update(
        uriOptions.get(URIBuilder.NAMESPACE_OPTION),
        uriOptions.get(URIBuilder.DATASET_NAME_OPTION), descriptor, type);
  }

View Full Code Here

        URIBuilder.DATASET_SCHEME.equals(uri.getScheme()),
        "Not a dataset URI: " + uri);


    Pair<DatasetRepository, Map<String, String>> pair =
        Registration.lookupDatasetUri(URI.create(uri.getRawSchemeSpecificPart()));
    DatasetRepository repo = pair.first();
    Map<String, String> uriOptions = pair.second();


    return repo.delete(
        uriOptions.get(URIBuilder.NAMESPACE_OPTION),
        uriOptions.get(URIBuilder.DATASET_NAME_OPTION));
  }

View Full Code Here

        URIBuilder.DATASET_SCHEME.equals(uri.getScheme()),
        "Not a dataset URI: " + uri);


    Pair<DatasetRepository, Map<String, String>> pair =
        Registration.lookupDatasetUri(URI.create(uri.getRawSchemeSpecificPart()));
    DatasetRepository repo = pair.first();
    Map<String, String> uriOptions = pair.second();


    return repo.exists(
        uriOptions.get(URIBuilder.NAMESPACE_OPTION),
        uriOptions.get(URIBuilder.DATASET_NAME_OPTION));
  }

View Full Code Here

   * @return the URIs present in the {@code DatasetRepository}
   */
  public static Collection<URI> list(URI uri) {
    boolean isRepo = URIBuilder.REPO_SCHEME.equals(uri.getScheme());
    Preconditions.checkArgument(isRepo, "Not a repository URI: " + uri);
    DatasetRepository repo = Registration
        .open(URI.create(uri.getRawSchemeSpecificPart()));


    // build a URI for each dataset name
    URI repoUri = repo.getUri();
    List<URI> datasets = Lists.newArrayList();
    for (String namespace : repo.namespaces()) {
      for (String dataset : repo.datasets(namespace)) {
        datasets.add(new URIBuilder(repoUri, namespace, dataset).build());
      }
    }


    return datasets;

View Full Code Here

    Registration.register(
        mockPattern, mockPattern,
        new OptionBuilder<DatasetRepository>() {
          @Override
          public DatasetRepository getFromOptions(Map<String, String> options) {
            DatasetRepository repo = repos.get(options.get("id"));
            if (repo == null) {
              repo = mock(org.kitesdk.data.spi.DatasetRepository.class);
              when(repo.getUri()).thenReturn(
                  URI.create("repo:" + mockPattern.construct(options)));
              repos.put(options.get("id"), repo);
            }
            return repo;
          }

View Full Code Here

    command.datasets = Lists.newArrayList(source, dest);


    int rc = command.run();
    Assert.assertEquals("Should return success", 0, rc);


    DatasetRepository repo = DatasetRepositories.repositoryFor("repo:file:target/data");
    int size = DatasetTestUtilities.datasetSize(repo.load("default", dest));
    Assert.assertEquals("Should contain copied records", 2, size);
  }

View Full Code Here

    command.datasets = Lists.newArrayList(source, dest);


    int rc = command.run();
    Assert.assertEquals("Should return success", 0, rc);


    DatasetRepository repo = DatasetRepositories.repositoryFor("repo:file:target/data");
    Set<GenericRecord> records = DatasetTestUtilities.materialize(
        repo.<GenericRecord>load("default", dest));
    Assert.assertEquals("Should contain copied records", 2, records.size());
    for (GenericRecord record : records) {
      Assert.assertTrue("Username should be upper case",
          UPPER_CASE.matcher(record.get("username").toString()).matches());
    }

View Full Code Here

    command.datasets = Lists.newArrayList(source, dest);


    int rc = command.run();
    Assert.assertEquals("Should return success", 0, rc);


    DatasetRepository repo = DatasetRepositories.repositoryFor("repo:" + repoUri);
    FileSystemDataset<GenericData.Record> ds =
        (FileSystemDataset<GenericData.Record>) repo.<GenericData.Record>
            load("default", dest);
    int size = DatasetTestUtilities.datasetSize(ds);
    Assert.assertEquals("Should contain copied records", 6, size);


    Assert.assertEquals("Should produce 1 files",

View Full Code Here

    command.datasets = Lists.newArrayList(source, dest);


    int rc = command.run();
    Assert.assertEquals("Should return success", 0, rc);


    DatasetRepository repo = DatasetRepositories.repositoryFor("repo:" + repoUri);
    FileSystemDataset<GenericData.Record> ds =
        (FileSystemDataset<GenericData.Record>) repo.<GenericData.Record>
            load("default", dest);
    int size = DatasetTestUtilities.datasetSize(ds);
    Assert.assertEquals("Should contain copied records", 6, size);


    Assert.assertEquals("Should produce 3 files",

View Full Code Here

        .build(), Object.class);


    int rc = command.run();
    Assert.assertEquals("Should return success", 0, rc);


    DatasetRepository repo = DatasetRepositories.repositoryFor("repo:" + repoUri);
    FileSystemDataset<GenericData.Record> ds =
        (FileSystemDataset<GenericData.Record>) repo.<GenericData.Record>
            load("default", "dest_partitioned");
    int size = DatasetTestUtilities.datasetSize(ds);
    Assert.assertEquals("Should contain copied records", 6, size);


    Assert.assertEquals("Should produce 2 partitions",

View Full Code Here

0 1 2 3 4 5 6 7

TOP

Related Classes of org.kitesdk.data.spi.DatasetRepository

org.kitesdk.cli.commands.DeleteDatasetCommand

org.kitesdk.cli.commands.TestCopyCommandCluster

org.kitesdk.cli.commands.TestCopyCommandLocal

org.kitesdk.cli.commands.TestCreateDatasetCommandCluster

org.kitesdk.cli.commands.TestTransformCommandCluster

org.kitesdk.cli.commands.TestTransformCommandLocal

org.kitesdk.data.Datasets

org.kitesdk.data.hbase.TestHBaseDatasetURIs

org.kitesdk.data.hbase.TestHBaseRepositoryURIs

org.kitesdk.data.mapreduce.DatasetKeyOutputFormat

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.