Examples of org.kitesdk.data.spi.DatasetRepository

org.kitesdk.data.spi.DatasetRepository

A logical repository (storage system) of {@link Dataset}s.

Implementations of {@code DatasetRepository} are storage systems that containzero or more {@link Dataset}s. A repository acts as a factory, as well as a registry, of datasets. You can {@link #create(String,String,DatasetDescriptor)} anew {@link Dataset} with a name and schema, or retrieve a handle to anexisting dataset, by name, by way of the {@link #load(String,String)} method. Whilenot expressly forbidden, most repositories are expected to support only a single concrete {@link Dataset} implementation.

No guarantees are made as to the durability, reliability, or availability of the underlying storage. That is, a {@code DatasetRepository} could be ondisk, in memory, or some combination. See the implementation class for details about the guarantees it provides.

Implementations of {@link DatasetRepository} are immutable.
@see Dataset @see DatasetDescriptor

  @Test
  public void testLoadChangedRelativePathURIMissingNamespace() {
    // this used to be a relative external URI, but is now a managed URI
    String uri = "dataset:hive:ds";


    DatasetRepository repo = DatasetRepositories
        .repositoryFor("repo:hive:/tmp/data");
    DatasetDescriptor withLocation = new DatasetDescriptor.Builder(DESCRIPTOR)
        .location("file:/tmp/data/ds") // old location
        .build();
    Dataset<GenericRecord> expected = repo.create(
        "default", "ds", withLocation, GenericRecord.class);


    Dataset<GenericRecord> actual = Datasets.load(uri);
    Assert.assertEquals("Should load existing dataset default.ds",
        expected, actual);

View Full Code Here

  @Test
  public void testLoadChangedAbsolutePathURI() {
    // this used to be a relative external URI, but is now a managed URI
    String uri = "dataset:hive:/ns/ds";


    DatasetRepository repo = DatasetRepositories
        .repositoryFor("repo:hive:/tmp/data");
    Dataset<GenericRecord> expected = repo.create(
        "ns", "ds", DESCRIPTOR, GenericRecord.class);


    Dataset<GenericRecord> actual = Datasets.load(uri);
    Assert.assertEquals("Should load existing dataset ns.ds",
        expected, actual);

View Full Code Here

  @Test
  public void testLoadChangedAbsolutePathURICompatibility() {
    // this used to be a relative external URI, but is now a managed URI
    String uri = "dataset:hive:/data/ds";


    DatasetRepository repo = DatasetRepositories
        .repositoryFor("repo:hive:/tmp/data");
    DatasetDescriptor withLocation = new DatasetDescriptor.Builder(DESCRIPTOR)
        .location("file:/tmp/data/ds") // old location
        .build();
    Dataset<GenericRecord> expected = repo.create(
        "default", "ds", withLocation, GenericRecord.class);


    Dataset<GenericRecord> actual = Datasets.load(uri);
    Assert.assertEquals("Should load existing dataset default.ds",
        expected, actual);

View Full Code Here

  @Test
  public void testLoadChangedAbsolutePathURIMissingNamespace() {
    // this used to be a relative external URI, but is now a managed URI
    String uri = "dataset:hive:/ds";


    DatasetRepository repo = DatasetRepositories
        .repositoryFor("repo:hive:/tmp/data");
    DatasetDescriptor withLocation = new DatasetDescriptor.Builder(DESCRIPTOR)
        .location("file:/tmp/data/ds") // old location
        .build();
    Dataset<GenericRecord> expected = repo.create(
        "default", "ds", withLocation, GenericRecord.class);


    Dataset<GenericRecord> actual = Datasets.load(uri);
    Assert.assertEquals("Should load existing dataset default.ds",
        expected, actual);

View Full Code Here

    }
  }


  @Test
  public void testManagedURI() throws Exception {
    DatasetRepository repo = DatasetRepositories.repositoryFor("repo:hive");


    Assert.assertNotNull("Received a repository", repo);
    Assert.assertTrue("Repo should be a HCatalogDatasetRepository",
        repo instanceof HiveManagedDatasetRepository);
    MetadataProvider provider = ((HiveManagedDatasetRepository) repo)
        .getMetadataProvider();
    Assert.assertTrue("Repo should be using a HCatalogManagedMetadataProvider",
        provider instanceof HiveManagedMetadataProvider);
    Assert.assertEquals("Repository URI", new URI("repo:hive"), repo.getUri());
  }

View Full Code Here

    Assert.assertEquals("Repository URI", new URI("repo:hive"), repo.getUri());
  }


  @Test
  public void testManagedURIWithHostAndPort() {
    DatasetRepository repo = DatasetRepositories
        .repositoryFor("repo:hive://meta-host:1234");
    Assert.assertNotNull("Received a repository", repo);
    Assert.assertTrue("Repo should be a HCatalogDatasetRepository",
        repo instanceof HiveManagedDatasetRepository);
    MetadataProvider provider = ((HiveManagedDatasetRepository) repo)
        .getMetadataProvider();
    Assert.assertTrue("Repo should be using a HCatalogManagedMetadataProvider",
        provider instanceof HiveManagedMetadataProvider);
    Assert.assertEquals("Repository URI",
        URI.create("repo:hive://meta-host:1234"), repo.getUri());
  }

View Full Code Here

  }


  @Test
  public void testExternalURILocalFileSystem() {
    URI repoUri = URI.create("repo:hive:/tmp/hive-repo");
    DatasetRepository repo = DatasetRepositories.repositoryFor(repoUri);


    Assert.assertNotNull("Received a repository", repo);
    org.junit.Assert.assertTrue("Repo should be a HCatalogExternalDatasetRepository",
        repo instanceof HiveExternalDatasetRepository);
    Assert.assertEquals("Repository URI", repoUri, repo.getUri());


    // verify location
    DatasetDescriptor created = repo.create("ns", "test",
        new DatasetDescriptor.Builder()
            .schemaLiteral("\"string\"")
            .build()).getDescriptor();
    Assert.assertEquals("Location should be in local FS",
        "file", created.getLocation().getScheme());

View Full Code Here

  }


  @Test
  public void testExternalOpaqueURILocalFileSystem() {
    URI repoUri = URI.create("repo:hive:/tmp/hive-repo");
    DatasetRepository repo = DatasetRepositories.repositoryFor(repoUri);


    Assert.assertNotNull("Received a repository", repo);
    org.junit.Assert.assertTrue("Repo should be a HCatalogExternalDatasetRepository",
        repo instanceof HiveExternalDatasetRepository);
    Assert.assertEquals("Repository URI", repoUri, repo.getUri());


    // verify location
    DatasetDescriptor created = repo.create("ns", "test",
        new DatasetDescriptor.Builder()
            .schemaLiteral("\"string\"")
            .build()).getDescriptor();
    Assert.assertEquals("Location should be in local FS",
        "file", created.getLocation().getScheme());

View Full Code Here

        "The entity type can't be null, use Object.class to have the type"
        + " determined by the schema.");


    Pair<DatasetRepository, Map<String, String>> pair =
        Registration.lookupDatasetUri(URI.create(uri.getRawSchemeSpecificPart()));
    DatasetRepository repo = pair.first();
    Map<String, String> uriOptions = pair.second();


    Dataset<E> dataset = repo.load(
        uriOptions.get(URIBuilder.NAMESPACE_OPTION),
        uriOptions.get(URIBuilder.DATASET_NAME_OPTION), type);


    if (isView) {
      return Datasets.<E, V> view(dataset, uriOptions);

View Full Code Here

        "The entity type can't be null, use Object.class to have the type"
        + " determined by the schema.");


    Pair<DatasetRepository, Map<String, String>> pair =
        Registration.lookupDatasetUri(URI.create(uri.getRawSchemeSpecificPart()));
    DatasetRepository repo = pair.first();
    Map<String, String> uriOptions = pair.second();


    if (descriptor.getLocation() == null && uriOptions.containsKey("location")) {
      descriptor = new DatasetDescriptor.Builder(descriptor)
          .location(uriOptions.get("location"))
          .build();
    }


    Dataset<E> dataset = repo.create(
        uriOptions.get(URIBuilder.NAMESPACE_OPTION),
        uriOptions.get(URIBuilder.DATASET_NAME_OPTION), descriptor, type);


    if (isView) {
      return Datasets.<E, V> view(dataset, uriOptions);

View Full Code Here

0 1 2 3 4 5 6 7

TOP

Related Classes of org.kitesdk.data.spi.DatasetRepository

org.kitesdk.cli.commands.DeleteDatasetCommand

org.kitesdk.cli.commands.TestCopyCommandCluster

org.kitesdk.cli.commands.TestCopyCommandLocal

org.kitesdk.cli.commands.TestCreateDatasetCommandCluster

org.kitesdk.cli.commands.TestTransformCommandCluster

org.kitesdk.cli.commands.TestTransformCommandLocal

org.kitesdk.data.Datasets

org.kitesdk.data.hbase.TestHBaseDatasetURIs

org.kitesdk.data.hbase.TestHBaseRepositoryURIs

org.kitesdk.data.mapreduce.DatasetKeyOutputFormat

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.