Examples of org.kitesdk.data.spi.DatasetRepository

org.kitesdk.data.spi.DatasetRepository

A logical repository (storage system) of {@link Dataset}s.

Implementations of {@code DatasetRepository} are storage systems that containzero or more {@link Dataset}s. A repository acts as a factory, as well as a registry, of datasets. You can {@link #create(String,String,DatasetDescriptor)} anew {@link Dataset} with a name and schema, or retrieve a handle to anexisting dataset, by name, by way of the {@link #load(String,String)} method. Whilenot expressly forbidden, most repositories are expected to support only a single concrete {@link Dataset} implementation.

No guarantees are made as to the durability, reliability, or availability of the underlying storage. That is, a {@code DatasetRepository} could be ondisk, in memory, or some combination. See the implementation class for details about the guarantees it provides.

Implementations of {@link DatasetRepository} are immutable.
@see Dataset @see DatasetDescriptor

    Registration.register(
        new URIPattern("mock::id"), new URIPattern("mock::id"),
        new OptionBuilder<DatasetRepository>() {
          @Override
          public DatasetRepository getFromOptions(Map<String, String> options) {
            DatasetRepository repo = mock(DatasetRepository.class);
            repos.put(options.get("id"), repo);
            return repo;
          }
        }
    );

View Full Code Here

  }


  @Test
  public void testLocalRelative() throws URISyntaxException {
    URI repositoryUri = new URI("repo:file:target/dsr-repo-test");
    DatasetRepository repository = DatasetRepositories.repositoryFor(repositoryUri);


    // We only do the deeper implementation checks one per combination.
    Assert.assertNotNull("Received a repository", repository);
    Assert.assertTrue("Repo is a FileSystem repo",
        repository instanceof FileSystemDatasetRepository);
    MetadataProvider provider = ((FileSystemDatasetRepository) repository)
        .getMetadataProvider();
    Assert.assertTrue("Repo is using a FileSystemMetadataProvider",
        provider instanceof FileSystemMetadataProvider);
    FileSystemMetadataProvider fsProvider = (FileSystemMetadataProvider) provider;
    Assert.assertTrue("FileSystem is a LocalFileSystem",
        fsProvider.getFileSytem() instanceof LocalFileSystem);
    Path expected = fsProvider.getFileSytem().makeQualified(
        new Path("target/dsr-repo-test"));
    Assert.assertEquals("Root directory should be the correct qualified path",
        expected, fsProvider.getRootDirectory());
    Assert.assertEquals("Repository URI scheme", "repo", repository.getUri()
        .getScheme());
    Assert.assertEquals("Repository URI scheme", expected.toUri(),
        new URI(repository.getUri().getSchemeSpecificPart()));
  }

View Full Code Here

  }


  @Test
  public void testLocalAbsolute() throws URISyntaxException {
    URI repositoryUri = new URI("repo:file:///tmp/dsr-repo-test");
    DatasetRepository repository = DatasetRepositories.repositoryFor(repositoryUri);


    FileSystemMetadataProvider provider = (FileSystemMetadataProvider)
        ((FileSystemDatasetRepository) repository).getMetadataProvider();
    Assert.assertEquals("Root directory should be the correct qualified path",
        new Path("file:/tmp/dsr-repo-test"), provider.getRootDirectory());
    Assert.assertEquals("Repository URI", repositoryUri, repository.getUri());
  }

View Full Code Here


  @Test
  public void testHdfsAbsolute() throws URISyntaxException {
    URI hdfsUri = getDFS().getUri();
    URI repositoryUri = new URI("repo:hdfs://" + hdfsUri.getAuthority() + "/tmp/dsr-repo-test");
    DatasetRepository repository = DatasetRepositories.repositoryFor(repositoryUri);


    // We only do the deeper implementation checks one per combination.
    Assert.assertNotNull("Received a repository", repository);
    Assert.assertTrue("Repo is a FileSystem repo",
        repository instanceof FileSystemDatasetRepository);
    MetadataProvider provider = ((FileSystemDatasetRepository) repository)
        .getMetadataProvider();
    Assert.assertTrue("Repo is using a FileSystemMetadataProvider",
        provider instanceof FileSystemMetadataProvider);
    FileSystemMetadataProvider fsProvider = (FileSystemMetadataProvider) provider;
    Assert.assertTrue("FileSystem is a DistributedFileSystem",
      fsProvider.getFileSytem() instanceof DistributedFileSystem);
    Path expected = fsProvider.getFileSytem().makeQualified(
        new Path("/tmp/dsr-repo-test"));
    Assert.assertEquals("Root directory should be the correct qualified path",
        expected, fsProvider.getRootDirectory());
    Assert.assertEquals("Repository URI", repositoryUri, repository.getUri());
  }

View Full Code Here

        .build();
  }


  @Test
  public void testAbsolute() {
    DatasetRepository repo = DatasetRepositories
        .repositoryFor("repo:hdfs://" + hdfsAuth + "/tmp/data");
    repo.delete("ns", "test");
    repo.create("ns", "test", descriptor);


    Dataset<Object> ds = Datasets.<Object, Dataset<Object>>
        load("dataset:hdfs://" + hdfsAuth + "/tmp/data/ns/test", Object.class);


    Assert.assertNotNull("Should load dataset", ds);
    Assert.assertTrue(ds instanceof FileSystemDataset);
    Assert.assertEquals("Locations should match",
        URI.create("hdfs://" + hdfsAuth + "/tmp/data/ns/test"),
        ds.getDescriptor().getLocation());
    Assert.assertEquals("Descriptors should match",
        repo.load("ns", "test").getDescriptor(), ds.getDescriptor());
    Assert.assertEquals("Should report correct namespace",
        "ns", ds.getNamespace());
    Assert.assertEquals("Should report correct name",
        "test", ds.getName());


    repo.delete("ns", "test");
  }

View Full Code Here

    repo.delete("ns", "test");
  }


  @Test
  public void testAbsoluteRoot() {
    DatasetRepository repo = DatasetRepositories
        .repositoryFor("repo:hdfs://" + hdfsAuth + "/");
    repo.delete("ns", "test");
    repo.create("ns", "test", descriptor);


    Dataset<Object> ds = Datasets.<Object, Dataset<Object>>
        load("dataset:hdfs://" + hdfsAuth + "/ns/test",
        Object.class);


    Assert.assertNotNull("Should load dataset", ds);
    Assert.assertTrue(ds instanceof FileSystemDataset);
    Assert.assertEquals("Locations should match",
        URI.create("hdfs://" + hdfsAuth + "/ns/test"),
        ds.getDescriptor().getLocation());
    Assert.assertEquals("Descriptors should match",
        repo.load("ns", "test").getDescriptor(), ds.getDescriptor());
    Assert.assertEquals("Should report correct namespace",
        "ns", ds.getNamespace());
    Assert.assertEquals("Should report correct name",
        "test", ds.getName());


    repo.delete("ns", "test");
  }

View Full Code Here

    repo.delete("ns", "test");
  }


  @Test
  public void testRelative() {
    DatasetRepository repo = DatasetRepositories
        .repositoryFor("repo:hdfs://" + hdfsAuth + "/data?absolute=false");
    repo.delete("ns", "test");
    repo.create("ns", "test", descriptor);


    Dataset<Object> ds = Datasets.<Object, Dataset<Object>>
        load("dataset:hdfs://" + hdfsAuth + "/data/ns/test?absolute=false",
        Object.class);


    Assert.assertNotNull("Should load dataset", ds);
    Assert.assertTrue(ds instanceof FileSystemDataset);
    Path cwd = getDFS().makeQualified(new Path("."));
    Assert.assertEquals("Locations should match",
        new Path(cwd, "data/ns/test").toUri(), ds.getDescriptor().getLocation());
    Assert.assertEquals("Descriptors should match",
        repo.load("ns", "test").getDescriptor(), ds.getDescriptor());
    Assert.assertEquals("Should report correct namespace",
        "ns", ds.getNamespace());
    Assert.assertEquals("Should report correct name",
        "test", ds.getName());


    repo.delete("ns", "test");
  }

View Full Code Here

  public void testAbsoluteWebHdfs() {
    Assume.assumeTrue(!Hadoop.isHadoop1());


    String webhdfsAuth = getConfiguration().get(
        DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY);
    DatasetRepository repo = DatasetRepositories
        .repositoryFor("repo:webhdfs://" + webhdfsAuth + "/tmp/data");
    repo.delete("ns", "test");
    repo.create("ns", "test", descriptor);


    Dataset<Object> ds = Datasets.<Object, Dataset<Object>>
        load("dataset:webhdfs://" + webhdfsAuth + "/tmp/data/ns/test", Object.class);


    Assert.assertNotNull("Should load dataset", ds);
    Assert.assertTrue(ds instanceof FileSystemDataset);
    Assert.assertEquals("Locations should match",
        URI.create("webhdfs://" + webhdfsAuth + "/tmp/data/ns/test"),
        ds.getDescriptor().getLocation());
    Assert.assertEquals("Descriptors should match",
        repo.load("ns", "test").getDescriptor(), ds.getDescriptor());
    Assert.assertEquals("Should report correct namespace",
        "ns", ds.getNamespace());
    Assert.assertEquals("Should report correct name",
        "test", ds.getName());


    repo.delete("ns", "test");
  }

View Full Code Here

    command.datasets = Lists.newArrayList(source, dest);


    int rc = command.run();
    Assert.assertEquals("Should return success", 0, rc);


    DatasetRepository repo = DatasetRepositories.repositoryFor("repo:" + repoUri);
    int size = DatasetTestUtilities.datasetSize(repo.load("default", dest));
    Assert.assertEquals("Should contain copied records", 6, size);


    verify(console).info("Added {} records to \"{}\"", 6l, dest);
    verifyNoMoreInteractions(console);
  }

View Full Code Here

    command.datasets = Lists.newArrayList(source, dest);


    int rc = command.run();
    Assert.assertEquals("Should return success", 0, rc);


    DatasetRepository repo = DatasetRepositories.repositoryFor("repo:" + repoUri);
    FileSystemDataset<GenericData.Record> ds =
        (FileSystemDataset<GenericData.Record>) repo.<GenericData.Record>
            load("default", dest);
    int size = DatasetTestUtilities.datasetSize(ds);
    Assert.assertEquals("Should contain copied records", 6, size);


    Assert.assertEquals("Should produce 1 files",

View Full Code Here

0 1 2 3 4 5 6 7

TOP

Related Classes of org.kitesdk.data.spi.DatasetRepository

org.kitesdk.cli.commands.DeleteDatasetCommand

org.kitesdk.cli.commands.TestCopyCommandCluster

org.kitesdk.cli.commands.TestCopyCommandLocal

org.kitesdk.cli.commands.TestCreateDatasetCommandCluster

org.kitesdk.cli.commands.TestTransformCommandCluster

org.kitesdk.cli.commands.TestTransformCommandLocal

org.kitesdk.data.Datasets

org.kitesdk.data.hbase.TestHBaseDatasetURIs

org.kitesdk.data.hbase.TestHBaseRepositoryURIs

org.kitesdk.data.mapreduce.DatasetKeyOutputFormat

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.