Examples of org.kitesdk.data.spi.DatasetRepository.create()

Class org.kitesdk.data.spi.DatasetRepository

Examples of org.kitesdk.data.spi.DatasetRepository.create()

org.kitesdk.data.spi.DatasetRepository.create()
/ Create a {@link Dataset} with the supplied {@code descriptor}. Depending on the underlying dataset storage, some schema types or configurations might not be supported. If you supply an illegal schema, the implementing class throws an exception. It is illegal to create more than one dataset with the same name. If you provide a duplicate name, the implementing class throws an exception. @param namespace A namespace, or logical group name, for the dataset. @param name The fully qualified dataset name @param descriptor A descriptor that describes the schema and otherproperties of the dataset @return The newly created dataset @throws IllegalArgumentException if {@code name} or {@code descriptor}is {@code null} @throws DatasetExistsException if a {@code Dataset} named {@code name}already exists. @throws ConcurrentSchemaModificationException if the {@code Dataset}schema is updated concurrently. @throws IncompatibleSchemaException if the schema is not compatible with existing datasets with shared storage (for example, in the same HBase table). @since 0.17.0


  @Test
  public void testBasic() {
    DatasetRepository repo = DatasetRepositories.repositoryFor(repositoryUri);
    repo.delete("default", "test");
    repo.create("default", "test", descriptor);


    RandomAccessDataset<Object> ds = Datasets
  .<Object, RandomAccessDataset<Object>>load(URI.create("dataset:hbase:" + zk + "/test"), Object.class);


    Assert.assertNotNull("Should load dataset", ds);

View Full Code Here

  @SuppressWarnings("unchecked")
  private static <E> Dataset<E> createJobDataset(JobContext jobContext) {
    Dataset<Object> dataset = load(jobContext).getDataset();
    String jobDatasetName = getJobDatasetName(jobContext);
    DatasetRepository repo = getDatasetRepository(jobContext);
    return repo.create(TEMP_NAMESPACE, jobDatasetName,
        copy(dataset.getDescriptor()),
        DatasetKeyOutputFormat.<E>getType(jobContext));
  }


  private static <E> Dataset<E> loadJobDataset(JobContext jobContext) {

View Full Code Here

    DatasetRepository repo = getDatasetRepository(taskContext);
    Dataset<E> jobDataset = loadJobDataset(taskContext);
    if (repo.exists(TEMP_NAMESPACE, taskAttemptDatasetName)) {
      return repo.load(TEMP_NAMESPACE, taskAttemptDatasetName);
    } else {
      return repo.create(TEMP_NAMESPACE, taskAttemptDatasetName,
          copy(jobDataset.getDescriptor()));
    }
  }


  private static void deleteTaskAttemptDataset(TaskAttemptContext taskContext) {

View Full Code Here

  @Test
  public void testAbsolute() {
    DatasetRepository repo = DatasetRepositories
        .repositoryFor("repo:hdfs://" + hdfsAuth + "/tmp/data");
    repo.delete("ns", "test");
    repo.create("ns", "test", descriptor);


    Dataset<Object> ds = Datasets.<Object, Dataset<Object>>
        load("dataset:hdfs://" + hdfsAuth + "/tmp/data/ns/test", Object.class);


    Assert.assertNotNull("Should load dataset", ds);

View Full Code Here

  @Test
  public void testAbsoluteRoot() {
    DatasetRepository repo = DatasetRepositories
        .repositoryFor("repo:hdfs://" + hdfsAuth + "/");
    repo.delete("ns", "test");
    repo.create("ns", "test", descriptor);


    Dataset<Object> ds = Datasets.<Object, Dataset<Object>>
        load("dataset:hdfs://" + hdfsAuth + "/ns/test",
        Object.class);

View Full Code Here

  @Test
  public void testRelative() {
    DatasetRepository repo = DatasetRepositories
        .repositoryFor("repo:hdfs://" + hdfsAuth + "/data?absolute=false");
    repo.delete("ns", "test");
    repo.create("ns", "test", descriptor);


    Dataset<Object> ds = Datasets.<Object, Dataset<Object>>
        load("dataset:hdfs://" + hdfsAuth + "/data/ns/test?absolute=false",
        Object.class);

View Full Code Here

    String webhdfsAuth = getConfiguration().get(
        DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY);
    DatasetRepository repo = DatasetRepositories
        .repositoryFor("repo:webhdfs://" + webhdfsAuth + "/tmp/data");
    repo.delete("ns", "test");
    repo.create("ns", "test", descriptor);


    Dataset<Object> ds = Datasets.<Object, Dataset<Object>>
        load("dataset:webhdfs://" + webhdfsAuth + "/tmp/data/ns/test", Object.class);


    Assert.assertNotNull("Should load dataset", ds);

View Full Code Here


  @Test
  public void testExternal() {
    DatasetRepository repo = DatasetRepositories.repositoryFor("repo:hive:/tmp/data");
    repo.delete("ns", "test");
    repo.create("ns", "test", descriptor);


    Dataset<GenericRecord> ds = Datasets.load("dataset:hive:/tmp/data/ns/test");


    Assert.assertNotNull("Should load dataset", ds);
    Assert.assertTrue(ds instanceof FileSystemDataset);

View Full Code Here


  @Test
  public void testExternalRoot() {
    DatasetRepository repo = DatasetRepositories.repositoryFor("repo:hive:/");
    repo.delete("ns", "test");
    repo.create("ns", "test", descriptor);


    Dataset<GenericRecord> ds = Datasets.load("dataset:hive:/ns/test");


    Assert.assertNotNull("Should load dataset", ds);
    Assert.assertTrue(ds instanceof FileSystemDataset);

View Full Code Here


  @Test
  public void testExternalRelative() {
    DatasetRepository repo = DatasetRepositories.repositoryFor("repo:hive:data");
    repo.delete("ns", "test");
    repo.create("ns", "test", descriptor);


    Dataset<GenericRecord> ds = Datasets.load("dataset:hive:data/ns/test");


    Assert.assertNotNull("Should load dataset", ds);
    Assert.assertTrue(ds instanceof FileSystemDataset);

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.