Examples of org.kitesdk.data.spi.DatasetRepository.load()

Class org.kitesdk.data.spi.DatasetRepository

Examples of org.kitesdk.data.spi.DatasetRepository.load()

org.kitesdk.data.spi.DatasetRepository.load()
Get the latest version of a named {@link Dataset}. If no dataset with the provided {@code name} exists, a {@link DatasetNotFoundException} is thrown. @param namespace A namespace, or logical group name, for the dataset. @param name The name of the dataset. @throws DatasetNotFoundException if there is no data set named {@code name} @since 0.17.0

  .<Object, RandomAccessDataset<Object>>load(URI.create("dataset:hbase:" + zk + "/test"), Object.class);


    Assert.assertNotNull("Should load dataset", ds);
    Assert.assertTrue(ds instanceof DaoDataset);
    Assert.assertEquals("Descriptors should match",
        repo.load("default", "test").getDescriptor(), ds.getDescriptor());


    repo.delete("default", "test");
  }


  @Test

View Full Code Here

        DatasetKeyOutputFormat.<E>getType(jobContext));
  }


  private static <E> Dataset<E> loadJobDataset(JobContext jobContext) {
    DatasetRepository repo = getDatasetRepository(jobContext);
    return repo.load(TEMP_NAMESPACE, getJobDatasetName(jobContext));
  }


  private static void deleteJobDataset(JobContext jobContext) {
    DatasetRepository repo = getDatasetRepository(jobContext);
    repo.delete(TEMP_NAMESPACE, getJobDatasetName(jobContext));

View Full Code Here

  private static <E> Dataset<E> loadOrCreateTaskAttemptDataset(TaskAttemptContext taskContext) {
    String taskAttemptDatasetName = getTaskAttemptDatasetName(taskContext);
    DatasetRepository repo = getDatasetRepository(taskContext);
    Dataset<E> jobDataset = loadJobDataset(taskContext);
    if (repo.exists(TEMP_NAMESPACE, taskAttemptDatasetName)) {
      return repo.load(TEMP_NAMESPACE, taskAttemptDatasetName);
    } else {
      return repo.create(TEMP_NAMESPACE, taskAttemptDatasetName,
          copy(jobDataset.getDescriptor()));
    }
  }

View Full Code Here

      DatasetRepository repo = getDatasetRepository(jobContext);
      boolean isTemp = repo instanceof TemporaryDatasetRepository;


      String jobDatasetName = getJobDatasetName(jobContext);
      View<E> targetView = load(jobContext);
      Dataset<E> jobDataset = repo.load(TEMP_NAMESPACE, jobDatasetName);
      ((Mergeable<Dataset<E>>) targetView.getDataset()).merge(jobDataset);


      if (isTemp) {
        ((TemporaryDatasetRepository) repo).delete();
      } else {

View Full Code Here

    @SuppressWarnings("unchecked")
    public void commitTask(TaskAttemptContext taskContext) throws IOException {
      DatasetRepository repo = getDatasetRepository(taskContext);
      boolean inTempRepo = repo instanceof TemporaryDatasetRepository;


      Dataset<E> jobDataset = repo.load(TEMP_NAMESPACE, getJobDatasetName(taskContext));
      String taskAttemptDatasetName = getTaskAttemptDatasetName(taskContext);
      if (repo.exists(TEMP_NAMESPACE, taskAttemptDatasetName)) {
        Dataset<E> taskAttemptDataset = repo.load(TEMP_NAMESPACE, taskAttemptDatasetName);
        ((Mergeable<Dataset<E>>) jobDataset).merge(taskAttemptDataset);
        if (!inTempRepo) {

View Full Code Here

      boolean inTempRepo = repo instanceof TemporaryDatasetRepository;


      Dataset<E> jobDataset = repo.load(TEMP_NAMESPACE, getJobDatasetName(taskContext));
      String taskAttemptDatasetName = getTaskAttemptDatasetName(taskContext);
      if (repo.exists(TEMP_NAMESPACE, taskAttemptDatasetName)) {
        Dataset<E> taskAttemptDataset = repo.load(TEMP_NAMESPACE, taskAttemptDatasetName);
        ((Mergeable<Dataset<E>>) jobDataset).merge(taskAttemptDataset);
        if (!inTempRepo) {
          repo.delete(TEMP_NAMESPACE, taskAttemptDatasetName);
        }
      }

View Full Code Here

    Assert.assertTrue(ds instanceof FileSystemDataset);
    Assert.assertEquals("Locations should match",
        URI.create("hdfs://" + hdfsAuth + "/tmp/data/ns/test"),
        ds.getDescriptor().getLocation());
    Assert.assertEquals("Descriptors should match",
        repo.load("ns", "test").getDescriptor(), ds.getDescriptor());
    Assert.assertEquals("Should report correct namespace",
        "ns", ds.getNamespace());
    Assert.assertEquals("Should report correct name",
        "test", ds.getName());

View Full Code Here

    Assert.assertTrue(ds instanceof FileSystemDataset);
    Assert.assertEquals("Locations should match",
        URI.create("hdfs://" + hdfsAuth + "/ns/test"),
        ds.getDescriptor().getLocation());
    Assert.assertEquals("Descriptors should match",
        repo.load("ns", "test").getDescriptor(), ds.getDescriptor());
    Assert.assertEquals("Should report correct namespace",
        "ns", ds.getNamespace());
    Assert.assertEquals("Should report correct name",
        "test", ds.getName());

View Full Code Here

    Assert.assertTrue(ds instanceof FileSystemDataset);
    Path cwd = getDFS().makeQualified(new Path("."));
    Assert.assertEquals("Locations should match",
        new Path(cwd, "data/ns/test").toUri(), ds.getDescriptor().getLocation());
    Assert.assertEquals("Descriptors should match",
        repo.load("ns", "test").getDescriptor(), ds.getDescriptor());
    Assert.assertEquals("Should report correct namespace",
        "ns", ds.getNamespace());
    Assert.assertEquals("Should report correct name",
        "test", ds.getName());

View Full Code Here

    Assert.assertTrue(ds instanceof FileSystemDataset);
    Assert.assertEquals("Locations should match",
        URI.create("webhdfs://" + webhdfsAuth + "/tmp/data/ns/test"),
        ds.getDescriptor().getLocation());
    Assert.assertEquals("Descriptors should match",
        repo.load("ns", "test").getDescriptor(), ds.getDescriptor());
    Assert.assertEquals("Should report correct namespace",
        "ns", ds.getNamespace());
    Assert.assertEquals("Should report correct name",
        "test", ds.getName());

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.