Package org.kitesdk.data.spi

Examples of org.kitesdk.data.spi.DatasetRepository

159
160
161
162
163
164
165
166
167
168
169
170
171
    command.datasets = Lists.newArrayList(source, dest);

    int rc = command.run();
    Assert.assertEquals("Should return success", 0, rc);

    DatasetRepository repo = DatasetRepositories.repositoryFor("repo:" + repoUri);
    FileSystemDataset<GenericData.Record> ds =
        (FileSystemDataset<GenericData.Record>) repo.<GenericData.Record>
            load("default", dest);
    int size = DatasetTestUtilities.datasetSize(ds);
    Assert.assertEquals("Should contain copied records", 6, size);

    Assert.assertEquals("Should produce 3 files",
View Full Code Here
214
215
216
217
218
219
220
221
222
223
224
225
226
        .build(), Object.class);

    int rc = command.run();
    Assert.assertEquals("Should return success", 0, rc);

    DatasetRepository repo = DatasetRepositories.repositoryFor("repo:" + repoUri);
    FileSystemDataset<GenericData.Record> ds =
        (FileSystemDataset<GenericData.Record>) repo.<GenericData.Record>
            load("default", "dest_partitioned");
    int size = DatasetTestUtilities.datasetSize(ds);
    Assert.assertEquals("Should contain copied records", 6, size);

    Assert.assertEquals("Should produce 2 partitions",
View Full Code Here
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
    }
  }

  @Test
  public void testExternal() {
    DatasetRepository repo = DatasetRepositories.repositoryFor("repo:hive:/tmp/data");
    repo.delete("ns", "test");
    repo.create("ns", "test", descriptor);

    Dataset<GenericRecord> ds = Datasets.load("dataset:hive:/tmp/data/ns/test");

    Assert.assertNotNull("Should load dataset", ds);
    Assert.assertTrue(ds instanceof FileSystemDataset);
    Assert.assertEquals("Locations should match",
        URI.create("hdfs://" + hdfsAuth + "/tmp/data/ns/test"),
        ds.getDescriptor().getLocation());
    Assert.assertEquals("Descriptors should match",
        repo.load("ns", "test").getDescriptor(), ds.getDescriptor());

    repo.delete("ns", "test");
  }
View Full Code Here
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
    repo.delete("ns", "test");
  }

  @Test
  public void testExternalRoot() {
    DatasetRepository repo = DatasetRepositories.repositoryFor("repo:hive:/");
    repo.delete("ns", "test");
    repo.create("ns", "test", descriptor);

    Dataset<GenericRecord> ds = Datasets.load("dataset:hive:/ns/test");

    Assert.assertNotNull("Should load dataset", ds);
    Assert.assertTrue(ds instanceof FileSystemDataset);
    Assert.assertEquals("Locations should match",
        URI.create("hdfs://" + hdfsAuth + "/ns/test"),
        ds.getDescriptor().getLocation());
    Assert.assertEquals("Descriptors should match",
        repo.load("ns", "test").getDescriptor(), ds.getDescriptor());

    repo.delete("ns", "test");
  }
View Full Code Here
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
    repo.delete("ns", "test");
  }

  @Test
  public void testExternalRelative() {
    DatasetRepository repo = DatasetRepositories.repositoryFor("repo:hive:data");
    repo.delete("ns", "test");
    repo.create("ns", "test", descriptor);

    Dataset<GenericRecord> ds = Datasets.load("dataset:hive:data/ns/test");

    Assert.assertNotNull("Should load dataset", ds);
    Assert.assertTrue(ds instanceof FileSystemDataset);
    Path cwd = getDFS().makeQualified(new Path("."));
    Assert.assertEquals("Locations should match",
        new Path(cwd, "data/ns/test").toUri(), ds.getDescriptor().getLocation());
    Assert.assertEquals("Descriptors should match",
        repo.load("ns", "test").getDescriptor(), ds.getDescriptor());

    repo.delete("ns", "test");
  }
View Full Code Here
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
    repo.delete("ns", "test");
  }

  @Test
  public void testManaged() {
    DatasetRepository repo = DatasetRepositories.repositoryFor("repo:hive");
    repo.delete("ns", "test");
    repo.create("ns", "test", descriptor);

    Dataset<GenericRecord> ds = Datasets.load("dataset:hive?dataset=test&namespace=ns");

    Assert.assertNotNull("Should load dataset", ds);
    Assert.assertTrue(ds instanceof FileSystemDataset);
    Assert.assertEquals("Descriptors should match",
        repo.load("ns", "test").getDescriptor(), ds.getDescriptor());

    repo.delete("ns", "test");
  }
View Full Code Here
133
134
135
136
137
138
139
140
141
142
    Assert.assertEquals("string", columns.get(10).getType());
  }

  @Test
  public void testManagedDatasetCreation() {
    DatasetRepository repo = DatasetRepositories.repositoryFor("repo:hive");
    repo.delete("ns", "records"); // ensure it does not already exist
    repo.create("ns", "records", descriptor);
    repo.delete("ns", "records"); // clean up
  }
View Full Code Here
141
142
143
144
145
146
147
148
149
150
    repo.delete("ns", "records"); // clean up
  }

  @Test
  public void testExternalDatasetCreation() {
    DatasetRepository repo = DatasetRepositories.repositoryFor("repo:hive:target/");
    repo.delete("ns", "records"); // ensure it does not already exist
    repo.create("ns", "records", descriptor);
    repo.delete("ns", "records"); // clean up
  }
View Full Code Here
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
    super(console);
  }

  @Override
  public int run() throws IOException {
    DatasetRepository repo = getDatasetRepository();

    if (datasets == null || datasets.isEmpty()) {
      throw new IllegalArgumentException("No dataset names were specified.");
    }

    for (String datasetUriOrName : datasets) {
      if (isDataUri(datasets.get(0))) {
        Datasets.delete(datasetUriOrName);
      } else {
        repo.delete(namespace, datasetUriOrName);
      }
      console.debug("Deleted dataset {}", datasetUriOrName);
    }

    return 0;
View Full Code Here
54
55
56
57
58
59
60
61
62
    } else {
      LOG.warn(
          "kite.datasetName is deprecated, instead use kite.uri=<dataset-uri>");
      Preconditions.checkArgument(datasetName != null,
          "kite.datasetName is required if kite.uri is not used");
      DatasetRepository repo = getDatasetRepository();
      repo.delete(datasetNamespace, datasetName);
    }
  }
View Full Code Here
TOP

Related Classes of org.kitesdk.data.spi.DatasetRepository

  • org.kitesdk.cli.commands.DeleteDatasetCommand
  • org.kitesdk.cli.commands.TestCopyCommandCluster
  • org.kitesdk.cli.commands.TestCopyCommandLocal
  • org.kitesdk.cli.commands.TestCreateDatasetCommandCluster
  • org.kitesdk.cli.commands.TestTransformCommandCluster
  • org.kitesdk.cli.commands.TestTransformCommandLocal
  • org.kitesdk.data.Datasets
  • org.kitesdk.data.hbase.TestHBaseDatasetURIs
  • org.kitesdk.data.hbase.TestHBaseRepositoryURIs
  • org.kitesdk.data.mapreduce.DatasetKeyOutputFormat
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.