Package org.kitesdk.data

Examples of org.kitesdk.data.DatasetDescriptor

28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
  @Test
  public void testRoundTripDescriptor() throws Exception {
    String namespace = "test_ns";
    String name = "test_table";
    DatasetDescriptor original = new DatasetDescriptor.Builder()
        .schemaUri("resource:schema/user.avsc")
        .location("file:/tmp/data/test_table")
        .build();
    boolean external = true;
    Table table = HiveUtils.tableForDescriptor(namespace, name, original, external);

    Configuration conf = new HiveConf();
    DatasetDescriptor result = HiveUtils.descriptorForTable(conf, table);
    assertEquals(original, result);
  }
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
  @Test
  public void testRoundTripDescriptorWithCompressionType() throws Exception {
    String namespace = "test_ns";
    String name = "test_table";
    DatasetDescriptor original = new DatasetDescriptor.Builder()
        .schemaUri("resource:schema/user.avsc")
        .location("file:/tmp/data/test_table")
        .compressionType(CompressionType.Deflate)
        .build();
    boolean external = true;
    Table table = HiveUtils.tableForDescriptor(namespace, name, original, external);

    Configuration conf = new HiveConf();
    DatasetDescriptor result = HiveUtils.descriptorForTable(conf, table);
    assertEquals(original, result);
  }
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
  @Test
  public void testRoundTripDescriptorNoCompressionProperty() throws Exception {
    String namespace = "test_ns";
    String name = "test_table";
    DatasetDescriptor original = new DatasetDescriptor.Builder()
        .schemaUri("resource:schema/user.avsc")
        .location("file:/tmp/data/test_table")
        .build();
    boolean external = true;
    Table table = HiveUtils.tableForDescriptor(namespace, name, original, external);
    assertEquals("snappy", table.getParameters().get("kite.compression.type"));
    table.getParameters().remove("kite.compression.type");

    Configuration conf = new HiveConf();
    DatasetDescriptor result = HiveUtils.descriptorForTable(conf, table);
    assertEquals(original, result);
  }
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
  @Test
  public void testGeneric() throws IOException {
    String datasetName = tableName + ".TestGenericEntity";

    DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
        .schemaLiteral(testGenericEntity)
        .build();

    Dataset<GenericRecord> inputDataset = repo.create("default", "in", descriptor);
    Dataset<GenericRecord> outputDataset = repo.create("default", datasetName, descriptor);
View Full Code Here
112
113
114
115
116
117
118
119
120
121
122
  @Test
  public void testSourceView() throws IOException {
    String datasetName = tableName + ".TestGenericEntity";

    DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
        .schemaLiteral(testGenericEntity)
        .build();

    Dataset<GenericRecord> inputDataset = repo.create("default", "in", descriptor);
    Dataset<GenericRecord> outputDataset = repo.create("default", datasetName, descriptor);
View Full Code Here
122
123
124
125
126
127
128
129
130
131
132
    Schema datasetSchema = target.getDataset().getDescriptor().getSchema();

    // TODO: replace this with a temporary Dataset from a FS repo
    // TODO: CDK-92: always use GenericRecord?

    DatasetDescriptor csvDescriptor = new DatasetDescriptor.Builder()
        .location(source.toUri())
        .schema(ColumnMappingParser.removeEmbeddedMapping(
            PartitionStrategyParser.removeEmbeddedStrategy(datasetSchema)))
        .format("csv")
        .build();
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
    final String NAME2 = "test2";

    PartitionStrategy partitionStrategy = new PartitionStrategy.Builder()
        .hash("username", 2).build();

    DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
        .schema(testSchema)
        .partitionStrategy(partitionStrategy)
        .build();

    Dataset<GenericRecord> dataset = repo.create(NAMESPACE, NAME2, descriptor);
View Full Code Here
118
119
120
121
122
123
124
125
126
127
128
    final String NAME3 = "test3";

    PartitionStrategy partitionStrategy = new PartitionStrategy.Builder()
        .hash("username", 2).build();

    DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
        .schema(testSchema)
        .partitionStrategy(partitionStrategy)
        .build();

    Dataset<GenericRecord> dataset = repo.create(NAMESPACE, NAME2, descriptor);
View Full Code Here
169
170
171
172
173
174
175
176
177
178
179
180
181
182
        new DatasetDescriptor.Builder(descriptor)
            .property("added.property", "true")
            .build());
    Assert.assertNotNull("Update should succeed", updated);

    DatasetDescriptor stored =
        HiveUtils.descriptorForTable(conf, metastore.getTable("default", "test"));

    Assert.assertEquals("Should update default.test descriptor",
        stored, updated.getDescriptor());

    Assert.assertEquals("Added property should be present",
        stored.getProperty("added.property"), "true");
  }
View Full Code Here
188
189
190
191
192
193
194
195
196
197
198
199
200
201
        new DatasetDescriptor.Builder(descriptor)
            .property("added.property", "true")
            .build());
    Assert.assertNotNull("Update should succeed", updated);

    DatasetDescriptor stored =
        HiveUtils.descriptorForTable(conf, metastore.getTable("default", "test"));

    Assert.assertEquals("Should update default.test descriptor",
        stored, updated.getDescriptor());

    Assert.assertEquals("Added property should be present",
        stored.getProperty("added.property"), "true");
  }
View Full Code Here
TOP

Related Classes of org.kitesdk.data.DatasetDescriptor

  • org.apache.avro.Schema
  • org.apache.flume.sink.kite.DatasetSink
  • org.apache.flume.sink.kite.TestDatasetSink
  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.Path
  • org.apache.sqoop.mapreduce.ParquetJob
  • org.kitesdk.cli.commands.CreateColumnMappingCommand
  • org.kitesdk.cli.commands.CreateDatasetCommand
  • org.kitesdk.cli.commands.CreatePartitionStrategyCommand
  • org.kitesdk.cli.commands.CSVImportCommand
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.