Package org.kitesdk.data

Examples of org.kitesdk.data.DatasetDescriptor$Builder

113
114
115
116
117
118
119
120
    provider.delete("old_1", "new_3");
  }

  @Test
  public void testLoad() {
    DatasetDescriptor loaded = provider.load("default", "old_1");
    Assert.assertNotNull("Should find old layout datasets", loaded);
  }
View Full Code Here
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
        provider.exists("default", "old_1"));
  }

  @Test
  public void testUpdate() throws IOException {
    DatasetDescriptor updated = new DatasetDescriptor.Builder(descriptor)
        .property("parquet.block.size", "1024")
        .build();

    DatasetDescriptor saved = provider.update("default", "old_2", updated);
    Assert.assertNotNull("Should find saved metadata", saved);
    Assert.assertEquals("Should update old dataset successfully",
        updated.getProperty("parquet.block.size"),
        saved.getProperty("parquet.block.size"));

    DatasetDescriptor loaded = provider.load("default", "old_2");
    Assert.assertNotNull("Should find saved metadata", loaded);
    Assert.assertEquals("Should make changes on disk",
        updated.getProperty("parquet.block.size"),
        loaded.getProperty("parquet.block.size"));

    Assert.assertFalse("Should not move metadata to new location",
        local.exists(new Path(root, new Path("default", "old_2"))));
  }
View Full Code Here
114
115
116
117
118
119
120
121
122
123
124
125
    out.close();
  }

  @Override
  public DatasetReader<GenericData.Record> newReader() throws IOException {
    final DatasetDescriptor desc = new DatasetDescriptor.Builder()
        .property("kite.csv.has-header", "true")
        .schema(VALIDATOR_SCHEMA)
        .build();
    return new CSVFileReader<GenericData.Record>(localfs, validatorFile, desc,
        DataModelUtil.accessor(GenericData.Record.class, desc.getSchema()));
  }
View Full Code Here
142
143
144
145
146
147
148
149
150
151
152
    };
  }

  @Test(expected = IllegalArgumentException.class)
  public void testRejectsNonRecordSchemas() {
    final DatasetDescriptor desc = new DatasetDescriptor.Builder()
        .schema(SchemaBuilder.array().items().stringType())
        .build();
    new CSVFileReader<GenericData.Record>(localfs, csvFile, desc,
        DataModelUtil.accessor(GenericData.Record.class, desc.getSchema()));
  }
View Full Code Here
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
        DataModelUtil.accessor(GenericData.Record.class, desc.getSchema()));
  }

  @Test
  public void testStringSchema() {
    final DatasetDescriptor desc = new DatasetDescriptor.Builder()
        .schema(STRINGS)
        .build();
    final CSVFileReader<GenericData.Record> reader =
        new CSVFileReader<GenericData.Record>(localfs, csvFile, desc,
            DataModelUtil.accessor(GenericData.Record.class, desc.getSchema()));

    reader.initialize();
    Assert.assertTrue(reader.hasNext());
    GenericData.Record rec = reader.next();
    Assert.assertEquals("str", rec.get(0));
View Full Code Here
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
    Assert.assertFalse(reader.hasNext());
  }

  @Test
  public void testTSV() {
    final DatasetDescriptor desc = new DatasetDescriptor.Builder()
        .property("kite.csv.delimiter", "\t")
        .property("kite.csv.lines-to-skip", "1")
        .schema(STRINGS)
        .build();
    final CSVFileReader<GenericData.Record> reader =
        new CSVFileReader<GenericData.Record>(localfs, tsvFile, desc,
            DataModelUtil.accessor(GenericData.Record.class, desc.getSchema()));

    reader.initialize();
    Assert.assertTrue(reader.hasNext());
    GenericData.Record rec = reader.next();
    Assert.assertEquals("str", rec.get(0));
View Full Code Here
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
    Assert.assertFalse(reader.hasNext());
  }

  @Test
  public void testTSVWithDeprecatedProperties() {
    final DatasetDescriptor desc = new DatasetDescriptor.Builder()
        .property("cdk.csv.delimiter", "\t")
        .property("cdk.csv.lines-to-skip", "1")
        .schema(STRINGS)
        .build();
    final CSVFileReader<GenericData.Record> reader =
        new CSVFileReader<GenericData.Record>(localfs, tsvFile, desc,
            DataModelUtil.accessor(GenericData.Record.class, desc.getSchema()));

    reader.initialize();
    Assert.assertTrue(reader.hasNext());
    GenericData.Record rec = reader.next();
    Assert.assertEquals("str", rec.get(0));
View Full Code Here
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
    Assert.assertFalse(reader.hasNext());
  }

  @Test
  public void testNormalSchema() {
    final DatasetDescriptor desc = new DatasetDescriptor.Builder()
        .schema(SCHEMA)
        .build();
    final CSVFileReader<GenericData.Record> reader =
        new CSVFileReader<GenericData.Record>(localfs, csvFile, desc,
            DataModelUtil.accessor(GenericData.Record.class, desc.getSchema()));

    reader.initialize();
    Assert.assertTrue(reader.hasNext());
    GenericData.Record rec = reader.next();
    Assert.assertEquals("str", rec.get(0));
View Full Code Here
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
    Assert.assertFalse(reader.hasNext());
  }

  @Test
  public void testReflectedRecords() {
    final DatasetDescriptor desc = new DatasetDescriptor.Builder()
        .schema(BEAN_SCHEMA)
        .build();
    final CSVFileReader<TestBean> reader =
        new CSVFileReader<TestBean>(localfs, csvFile, desc,
            DataModelUtil.accessor(TestBean.class, desc.getSchema()));

    reader.initialize();
    Assert.assertTrue(reader.hasNext());
    TestBean bean = reader.next();
    Assert.assertEquals("str", bean.myStr);
View Full Code Here
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
    Assert.assertFalse(reader.hasNext());
  }

  @Test
  public void testCustomGenericRecords() {
    final DatasetDescriptor desc = new DatasetDescriptor.Builder()
        .schema(SCHEMA)
        .build();
    final CSVFileReader<TestGenericRecord> reader =
        new CSVFileReader<TestGenericRecord>(localfs, csvFile, desc,
        DataModelUtil.accessor(TestGenericRecord.class, desc.getSchema()));

    reader.initialize();
    Assert.assertTrue(reader.hasNext());
    TestGenericRecord record = reader.next();
    Assert.assertEquals("str", record.get(0));
View Full Code Here
TOP

Related Classes of org.kitesdk.data.DatasetDescriptor$Builder

  • co.cask.tigon.sql.internal.DefaultGDATField
  • com.asakusafw.utils.java.model.syntax.Expression
  • com.asakusafw.utils.java.model.syntax.ModelFactory
  • com.asakusafw.utils.java.model.util.ImportBuilder
  • com.asakusafw.utils.java.model.util.TypeBuilder
  • com.fasterxml.jackson.databind.JsonNode
  • com.fasterxml.jackson.databind.ObjectMapper
  • com.google.gwt.autobean.shared.AutoBean.PropertyName
  • com.google.javascript.rhino.JSDocInfo
  • com.google.javascript.rhino.Node
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.