Examples of DatasetDescriptor

com.cloudera.cdk.data.DatasetDescriptor

The structural definition of a {@link Dataset}.

Each {@code Dataset} has an associated {@link Schema} and optional{@link PartitionStrategy} defined at the time of creation. Instances of thisclass are used to hold this information. Users are strongly encouraged to use the inner {@link Builder} to create new instances.
org.kitesdk.data.DatasetDescriptor

The structural definition of a {@link Dataset}.

Each {@code Dataset} has an associated {@link Schema} and optional{@link PartitionStrategy} defined at the time of creation. You use instancesof this class to hold this information. You are strongly encouraged to use the inner {@link Builder} to create new instances.

Examples of org.kitesdk.data.DatasetDescriptor

    out.close();
    command.avroSchemaFile = avsc;
    command.datasets = Lists.newArrayList("users");
    command.run();


    DatasetDescriptor expectedDescriptor = new DatasetDescriptor.Builder()
        .schemaUri("resource:test-schemas/user.avsc")
        .build();


    verify(getMockRepo()).create("default", "users", expectedDescriptor);
    verify(console).debug(contains("Created"), eq("users"));

View Full Code Here

Examples of org.kitesdk.data.DatasetDescriptor

    provider.delete("old_1", "new_3");
  }


  @Test
  public void testLoad() {
    DatasetDescriptor loaded = provider.load("default", "old_1");
    Assert.assertNotNull("Should find old layout datasets", loaded);
  }

View Full Code Here

Examples of org.kitesdk.data.DatasetDescriptor

        provider.exists("default", "old_1"));
  }


  @Test
  public void testUpdate() throws IOException {
    DatasetDescriptor updated = new DatasetDescriptor.Builder(descriptor)
        .property("parquet.block.size", "1024")
        .build();


    DatasetDescriptor saved = provider.update("default", "old_2", updated);
    Assert.assertNotNull("Should find saved metadata", saved);
    Assert.assertEquals("Should update old dataset successfully",
        updated.getProperty("parquet.block.size"),
        saved.getProperty("parquet.block.size"));


    DatasetDescriptor loaded = provider.load("default", "old_2");
    Assert.assertNotNull("Should find saved metadata", loaded);
    Assert.assertEquals("Should make changes on disk",
        updated.getProperty("parquet.block.size"),
        loaded.getProperty("parquet.block.size"));


    Assert.assertFalse("Should not move metadata to new location",
        local.exists(new Path(root, new Path("default", "old_2"))));
  }

View Full Code Here

Examples of org.kitesdk.data.DatasetDescriptor

    out.close();
  }


  @Override
  public DatasetReader<GenericData.Record> newReader() throws IOException {
    final DatasetDescriptor desc = new DatasetDescriptor.Builder()
        .property("kite.csv.has-header", "true")
        .schema(VALIDATOR_SCHEMA)
        .build();
    return new CSVFileReader<GenericData.Record>(localfs, validatorFile, desc,
        DataModelUtil.accessor(GenericData.Record.class, desc.getSchema()));
  }

View Full Code Here

Examples of org.kitesdk.data.DatasetDescriptor

    };
  }


  @Test(expected = IllegalArgumentException.class)
  public void testRejectsNonRecordSchemas() {
    final DatasetDescriptor desc = new DatasetDescriptor.Builder()
        .schema(SchemaBuilder.array().items().stringType())
        .build();
    new CSVFileReader<GenericData.Record>(localfs, csvFile, desc,
        DataModelUtil.accessor(GenericData.Record.class, desc.getSchema()));
  }

View Full Code Here

Examples of org.kitesdk.data.DatasetDescriptor

        DataModelUtil.accessor(GenericData.Record.class, desc.getSchema()));
  }


  @Test
  public void testStringSchema() {
    final DatasetDescriptor desc = new DatasetDescriptor.Builder()
        .schema(STRINGS)
        .build();
    final CSVFileReader<GenericData.Record> reader =
        new CSVFileReader<GenericData.Record>(localfs, csvFile, desc,
            DataModelUtil.accessor(GenericData.Record.class, desc.getSchema()));


    reader.initialize();
    Assert.assertTrue(reader.hasNext());
    GenericData.Record rec = reader.next();
    Assert.assertEquals("str", rec.get(0));

View Full Code Here

Examples of org.kitesdk.data.DatasetDescriptor

    Assert.assertFalse(reader.hasNext());
  }


  @Test
  public void testTSV() {
    final DatasetDescriptor desc = new DatasetDescriptor.Builder()
        .property("kite.csv.delimiter", "\t")
        .property("kite.csv.lines-to-skip", "1")
        .schema(STRINGS)
        .build();
    final CSVFileReader<GenericData.Record> reader =
        new CSVFileReader<GenericData.Record>(localfs, tsvFile, desc,
            DataModelUtil.accessor(GenericData.Record.class, desc.getSchema()));


    reader.initialize();
    Assert.assertTrue(reader.hasNext());
    GenericData.Record rec = reader.next();
    Assert.assertEquals("str", rec.get(0));

View Full Code Here

Examples of org.kitesdk.data.DatasetDescriptor

    Assert.assertFalse(reader.hasNext());
  }


  @Test
  public void testTSVWithDeprecatedProperties() {
    final DatasetDescriptor desc = new DatasetDescriptor.Builder()
        .property("cdk.csv.delimiter", "\t")
        .property("cdk.csv.lines-to-skip", "1")
        .schema(STRINGS)
        .build();
    final CSVFileReader<GenericData.Record> reader =
        new CSVFileReader<GenericData.Record>(localfs, tsvFile, desc,
            DataModelUtil.accessor(GenericData.Record.class, desc.getSchema()));


    reader.initialize();
    Assert.assertTrue(reader.hasNext());
    GenericData.Record rec = reader.next();
    Assert.assertEquals("str", rec.get(0));

View Full Code Here

Examples of org.kitesdk.data.DatasetDescriptor

    Assert.assertFalse(reader.hasNext());
  }


  @Test
  public void testNormalSchema() {
    final DatasetDescriptor desc = new DatasetDescriptor.Builder()
        .schema(SCHEMA)
        .build();
    final CSVFileReader<GenericData.Record> reader =
        new CSVFileReader<GenericData.Record>(localfs, csvFile, desc,
            DataModelUtil.accessor(GenericData.Record.class, desc.getSchema()));


    reader.initialize();
    Assert.assertTrue(reader.hasNext());
    GenericData.Record rec = reader.next();
    Assert.assertEquals("str", rec.get(0));

View Full Code Here

Examples of org.kitesdk.data.DatasetDescriptor

    Assert.assertFalse(reader.hasNext());
  }


  @Test
  public void testReflectedRecords() {
    final DatasetDescriptor desc = new DatasetDescriptor.Builder()
        .schema(BEAN_SCHEMA)
        .build();
    final CSVFileReader<TestBean> reader =
        new CSVFileReader<TestBean>(localfs, csvFile, desc,
            DataModelUtil.accessor(TestBean.class, desc.getSchema()));


    reader.initialize();
    Assert.assertTrue(reader.hasNext());
    TestBean bean = reader.next();
    Assert.assertEquals("str", bean.myStr);

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.