Examples of DatasetDescriptor

com.cloudera.cdk.data.DatasetDescriptor

The structural definition of a {@link Dataset}.

Each {@code Dataset} has an associated {@link Schema} and optional{@link PartitionStrategy} defined at the time of creation. Instances of thisclass are used to hold this information. Users are strongly encouraged to use the inner {@link Builder} to create new instances.
org.kitesdk.data.DatasetDescriptor

The structural definition of a {@link Dataset}.

Each {@code Dataset} has an associated {@link Schema} and optional{@link PartitionStrategy} defined at the time of creation. You use instancesof this class to hold this information. You are strongly encouraged to use the inner {@link Builder} to create new instances.

Examples of org.kitesdk.data.DatasetDescriptor


  @Test
  public void testLoad() {
    ensureCreated();


    DatasetDescriptor loaded = provider.load(NAMESPACE, NAME);


    Assert.assertNotNull("DatasetDescriptor should be returned", loaded);
    Assert.assertEquals("Schema should match",
        testDescriptor.getSchema(), loaded.getSchema());
    Assert.assertEquals("PartitionStrategy should match",
        testDescriptor.getPartitionStrategy(), loaded.getPartitionStrategy());
    Assert.assertEquals("Format should match",
        testDescriptor.getFormat(), loaded.getFormat());
  }

View Full Code Here

Examples of org.kitesdk.data.DatasetDescriptor

    }
  }


  @Test
  public void testCreateIgnoresLocation() throws IOException {
    DatasetDescriptor created = provider.create(NAMESPACE, NAME, testDescriptor);
    Assert.assertNull("Created descriptor should not have a location",
        created.getLocation());
  }

View Full Code Here

Examples of org.kitesdk.data.DatasetDescriptor


  @Test
  public void testDeleteRemovesMetadataFiles() throws IOException {
    testCreateMetadataFiles();


    DatasetDescriptor loaded = provider.load(NAMESPACE, NAME);


    Path namedDirectory = new Path(loaded.getLocation());
    Path metadataDirectory = new Path(namedDirectory, ".metadata");
    Path propertiesFile = new Path(metadataDirectory, "descriptor.properties");
    Path schemaFile = new Path(metadataDirectory, "schema.avsc");


    boolean result = provider.delete(NAMESPACE, NAME);

View Full Code Here

Examples of org.kitesdk.data.DatasetDescriptor

     * happily saved by the MetadataProvider. Rule enforcement is done upstream
     * by libraries that are in a better position to make decisions about what
     * changes are incompatible.
     */


    final DatasetDescriptor saved = provider.update(NAMESPACE, NAME, anotherDescriptor);


    Assert.assertNotNull("Updated Descriptor should be returned", saved);
    Assert.assertEquals("Schema should match update",
        anotherDescriptor.getSchema(), saved.getSchema());
    Assert.assertEquals("PartitionStrategy should match update",
        anotherDescriptor.getPartitionStrategy(), saved.getPartitionStrategy());
    Assert.assertEquals("Format should match update",
        anotherDescriptor.getFormat(), saved.getFormat());
  }

View Full Code Here

Examples of org.kitesdk.data.DatasetDescriptor


  @Test
  public void testCustomProperties() {
    final String propName = "my.custom.property";
    final String propValue = "string";
    DatasetDescriptor descriptorWithProp =
        new DatasetDescriptor.Builder(testDescriptor)
        .property(propName, propValue)
        .build();


    DatasetDescriptor created = provider.create(NAMESPACE, NAME, descriptorWithProp);
    Assert.assertTrue("Should have custom property",
        created.hasProperty(propName));
    Assert.assertEquals("Should have correct custom property value",
        propValue, created.getProperty(propName));
    Assert.assertTrue("List should contain property name",
        created.listProperties().contains(propName));


    DatasetDescriptor loaded = provider.load(NAMESPACE, NAME);
    Assert.assertTrue("Should have custom property",
        loaded.hasProperty(propName));
    Assert.assertEquals("Should have correct custom property value",
        propValue, loaded.getProperty(propName));
    Assert.assertTrue("List should contain property name",
        created.listProperties().contains(propName));
  }

View Full Code Here

Examples of org.kitesdk.data.DatasetDescriptor

      recordClass = loader.loadClass(className);
    } catch (ClassNotFoundException e) {
      throw new IllegalArgumentException("Cannot find class: " + className, e);
    }


    DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
        .schema(recordClass)
        .build();
    String schema = descriptor.getSchema().toString(!minimize);


    output(schema, console, outputPath);


    return 0;
  }

View Full Code Here

Examples of org.kitesdk.data.DatasetDescriptor


    return 0;
  }


  private static void printInfo(Logger console, Dataset<?> dataset) {
    DatasetDescriptor desc = dataset.getDescriptor();
    String schema = ColumnMappingParser.removeEmbeddedMapping(
        PartitionStrategyParser.removeEmbeddedStrategy(desc.getSchema()))
        .toString(true);
    Collection<String> properties = desc.listProperties();


    console.info("\nDataset \"{}\":", dataset.getName());
    console.info("\tURI: \"{}\"", dataset.getUri());
    console.info("\tSchema: {}", indent(schema));
    if (desc.isPartitioned()) {
      console.info("\tPartition strategy: {}",
          indent(desc.getPartitionStrategy().toString(true)));
    } else {
      console.info("\tNot partitioned");
    }
    if (desc.isColumnMapped()) {
      console.info("\tColumn mapping: {}",
          indent(desc.getColumnMapping().toString(true)));
    }
    if (!properties.isEmpty()) {
      StringBuilder sb = new StringBuilder();
      for (String prop : properties) {
        sb.append("\n\t\t").append(prop).append("=")
            .append(desc.getProperty(prop));
      }
      console.info("\tProperties:{}", sb.toString());
    }
  }

View Full Code Here

Examples of org.kitesdk.data.DatasetDescriptor

            Iterators.getNext(parts, null),
            Iterators.getNext(parts, null));
      }
    }


    DatasetDescriptor descriptor = descriptorBuilder.build();
    if (isDataUri(datasets.get(0))) {
      Datasets.<Object, Dataset<Object>> create(datasets.get(0), descriptor, Object.class);
    } else {
      getDatasetRepository().create(namespace, datasets.get(0), descriptor);
    }

View Full Code Here

Examples of org.kitesdk.data.DatasetDescriptor

    HBaseTestUtils.util.deleteTable(Bytes.toBytes(managedTableName));
    HBaseDatasetRepository repo = new HBaseDatasetRepository.Builder()
        .configuration(HBaseTestUtils.getConf()).build();
    String testGenericEntity = AvroUtils.inputStreamToString(
        HBaseDatasetRepositoryTest.class.getResourceAsStream("/TestGenericEntity.avsc"));
    DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
        .schemaLiteral(testGenericEntity)
        .build();
    dataset = repo.create("default", "testtable", descriptor);
    for (int i = 0; i < 10; i++) {
      dataset.put(HBaseDatasetRepositoryTest.createGenericEntity(i));

View Full Code Here

Examples of org.kitesdk.data.DatasetDescriptor

    if (avroSchemaFile == null && avroSchemaReflectClass == null) {
      throw new IllegalArgumentException("One of kite.avroSchemaFile or "
          + "kite.avroSchemaReflectClass must be specified");
    }


    DatasetDescriptor descriptor;
    if (uri != null) {
      descriptor = Datasets.load(uri).getDataset().getDescriptor();
    } else {
      LOG.warn(
          "kite.datasetName is deprecated, instead use kite.uri=<dataset-uri>");

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.