Examples of org.kitesdk.data.spi.PartitionKey

org.kitesdk.data.spi.PartitionKey

A key for retrieving partitions from a {@link org.kitesdk.data.Dataset}.

A {@code PartitionKey} is an ordered sequence of values corresponding to the{@link org.kitesdk.data.spi.FieldPartitioner}s in a {@link org.kitesdk.data.PartitionStrategy}. You can obtain a {@link PartitionKey} using{@link PartitionKey#partitionKeyForEntity(org.kitesdk.data.PartitionStrategy,Object,EntityAccessor,PartitionKey)}.

Implementations of {@link PartitionKey} are typically not thread-safe; that is, the behavior when accessing a single instance from multiple threads is undefined.
@see org.kitesdk.data.PartitionStrategy @see org.kitesdk.data.spi.FieldPartitioner @see org.kitesdk.data.Dataset

    possibleValues.add("field1_20");
    possibleValues.add("field1_21");
    possibleValues.add("field1_22");
    possibleValues.add("field1_23");
    possibleValues.add("field1_24");
    PartitionKey startKey = new PartitionKey("part1_20", "part2_20");
    PartitionKey stopKey = new PartitionKey("part1_25", "part2_25");


    EntityScannerBuilder<GenericRecord> builder = dao.getScannerBuilder()
        .setStartKey(startKey).setStopKey(stopKey);


    checkScannerYieldValues(builder.build(), possibleValues);

View Full Code Here

    Dataset<Record> outputDataset = repo.create("ns", "out", new DatasetDescriptor.Builder()
        .schema(USER_SCHEMA).format(Formats.PARQUET).build());


    writeTestUsers(inputDataset, 10);


    PartitionKey key = new PartitionKey(0);
    Dataset<Record> inputPart0 =
        ((PartitionedDataset<Record>) inputDataset).getPartition(key, false);


    Pipeline pipeline = new MRPipeline(TestCrunchDatasets.class);
    PCollection<GenericData.Record> data = pipeline.read(

View Full Code Here

    Dataset<Record> outputDataset = repo.create("ns", "out", new DatasetDescriptor.Builder()
        .schema(USER_SCHEMA).partitionStrategy(partitionStrategy).build());


    writeTestUsers(inputDataset, 10);


    PartitionKey key = new PartitionKey(0);
    Dataset<Record> inputPart0 =
        ((PartitionedDataset<Record>) inputDataset).getPartition(key, false);
    Dataset<Record> outputPart0 =
        ((PartitionedDataset<Record>) outputDataset).getPartition(key, true);

View Full Code Here

    Dataset<Record> outputDataset = repo.create("ns", "out", new DatasetDescriptor.Builder()
        .schema(USER_SCHEMA).partitionStrategy(partitionStrategy).build());


    writeTestUsers(inputDataset, 10);


    PartitionKey key = new PartitionKey(0);
    Dataset<Record> inputPart0 =
        ((PartitionedDataset<Record>) inputDataset).getPartition(key, false);


    Pipeline pipeline = new MRPipeline(TestCrunchDatasets.class);
    PCollection<GenericData.Record> data = pipeline.read(

View Full Code Here

  }


  @Override
  public E mapToEntity(Result result) {
    boolean allNull = true;
    PartitionKey partitionKey;
    if (keySerDe == null) {
      partitionKey = null;
    } else {
      partitionKey = keySerDe.deserialize(result.getRow());
    }
    EntityComposer.Builder<E> builder = getEntityComposer().getBuilder();
    for (FieldMapping fieldMapping : entitySchema.getColumnMappingDescriptor()
        .getFieldMappings()) {
      Object fieldValue;
      if (fieldMapping.getMappingType() == MappingType.KEY) {
        if (partitionKey != null) {
          // KEY field mappings are always associated with identity mappers,
          // which is enforced by the DatasetDescriptor. Get the value from the
          // key at the correct position.
          fieldValue = partitionKey.get(
              keySchema.position(fieldMapping.getFieldName()));
        } else {
          // This should never happen. The partitionKey is null only when
          // a keySerDe hasn't been set, which indicates we have an entity
          // without a key. That means there should be no KEY mapping types, but

View Full Code Here


  @Override
  public boolean delete(E entity) {
    VersionCheckAction checkAction = entityMapper.mapFromEntity(entity)
        .getVersionCheckAction();
    PartitionKey key = entityMapper.mapToKey(entity);
    return clientTemplate.delete(key, entityMapper.getRequiredColumns(),
        checkAction, entityMapper.getKeySerDe());
  }

View Full Code Here

   *          The last name of users to scan.
   */
  public void printUserProfileActionsForLastName(String lastName) {
    // Create a partial key that will allow us to start the scanner from the
    // first user record that has last name equal to the one provided.
    PartitionKey startKey = new PartitionKey("lastName");


    // Get the scanner with the start key. Null for stopKey in the getScanner
    // method indicates that the scanner will scan to the end of the table. Our
    // loop will break out when it encounters a record without the last name.

View Full Code Here

    // Get the timestamp we'll use to set the value of the profile_updated
    // action.
    long ts = System.currentTimeMillis();


    // Construct the key we'll use to fetch the user.
    PartitionKey key = new PartitionKey(lastName, firstName);


    // Get the profile and actions entity from the composite dao.
    UserProfileActionsModel profileActionsModel = userProfileActionsDao
        .get(key);

View Full Code Here

      throw new DatasetIOException("Unable to list partition directory for directory " + directory, e);
    }


    for (FileStatus stat : fileStatuses) {
      Path p = fileSystem.makeQualified(stat.getPath());
      PartitionKey key = keyFromDirectory(p.getName());
      PartitionStrategy subPartitionStrategy = Accessor.getDefault()
          .getSubpartitionStrategy(partitionStrategy, 1);
      Builder<E> builder = new FileSystemDataset.Builder<E>()
          .namespace(namespace)
          .name(name)

View Full Code Here

      values.addAll(partitionKey.getValues());
    }


    values.add(convert.valueForDirname(fp, name));


    return new PartitionKey(values.toArray());
  }

View Full Code Here

0 1 2 3 4

TOP

Related Classes of org.kitesdk.data.spi.PartitionKey

org.kitesdk.data.crunch.TestCrunchDatasets

org.kitesdk.data.hbase.avro.AvroDaoTest

org.kitesdk.data.hbase.avro.AvroKeySerDe

org.kitesdk.data.hbase.avro.CompositeDaoTest

org.kitesdk.data.hbase.avro.example.UserProfileExample

org.kitesdk.data.hbase.avro.ManagedDaoTest

org.kitesdk.data.hbase.DaoDataset

org.kitesdk.data.hbase.DaoView

org.kitesdk.data.hbase.filters.ScannerFilterTest

org.kitesdk.data.hbase.impl.BaseDao

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.