Examples of com.cloudera.cdk.data.FieldPartitioner

com.cloudera.cdk.data.FieldPartitioner

Partitions values for a named field.

Used by a {@link PartitionStrategy} to calculate which partition an entitybelongs in, based on the value of a given field, called the source field. A field partitioner can, in some cases, provide meaningful cardinality hints to query systems. A good example of this is a hash partitioner which always knows the number of buckets produced by the function.

Implementations of {@link FieldPartitioner} are immutable.
@param < S> The type of the source field in the entity. The partition function mustaccept values of this type. @param < T> The type of the target field, which is the type of the return value of thepartition function. @see PartitionStrategy

  @SuppressWarnings("unchecked")
  public StorageKey reuseFor(Object entity) {
    final List<FieldPartitioner> partitioners = strategy.getFieldPartitioners();


    for (int i = 0; i < partitioners.size(); i++) {
      final FieldPartitioner fp = partitioners.get(i);
      final Object value;
      // TODO: this should probably live elsewhere and be extensible
      if (entity instanceof GenericRecord) {
        value = ((GenericRecord) entity).get(fp.getSourceName());
      } else {
        final String name = fp.getSourceName();
        try {
          PropertyDescriptor propertyDescriptor = new PropertyDescriptor(name,
              entity.getClass(), getter(name), null /* assume read only */);
          value = propertyDescriptor.getReadMethod().invoke(entity);
        } catch (IllegalAccessException e) {
          throw new IllegalStateException("Cannot read property " + name +
              " from " + entity, e);
        } catch (InvocationTargetException e) {
          throw new IllegalStateException("Cannot read property " + name +
              " from " + entity, e);
        } catch (IntrospectionException e) {
          throw new IllegalStateException("Cannot read property " + name +
              " from " + entity, e);
        }
      }
      replace(i, fp.apply(value));
    }


    return this;
  }

View Full Code Here

    final StringBuilder pathBuilder = new StringBuilder();
    final List<FieldPartitioner> partitioners =
        key.getPartitionStrategy().getFieldPartitioners();


    for (int i = 0; i < partitioners.size(); i++) {
      final FieldPartitioner fp = partitioners.get(i);
      if (i != 0) {
        pathBuilder.append(Path.SEPARATOR_CHAR);
      }
      @SuppressWarnings("unchecked")
      String dirname = dirnameForValue(fp, key.get(i));

View Full Code Here

      throw new RuntimeException("PartitionStrategy does not match");
    }


    final List<FieldPartitioner> partitioners = strategy.getFieldPartitioners();
    for (int i = 0; i < partitioners.size(); i += 1) {
      final FieldPartitioner fp = partitioners.get(i);
      final int cmp = fp.compare(get(i), other.get(i));
      if (cmp != 0) {
        return cmp;
      }
    }
    return 0;

View Full Code Here


  @SuppressWarnings("unchecked")
  private Path toDirectoryName(Path dir, PartitionKey key) {
    Path result = dir;
    for (int i = 0; i < key.getLength(); i++) {
      final FieldPartitioner fp = partitionStrategy.getFieldPartitioners().get(i);
      result = new Path(result, convert.dirnameForValue(fp, key.get(i)));
    }
    return result;
  }

View Full Code Here

    return result;
  }


  @SuppressWarnings("unchecked")
  private PartitionKey fromDirectoryName(Path dir) {
    final FieldPartitioner fp = partitionStrategy.getFieldPartitioners().get(0);
    final List<Object> values = Lists.newArrayList();


    if (partitionKey != null) {
      values.addAll(partitionKey.getValues());
    }

View Full Code Here

    List<Object> values = Lists.newArrayList();
    int i = 0;
    for (String part : parts) {
      Iterator<String> split = Splitter.on('=').split(part).iterator();
      String fieldName = split.next();
      FieldPartitioner fp = fieldPartitioners.get(i++);
      if (!fieldName.equals(fp.getName())) {
        throw new IllegalArgumentException(String.format("Unrecognized partition name " +
            "'%s' in partition %s, expecting '%s'.", fieldName, partitionUri,
            fp.getName()));
      }
      if (!split.hasNext()) {
        throw new IllegalArgumentException(String.format("Missing partition value for " +
            "'%s' in partition %s.", fieldName, partitionUri));
      }
      String stringValue = split.next();
      Object value = fp.valueFromString(stringValue);
      values.add(value);
    }
    return com.cloudera.cdk.data.impl.Accessor.getDefault().newPartitionKey(
        values.toArray(new Object[values.size()]));
  }

View Full Code Here

        .build();


    List<FieldPartitioner> fieldPartitioners = p.getFieldPartitioners();
    Assert.assertEquals(2, fieldPartitioners.size());


    FieldPartitioner fp0 = fieldPartitioners.get(0);
    assertEquals("month", fp0.getName());
    assertEquals(12, fp0.getCardinality());


    FieldPartitioner fp1 = fieldPartitioners.get(1);
    assertEquals("userId", fp1.getName());
    assertEquals(7, fp1.getCardinality());


    assertEquals(12 * 7, p.getCardinality()); // useful for writers
  }

View Full Code Here

  static PartitionKey keyFor(PartitionStrategy strategy, Marker marker) {
    final List<FieldPartitioner> partitioners = strategy.getFieldPartitioners();
    final Object[] values = new Object[partitioners.size()];


    for (int i = 0, n = partitioners.size(); i < n; i += 1) {
      final FieldPartitioner fp = partitioners.get(i);
      values[i] = marker.valueFor(fp);
    }


    return strategy.partitionKey(values);
  }

View Full Code Here

    PartitionExpression expression = new PartitionExpression(expr, true);


    PartitionStrategy strategy = expression.evaluate();
    List<FieldPartitioner> fieldPartitioners = strategy.getFieldPartitioners();
    Assert.assertEquals(1, fieldPartitioners.size());
    FieldPartitioner fp = fieldPartitioners.get(0);
    Assert.assertEquals(HashFieldPartitioner.class, fp.getClass());
    Assert.assertEquals("username", fp.getSourceName());
    Assert.assertEquals("username_part", fp.getName());
    Assert.assertEquals(2, fp.getCardinality());


    Assert.assertEquals(expr, PartitionExpression.toExpression(strategy));
  }

View Full Code Here


    PartitionStrategy strategy = expression.evaluate();
    List<FieldPartitioner> fieldPartitioners = strategy.getFieldPartitioners();
    Assert.assertEquals(2, fieldPartitioners.size());


    FieldPartitioner fp0 = fieldPartitioners.get(0);
    Assert.assertEquals(HashFieldPartitioner.class, fp0.getClass());
    Assert.assertEquals("username_part", fp0.getName());
    Assert.assertEquals(2, fp0.getCardinality());


    FieldPartitioner fp1 = fieldPartitioners.get(1);
    Assert.assertEquals(HashFieldPartitioner.class, fp1.getClass());
    Assert.assertEquals("username2_part", fp1.getName());
    Assert.assertEquals(3, fp1.getCardinality());


    Assert.assertEquals(expr, PartitionExpression.toExpression(strategy));
  }

View Full Code Here

0 1

TOP

Related Classes of com.cloudera.cdk.data.FieldPartitioner

com.cloudera.cdk.data.filesystem.FileSystemDataset

com.cloudera.cdk.data.filesystem.FileSystemDatasetRepository

com.cloudera.cdk.data.filesystem.PathConversion

com.cloudera.cdk.data.hbase.DaoView

com.cloudera.cdk.data.spi.StorageKey

com.cloudera.cdk.data.TestPartitionExpression

com.cloudera.cdk.data.TestPartitionStrategy

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.