Examples of org.kitesdk.data.PartitionStrategy

org.kitesdk.data.PartitionStrategy

The strategy used to determine how a dataset is partitioned.

When a {@link Dataset} is configuredwith a partition strategy, that data is considered partitioned. Any entities written to a partitioned dataset are evaluated with its {@code PartitionStrategy} to determine which partition to write to.

You should use the inner {@link Builder} to create new instances.
@see DatasetDescriptor @see Dataset

    Assert.assertEquals(5, datasetSize(outputPart0));
  }


  @Test
  public void testSourceView() throws IOException {
    PartitionStrategy partitionStrategy = new PartitionStrategy.Builder().hash(
        "username", 2).build();


    Dataset<Record> inputDataset = repo.create("ns", "in", new DatasetDescriptor.Builder()
        .schema(USER_SCHEMA).partitionStrategy(partitionStrategy).build());
    Dataset<Record> outputDataset = repo.create("ns", "out", new DatasetDescriptor.Builder()

View Full Code Here

    Assert.assertEquals(1, datasetSize(outputDataset));
  }


  @Test
  public void testTargetView() throws IOException {
    PartitionStrategy partitionStrategy = new PartitionStrategy.Builder().hash(
        "username", 2).build();


    Dataset<Record> inputDataset = repo.create("ns", "in", new DatasetDescriptor.Builder()
        .schema(USER_SCHEMA).partitionStrategy(partitionStrategy).build());
    Dataset<Record> outputDataset = repo.create("ns", "out", new DatasetDescriptor.Builder()

View Full Code Here

    Assert.assertEquals(1, datasetSize(outputDataset));
  }
  
  @Test
  public void testViewUris() throws IOException {
    PartitionStrategy partitionStrategy = new PartitionStrategy.Builder().hash(
        "username", 2).build();


    Dataset<Record> inputDataset = repo.create("ns", "in", new DatasetDescriptor.Builder()
        .schema(USER_SCHEMA).partitionStrategy(partitionStrategy).build());
    Dataset<Record> outputDataset = repo.create("ns", "out", new DatasetDescriptor.Builder()

View Full Code Here

    Assert.assertEquals(1, datasetSize(outputDataset));
  }
  
  @Test
  public void testDatasetUris() throws IOException {
    PartitionStrategy partitionStrategy = new PartitionStrategy.Builder().hash(
        "username", 2).build();


    Dataset<Record> inputDataset = repo.create("ns", "in", new DatasetDescriptor.Builder()
        .schema(USER_SCHEMA).partitionStrategy(partitionStrategy).build());
    Dataset<Record> outputDataset = repo.create("ns", "out", new DatasetDescriptor.Builder()

View Full Code Here

  }


  public StorageKey keyFor(E object, @Nullable Map<String, Object> provided,
                           StorageKey reuse) {
    Preconditions.checkNotNull(reuse, "Cannot use null key");
    PartitionStrategy strategy = reuse.getPartitionStrategy();
    List<FieldPartitioner> partitioners = strategy.getFieldPartitioners();
    for (int i = 0, n = partitioners.size(); i < n; i += 1) {
      reuse.replace(i, partitionValue(object, provided, partitioners.get(i)));
    }
    return reuse;
  }

View Full Code Here

        Formats.AVRO, repo.load(NAMESPACE, NAME).getDescriptor().getFormat());
  }


  @Test
  public void testUpdateFailsWithPartitionStrategyChange() {
    PartitionStrategy ps1 = new PartitionStrategy.Builder()
        .hash("username", 2)
        .build();
    PartitionStrategy ps2 = new PartitionStrategy.Builder()
        .hash("username", 2)
        .hash("email", 3)
        .build();


    Dataset<Record> dataset = repo.create(NAMESPACE, NAME,

View Full Code Here

    NOV_1 = new Marker.Builder()
        .add("year", 2013)
        .add("month", 11)
        .add("day", 1)
        .build();
    PartitionStrategy strategy = new PartitionStrategy.Builder()
        .year("timestamp")
        .month("timestamp")
        .day("timestamp")
        .build();
    comparator = new MarkerComparator(strategy);

View Full Code Here

    Assert.assertEquals("2013-11-20", yyyyMMdd_PDT.apply(time));
  }


  @Test
  public void testExpressionRoundTrip() {
    PartitionStrategy strategy = new PartitionStrategy.Builder()
        .dateFormat("timestamp", "day", "yyyy-MM-dd")
        .build();
    PartitionStrategy copy = Accessor.getDefault().fromExpression(
        Accessor.getDefault().toExpression(strategy));
    Assert.assertEquals(strategy, copy);
  }

View Full Code Here

        "%s is not contained in %s", uri, location);
    Preconditions.checkArgument(descriptor.isPartitioned(),
        "Dataset is not partitioned");


    Schema schema = descriptor.getSchema();
    PartitionStrategy strategy = descriptor.getPartitionStrategy();


    RefinableView<E> view = dataset;
    Iterator<String> parts = PATH_SPLITTER.split(relative.toString()).iterator();
    for (FieldPartitioner fp : strategy.getFieldPartitioners()) {
      if (!parts.hasNext()) {
        break;
      }
      String value = Iterables.getLast(KV_SPLITTER.split(parts.next()));
      Schema fieldSchema = SchemaUtil.fieldSchema(schema, strategy, fp.getName());

View Full Code Here

    } catch (IOException e) {
      throw new DatasetIOException("Unable to locate or create dataset partition directory " + partitionDirectory, e);
    }


    int partitionDepth = key.getLength();
    PartitionStrategy subpartitionStrategy = Accessor.getDefault()
        .getSubpartitionStrategy(partitionStrategy, partitionDepth);


    return new FileSystemDataset.Builder<E>()
        .namespace(namespace)
        .name(name)

View Full Code Here

0 1 2 3 4 5

TOP

Related Classes of org.kitesdk.data.PartitionStrategy

org.kitesdk.cli.commands.TestCreatePartitionStrategyCommand

org.kitesdk.data.crunch.CrunchDatasets$GetStorageKey

org.kitesdk.data.crunch.TestCrunchDatasets

org.kitesdk.data.hbase.avro.AvroKeyEntitySchemaParserTest

org.kitesdk.data.hbase.avro.AvroUtils

org.kitesdk.data.hbase.HBaseMetadataProviderTest

org.kitesdk.data.spi.Constraints

org.kitesdk.data.spi.EntityAccessor

org.kitesdk.data.spi.filesystem.FileSystemDataset

org.kitesdk.data.spi.filesystem.FileSystemDatasetRepository

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.