Package org.kitesdk.data.spi

Examples of org.kitesdk.data.spi.FieldPartitioner

59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
        .build();

    List<FieldPartitioner> fieldPartitioners = p.getFieldPartitioners();
    Assert.assertEquals(2, fieldPartitioners.size());

    FieldPartitioner fp0 = fieldPartitioners.get(0);
    assertEquals("month_ordinal", fp0.getName());
    assertEquals(12, fp0.getCardinality());

    FieldPartitioner fp1 = fieldPartitioners.get(1);
    assertEquals("userId_hash", fp1.getName());
    assertEquals(7, fp1.getCardinality());

    assertEquals(12 * 7, p.getCardinality()); // useful for writers
  }
View Full Code Here
192
193
194
195
196
197
198
199
200
201
202
  static PartitionKey keyFor(PartitionStrategy strategy, Marker marker) {
    final List<FieldPartitioner> partitioners = strategy.getFieldPartitioners();
    final Object[] values = new Object[partitioners.size()];

    for (int i = 0, n = partitioners.size(); i < n; i += 1) {
      final FieldPartitioner fp = partitioners.get(i);
      values[i] = marker.valueFor(fp);
    }

    return new PartitionKey(values);
  }
View Full Code Here
305
306
307
308
309
310
311
312
313
314
315
    @SuppressWarnings({"unchecked", "deprecation"})
    public <E> AvroStorageKey reuseFor(E entity, EntityAccessor<E> accessor) {
      List<FieldPartitioner> partitioners = strategy.getFieldPartitioners();

      for (int i = 0; i < partitioners.size(); i++) {
        FieldPartitioner fp = partitioners.get(i);
        put(i, fp.apply(accessor.get(entity, fp.getSourceName())));
      }

      return this;
    }
View Full Code Here
37
38
39
40
41
42
43
44
45
46
47
    this.partitionStrategy = partitionStrategy;
    this.fieldPositions = Maps.newHashMap();

    List<FieldPartitioner> partitioners = partitionStrategy.getFieldPartitioners();
    for (int i = 0; i < partitioners.size(); i += 1) {
      FieldPartitioner fp = partitioners.get(i);
      if (fp instanceof IdentityFieldPartitioner) {
        fieldPositions.put(fp.getSourceName(), i);
      }
    }
  }
View Full Code Here
32
33
34
35
36
37
38
39
40
41
42
43
44
    PartitionExpression expression = new PartitionExpression(expr, true);

    PartitionStrategy strategy = expression.evaluate();
    List<FieldPartitioner> fieldPartitioners = strategy.getFieldPartitioners();
    Assert.assertEquals(1, fieldPartitioners.size());
    FieldPartitioner fp = fieldPartitioners.get(0);
    Assert.assertEquals(HashFieldPartitioner.class, fp.getClass());
    Assert.assertEquals("username", fp.getSourceName());
    Assert.assertEquals("username_part", fp.getName());
    Assert.assertEquals(2, fp.getCardinality());

    Assert.assertEquals(expr, PartitionExpression.toExpression(strategy));
  }
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
    PartitionStrategy strategy = expression.evaluate();
    List<FieldPartitioner> fieldPartitioners = strategy.getFieldPartitioners();
    Assert.assertEquals(2, fieldPartitioners.size());

    FieldPartitioner fp0 = fieldPartitioners.get(0);
    Assert.assertEquals(HashFieldPartitioner.class, fp0.getClass());
    Assert.assertEquals("username_part", fp0.getName());
    Assert.assertEquals(2, fp0.getCardinality());

    FieldPartitioner fp1 = fieldPartitioners.get(1);
    Assert.assertEquals(HashFieldPartitioner.class, fp1.getClass());
    Assert.assertEquals("username2_part", fp1.getName());
    Assert.assertEquals(3, fp1.getCardinality());

    Assert.assertEquals(expr, PartitionExpression.toExpression(strategy));
  }
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
    PartitionStrategy strategy = expression.evaluate();
    List<FieldPartitioner> fieldPartitioners = strategy.getFieldPartitioners();
    Assert.assertEquals(2, fieldPartitioners.size());

    FieldPartitioner fp0 = fieldPartitioners.get(0);
    Assert.assertEquals(YearFieldPartitioner.class, fp0.getClass());
    Assert.assertEquals("timestamp", fp0.getSourceName());
    Assert.assertEquals("year", fp0.getName());

    FieldPartitioner fp1 = fieldPartitioners.get(1);
    Assert.assertEquals(MinuteFieldPartitioner.class, fp1.getClass());
    Assert.assertEquals("timestamp", fp1.getSourceName());
    Assert.assertEquals("minute", fp1.getName());

    Assert.assertEquals(expr, PartitionExpression.toExpression(strategy));
  }
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
    final StringBuilder pathBuilder = new StringBuilder();
    final List<FieldPartitioner> partitioners =
        key.getPartitionStrategy().getFieldPartitioners();

    for (int i = 0; i < partitioners.size(); i++) {
      final FieldPartitioner fp = partitioners.get(i);
      if (i != 0) {
        pathBuilder.append(Path.SEPARATOR_CHAR);
      }
      @SuppressWarnings("unchecked")
      String dirname = dirnameForValue(fp, key.get(i));
View Full Code Here
349
350
351
352
353
354
355
356
357
358
359
  @SuppressWarnings("unchecked")
  private Path toDirectoryName(@Nullable Path dir, PartitionKey key) {
    Path result = dir;
    for (int i = 0; i < key.getLength(); i++) {
      final FieldPartitioner fp = partitionStrategy.getFieldPartitioners().get(i);
      if (result != null) {
        result = new Path(result, PathConversion.dirnameForValue(fp, key.get(i)));
      } else {
        result = new Path(PathConversion.dirnameForValue(fp, key.get(i)));
      }
View Full Code Here
365
366
367
368
369
370
371
372
373
374
375
    return toDirectoryName(null, key);
  }

  @SuppressWarnings("unchecked")
  private PartitionKey keyFromDirectory(String name) {
    final FieldPartitioner fp = partitionStrategy.getFieldPartitioners().get(0);
    final List<Object> values = Lists.newArrayList();

    if (partitionKey != null) {
      values.addAll(partitionKey.getValues());
    }
View Full Code Here
TOP

Related Classes of org.kitesdk.data.spi.FieldPartitioner

  • org.kitesdk.data.crunch.CrunchDatasets$AvroStorageKey
  • org.kitesdk.data.hbase.DaoView
  • org.kitesdk.data.hbase.impl.KeySchema
  • org.kitesdk.data.spi.filesystem.FileSystemDataset
  • org.kitesdk.data.spi.filesystem.FileSystemDatasetRepository
  • org.kitesdk.data.spi.filesystem.PathConversion
  • org.kitesdk.data.TestPartitionExpression
  • org.kitesdk.data.TestPartitionStrategy
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.