Package org.kitesdk.data

Examples of org.kitesdk.data.PartitionStrategy$Builder

213
214
215
216
217
218
219
220
221
222
223
    Assert.assertEquals(5, datasetSize(outputPart0));
  }

  @Test
  public void testSourceView() throws IOException {
    PartitionStrategy partitionStrategy = new PartitionStrategy.Builder().hash(
        "username", 2).build();

    Dataset<Record> inputDataset = repo.create("ns", "in", new DatasetDescriptor.Builder()
        .schema(USER_SCHEMA).partitionStrategy(partitionStrategy).build());
    Dataset<Record> outputDataset = repo.create("ns", "out", new DatasetDescriptor.Builder()
View Full Code Here
237
238
239
240
241
242
243
244
245
246
247
    Assert.assertEquals(1, datasetSize(outputDataset));
  }

  @Test
  public void testTargetView() throws IOException {
    PartitionStrategy partitionStrategy = new PartitionStrategy.Builder().hash(
        "username", 2).build();

    Dataset<Record> inputDataset = repo.create("ns", "in", new DatasetDescriptor.Builder()
        .schema(USER_SCHEMA).partitionStrategy(partitionStrategy).build());
    Dataset<Record> outputDataset = repo.create("ns", "out", new DatasetDescriptor.Builder()
View Full Code Here
262
263
264
265
266
267
268
269
270
271
272
    Assert.assertEquals(1, datasetSize(outputDataset));
  }
 
  @Test
  public void testViewUris() throws IOException {
    PartitionStrategy partitionStrategy = new PartitionStrategy.Builder().hash(
        "username", 2).build();

    Dataset<Record> inputDataset = repo.create("ns", "in", new DatasetDescriptor.Builder()
        .schema(USER_SCHEMA).partitionStrategy(partitionStrategy).build());
    Dataset<Record> outputDataset = repo.create("ns", "out", new DatasetDescriptor.Builder()
View Full Code Here
292
293
294
295
296
297
298
299
300
301
302
    Assert.assertEquals(1, datasetSize(outputDataset));
  }
 
  @Test
  public void testDatasetUris() throws IOException {
    PartitionStrategy partitionStrategy = new PartitionStrategy.Builder().hash(
        "username", 2).build();

    Dataset<Record> inputDataset = repo.create("ns", "in", new DatasetDescriptor.Builder()
        .schema(USER_SCHEMA).partitionStrategy(partitionStrategy).build());
    Dataset<Record> outputDataset = repo.create("ns", "out", new DatasetDescriptor.Builder()
View Full Code Here
95
96
97
98
99
100
101
102
103
104
105
106
  }

  public StorageKey keyFor(E object, @Nullable Map<String, Object> provided,
                           StorageKey reuse) {
    Preconditions.checkNotNull(reuse, "Cannot use null key");
    PartitionStrategy strategy = reuse.getPartitionStrategy();
    List<FieldPartitioner> partitioners = strategy.getFieldPartitioners();
    for (int i = 0, n = partitioners.size(); i < n; i += 1) {
      reuse.replace(i, partitionValue(object, provided, partitioners.get(i)));
    }
    return reuse;
  }
View Full Code Here
103
104
105
106
107
108
109
110
111
112
113
114
115
116
        Formats.AVRO, repo.load(NAMESPACE, NAME).getDescriptor().getFormat());
  }

  @Test
  public void testUpdateFailsWithPartitionStrategyChange() {
    PartitionStrategy ps1 = new PartitionStrategy.Builder()
        .hash("username", 2)
        .build();
    PartitionStrategy ps2 = new PartitionStrategy.Builder()
        .hash("username", 2)
        .hash("email", 3)
        .build();

    Dataset<Record> dataset = repo.create(NAMESPACE, NAME,
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
    NOV_1 = new Marker.Builder()
        .add("year", 2013)
        .add("month", 11)
        .add("day", 1)
        .build();
    PartitionStrategy strategy = new PartitionStrategy.Builder()
        .year("timestamp")
        .month("timestamp")
        .day("timestamp")
        .build();
    comparator = new MarkerComparator(strategy);
View Full Code Here
38
39
40
41
42
43
44
45
46
47
48
49
    Assert.assertEquals("2013-11-20", yyyyMMdd_PDT.apply(time));
  }

  @Test
  public void testExpressionRoundTrip() {
    PartitionStrategy strategy = new PartitionStrategy.Builder()
        .dateFormat("timestamp", "day", "yyyy-MM-dd")
        .build();
    PartitionStrategy copy = Accessor.getDefault().fromExpression(
        Accessor.getDefault().toExpression(strategy));
    Assert.assertEquals(strategy, copy);
  }
View Full Code Here
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
        "%s is not contained in %s", uri, location);
    Preconditions.checkArgument(descriptor.isPartitioned(),
        "Dataset is not partitioned");

    Schema schema = descriptor.getSchema();
    PartitionStrategy strategy = descriptor.getPartitionStrategy();

    RefinableView<E> view = dataset;
    Iterator<String> parts = PATH_SPLITTER.split(relative.toString()).iterator();
    for (FieldPartitioner fp : strategy.getFieldPartitioners()) {
      if (!parts.hasNext()) {
        break;
      }
      String value = Iterables.getLast(KV_SPLITTER.split(parts.next()));
      Schema fieldSchema = SchemaUtil.fieldSchema(schema, strategy, fp.getName());
View Full Code Here
209
210
211
212
213
214
215
216
217
218
219
    } catch (IOException e) {
      throw new DatasetIOException("Unable to locate or create dataset partition directory " + partitionDirectory, e);
    }

    int partitionDepth = key.getLength();
    PartitionStrategy subpartitionStrategy = Accessor.getDefault()
        .getSubpartitionStrategy(partitionStrategy, partitionDepth);

    return new FileSystemDataset.Builder<E>()
        .namespace(namespace)
        .name(name)
View Full Code Here
TOP

Related Classes of org.kitesdk.data.PartitionStrategy$Builder

  • ch.grengine.code.groovy.DefaultGroovyCompilerFactory
  • co.cask.tigon.sql.internal.DefaultGDATField
  • com.asakusafw.utils.java.model.syntax.Expression
  • com.asakusafw.utils.java.model.syntax.ModelFactory
  • com.asakusafw.utils.java.model.util.ImportBuilder
  • com.asakusafw.utils.java.model.util.TypeBuilder
  • com.basho.riak.client.api.cap.Quorum
  • com.facebook.presto.serde.BlocksFileWriter
  • com.fasterxml.jackson.databind.JsonNode
  • com.fasterxml.jackson.databind.ObjectMapper
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.