Package org.kitesdk.data

Examples of org.kitesdk.data.PartitionStrategy$Builder

215
216
217
218
219
220
221
222
223
224
225
226
227
    }
    // Ensure schema is limited to keySchema's fields. The class may have more
    // fields
    // in the case that the entity is being used as a key.
    List<Field> fields = Lists.newArrayList();
    PartitionStrategy strategy = keySchema.getPartitionStrategy();
    for (Schema.Field field : keySchema.getAvroSchema().getFields()) {
      String sourceName = strategy.getPartitioner(field.name()).getSourceName();
      fields.add(copy(schemaField.getField(sourceName)));
    }
    Schema schema = Schema.createRecord(keySchema.getAvroSchema().getName(),
        keySchema.getAvroSchema().getDoc(), keySchema.getAvroSchema()
            .getNamespace(), keySchema.getAvroSchema().isError());
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
  @SuppressWarnings("deprecation")
  @Test
  public void testNewPartitionIsVisibleToHive() throws Exception {
    final String NAME2 = "test2";

    PartitionStrategy partitionStrategy = new PartitionStrategy.Builder()
        .hash("username", 2).build();

    DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
        .schema(testSchema)
        .partitionStrategy(partitionStrategy)
View Full Code Here
115
116
117
118
119
120
121
122
123
124
125
  @Test
  public void testMerge() throws Exception {
    final String NAME2 = "test2";
    final String NAME3 = "test3";

    PartitionStrategy partitionStrategy = new PartitionStrategy.Builder()
        .hash("username", 2).build();

    DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
        .schema(testSchema)
        .partitionStrategy(partitionStrategy)
View Full Code Here
50
51
52
53
54
55
56
57
58
59
60
    NOV_1 = new Marker.Builder()
        .add("year", 2013)
        .add("month", 11)
        .add("day", 1)
        .build();
    PartitionStrategy strategy = new PartitionStrategy.Builder()
        .year("timestamp")
        .month("timestamp")
        .day("timestamp")
        .build();
    comparator = new MarkerComparator(strategy);
View Full Code Here
276
277
278
279
280
281
282
283
284
285
286
    }

    @Override
    public void initialize() {
      if (key == null) {
        PartitionStrategy strategy = PartitionStrategyParser.parse(strategyString);
        Schema schema = new Schema.Parser().parse(schemaString);
        this.key = new AvroStorageKey(strategy, schema);
        this.accessor = DataModelUtil.accessor(type, schema);
      }
    }
View Full Code Here
513
514
515
516
517
518
519
520
521
522
523
        .from("created_at",
            new DateTime(2013, 1, 1, 0, 0, DateTimeZone.UTC).getMillis())
        .toBefore("created_at",
            new DateTime(2014, 1, 1, 0, 0, DateTimeZone.UTC).getMillis());

    PartitionStrategy strategy = new PartitionStrategy.Builder()
        .hash("color", "hash", 50)
        .year("created_at").month("created_at").day("created_at")
        .identity("color")
        .build();
View Full Code Here
541
542
543
544
545
546
547
548
549
550
551
552
553
554
        c3.minimizeFor(key).keySet());
  }

  @Test
  public void testAlignedWithPartitionBoundaries() {
    PartitionStrategy hashStrategy = new PartitionStrategy.Builder()
        .hash("id", "bucket", 32)
        .build();
    PartitionStrategy withColor = new PartitionStrategy.Builder()
        .range("color", "blue", "green", "red")
        .year("timestamp")
        .month("timestamp")
        .day("timestamp")
        .identity("id")
View Full Code Here
130
131
132
133
134
135
136
137
138
139
140
    checkTestUsers(outputDataset, 10);
  }

  @Test
  public void testPartitionedSource() throws IOException {
    PartitionStrategy partitionStrategy = new PartitionStrategy.Builder().hash(
        "username", 2).build();

    Dataset<Record> inputDataset = repo.create("ns", "in", new DatasetDescriptor.Builder()
        .schema(USER_SCHEMA).partitionStrategy(partitionStrategy).build());
    Dataset<Record> outputDataset = repo.create("ns", "out", new DatasetDescriptor.Builder()
View Full Code Here
155
156
157
158
159
160
161
162
163
164
165
    Assert.assertEquals(5, datasetSize(outputDataset));
  }

  @Test
  public void testPartitionedSourceAndTarget() throws IOException {
    PartitionStrategy partitionStrategy = new PartitionStrategy.Builder().hash(
        "username", 2).build();

    Dataset<Record> inputDataset = repo.create("ns", "in", new DatasetDescriptor.Builder()
        .schema(USER_SCHEMA).partitionStrategy(partitionStrategy).build());
    Dataset<Record> outputDataset = repo.create("ns", "out", new DatasetDescriptor.Builder()
View Full Code Here
182
183
184
185
186
187
188
189
190
191
192
    Assert.assertEquals(5, datasetSize(outputPart0));
  }

  @Test
  public void testPartitionedSourceAndTargetWritingToTopLevel() throws IOException {
    PartitionStrategy partitionStrategy = new PartitionStrategy.Builder().hash(
        "username", 2).build();

    Dataset<Record> inputDataset = repo.create("ns", "in", new DatasetDescriptor.Builder()
        .schema(USER_SCHEMA).partitionStrategy(partitionStrategy).build());
    Dataset<Record> outputDataset = repo.create("ns", "out", new DatasetDescriptor.Builder()
View Full Code Here
TOP

Related Classes of org.kitesdk.data.PartitionStrategy$Builder

  • ch.grengine.code.groovy.DefaultGroovyCompilerFactory
  • co.cask.tigon.sql.internal.DefaultGDATField
  • com.asakusafw.utils.java.model.syntax.Expression
  • com.asakusafw.utils.java.model.syntax.ModelFactory
  • com.asakusafw.utils.java.model.util.ImportBuilder
  • com.asakusafw.utils.java.model.util.TypeBuilder
  • com.basho.riak.client.api.cap.Quorum
  • com.facebook.presto.serde.BlocksFileWriter
  • com.fasterxml.jackson.databind.JsonNode
  • com.fasterxml.jackson.databind.ObjectMapper
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.