Package org.kitesdk.data

Examples of org.kitesdk.data.PartitionStrategy$Builder

74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
        strategy, descriptor.getPartitionStrategy());
  }

  @Test
  public void testDescriptorValidationFails() {
    final PartitionStrategy missingStartField =
        new PartitionStrategy.Builder()
            .identity("p.latitude")
            .identity("p.longitude")
            .build();

    TestHelpers.assertThrows("Should complain that p is missing",
        IllegalStateException.class, new Runnable() {
          @Override
          public void run() {
            new DatasetDescriptor.Builder()
                .schema(schema)
                .partitionStrategy(missingStartField)
                .build();
          }
        });

    final PartitionStrategy missingLongField =
        new PartitionStrategy.Builder()
            .identity("position.latitude")
            .identity("position.long")
            .build();

    TestHelpers.assertThrows("Should complain that position.long is missing",
        IllegalStateException.class, new Runnable() {
          @Override
          public void run() {
            new DatasetDescriptor.Builder()
                .schema(schema)
                .partitionStrategy(missingLongField)
                .build();
          }
        });

    final PartitionStrategy badExpectedType =
        new PartitionStrategy.Builder()
            .identity("position.latitude")
            .year("position.longitude")
            .build();
View Full Code Here
159
160
161
162
163
164
165
166
167
168
169
  }

  @Test
  public void testRefineIdentity() throws Exception {
      PartitionStrategy strategy = new PartitionStrategy.Builder()
              .identity("user_id")
              .build();

      DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
              .schemaUri("resource:standard_event.avsc")
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
    NOV_1 = new Marker.Builder()
        .add("year", 2013)
        .add("month", 11)
        .add("day", 1)
        .build();
    PartitionStrategy strategy = new PartitionStrategy.Builder()
        .year("timestamp")
        .month("timestamp")
        .day("timestamp")
        .build();
    comparator = new MarkerComparator(strategy);
View Full Code Here
135
136
137
138
139
140
141
142
143
144
145
146
147
148
  @VisibleForTesting
  @SuppressWarnings("unchecked")
  Map<String, Predicate> minimizeFor(StorageKey key) {
    Map<String, Predicate> unsatisfied = Maps.newHashMap(constraints);
    PartitionStrategy strategy = key.getPartitionStrategy();
    Set<String> timeFields = Sets.newHashSet();
    int i = 0;
    for (FieldPartitioner fp : strategy.getFieldPartitioners()) {
      String partition = fp.getName();
      Predicate partitionPredicate = unsatisfied.get(partition);
      if (partitionPredicate != null && partitionPredicate.apply(key.get(i))) {
        unsatisfied.remove(partition);
        LOG.debug("removing " + partition + " satisfied by " + key.get(i));
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
    this.conf = new Configuration();
    this.fileSystem = FileSystem.get(conf);
    this.testDirectory = new Path(Files.createTempDir().getAbsolutePath());
    this.repo = new FileSystemDatasetRepository(conf, testDirectory);

    PartitionStrategy partitionStrategy = new PartitionStrategy.Builder()
        .hash("username", 2).build();
    FileSystemDataset<Object> users = (FileSystemDataset<Object>) repo.create(
        "ns", "users",
        new DatasetDescriptor.Builder()
            .schema(USER_SCHEMA)
View Full Code Here
106
107
108
109
110
111
112
113
114
115
116
  public void testProvidedPartitioner() throws IOException {
    Schema user = SchemaBuilder.record("User").fields()
        .requiredString("username")
        .requiredString("email")
        .endRecord();
    PartitionStrategy strategy = new PartitionStrategy.Builder()
        .provided("version", "int")
        .build();
    DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
        .schema(user)
        .partitionStrategy(strategy)
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
  }

  @Test
  @SuppressWarnings("unchecked")
  public void testFromKey() {
    PartitionStrategy strategy = new PartitionStrategy.Builder()
        .year("timestamp")
        .month("timestamp")
        .day("timestamp")
        .build();
View Full Code Here
109
110
111
112
113
114
115
116
117
118
119
  }

  @Test
  @SuppressWarnings("unchecked")
  public void testToKey() {
    PartitionStrategy strategy = new PartitionStrategy.Builder()
        .year("timestamp")
        .month("timestamp")
        .day("timestamp")
        .build();
View Full Code Here
125
126
127
128
129
130
131
132
133
134
135
  }

  @Test
  @SuppressWarnings("unchecked")
  public void toDirNameIdentityWithSlashes() {
    PartitionStrategy strategy = new PartitionStrategy.Builder()
        .identity("name")
        .identity("address")
        .build();

    StorageKey key = new StorageKey(strategy);
View Full Code Here
141
142
143
144
145
146
147
148
149
150
151
  }

  @Test
  @SuppressWarnings("unchecked")
  public void toDirNameIdentityWithNonString() {
    PartitionStrategy strategy = new PartitionStrategy.Builder()
        .identity("id")
        .build();

    StorageKey expected = new StorageKey(strategy);
    expected.replace(0, 0L);
View Full Code Here
TOP

Related Classes of org.kitesdk.data.PartitionStrategy$Builder

  • ch.grengine.code.groovy.DefaultGroovyCompilerFactory
  • co.cask.tigon.sql.internal.DefaultGDATField
  • com.asakusafw.utils.java.model.syntax.Expression
  • com.asakusafw.utils.java.model.syntax.ModelFactory
  • com.asakusafw.utils.java.model.util.ImportBuilder
  • com.asakusafw.utils.java.model.util.TypeBuilder
  • com.basho.riak.client.api.cap.Quorum
  • com.facebook.presto.serde.BlocksFileWriter
  • com.fasterxml.jackson.databind.JsonNode
  • com.fasterxml.jackson.databind.ObjectMapper
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.