Package org.kitesdk.data

Examples of org.kitesdk.data.DatasetDescriptor$Builder

92
93
94
95
96
97
98
99
100
101
102
    command.datasets = Lists.newArrayList("users");
    command.avroSchemaFile = avroSchemaFile.toString();
    command.run();

    DatasetDescriptor updated = new DatasetDescriptor.Builder(original)
        .schemaLiteral(schema2)
        .build();

    verify(repo).load("default", "users"); // need to load the current dataset
    verify(ds).getDescriptor(); // should inspect and use its descriptor
View Full Code Here
111
112
113
114
115
116
117
118
119
120
121
        "new.property=1234",
        "prop=with=equals"
    );
    command.run();

    DatasetDescriptor updated = new DatasetDescriptor.Builder(original)
        .property("new.property", "1234")
        .property("prop", "with=equals")
        .build();

    verify(repo).load("default", "users"); // need to load the current dataset
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
73
74
75
    }
  }

  @Test
  public void testDescriptorValidationPasses() {
    DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
        .schema(schema)
        .partitionStrategy(strategy)
        .build();
    Assert.assertEquals("Descriptor should have correct schema",
        schema, descriptor.getSchema());
    Assert.assertEquals("Descriptor should have correct strategy",
        strategy, descriptor.getPartitionStrategy());
  }
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
  // This class is Immutable and must be thread-safe
  protected final ThreadLocal<StorageKey> keys;

  protected AbstractRefinableView(Dataset<E> dataset, Class<E> type) {
    this.dataset = dataset;
    final DatasetDescriptor descriptor = dataset.getDescriptor();
    if (descriptor.isPartitioned()) {
      this.constraints = new Constraints(
          descriptor.getSchema(), descriptor.getPartitionStrategy());
      this.comparator = new MarkerComparator(descriptor.getPartitionStrategy());
      this.keys = new ThreadLocal<StorageKey>() {
        @Override
        protected StorageKey initialValue() {
          return new StorageKey(descriptor.getPartitionStrategy());
        }
      };
    } else {
      this.constraints = new Constraints(descriptor.getSchema());
      this.comparator = null;
      this.keys = null;
    }
    this.accessor = DataModelUtil.accessor(type, descriptor.getSchema());
    this.entityTest = constraints.toEntityPredicate(accessor);
  }
View Full Code Here
163
164
165
166
167
168
169
170
171
172
173
  public void testRefineIdentity() throws Exception {
      PartitionStrategy strategy = new PartitionStrategy.Builder()
              .identity("user_id")
              .build();

      DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
              .schemaUri("resource:standard_event.avsc")
              .partitionStrategy(strategy)
              .build();

      // Create a separate dataset to avoid conflicts with the above.
View Full Code Here
109
110
111
112
113
114
115
116
117
118
119
        .requiredString("email")
        .endRecord();
    PartitionStrategy strategy = new PartitionStrategy.Builder()
        .provided("version", "int")
        .build();
    DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
        .schema(user)
        .partitionStrategy(strategy)
        .build();

    Path datasetPath = new Path("file:" + testDirectory + "/provided/users");
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
  @Test
  public void testCreate() {
    Assert.assertFalse("Sanity check", provider.exists(NAMESPACE, NAME));

    DatasetDescriptor created = provider.create(NAMESPACE, NAME, testDescriptor);

    Assert.assertNotNull("Descriptor should be returned", created);
    Assert.assertTrue("Descriptor should exist", provider.exists(NAMESPACE, NAME));
    Assert.assertEquals("Schema should match",
        testDescriptor.getSchema(), created.getSchema());
    Assert.assertEquals("PartitionStrategy should match",
        testDescriptor.getPartitionStrategy(), created.getPartitionStrategy());
    Assert.assertEquals("Format should match",
        testDescriptor.getFormat(), created.getFormat());
    // the MetadataProvider optionally sets the location, nothing to test
  }
View Full Code Here
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
  public void testCreateWithLocation() throws URISyntaxException {
    Assert.assertFalse("Sanity check", provider.exists(NAMESPACE, NAME));

    String auth = getDFS().getUri().getAuthority();
    URI requestedLocation = new URI("hdfs://" + auth + "/tmp/data/my_data_set");
    DatasetDescriptor requested = new DatasetDescriptor.Builder(testDescriptor)
        .location(requestedLocation)
        .build();

    final DatasetDescriptor created;
    try {
      created = provider.create(NAMESPACE, NAME, requested);
    } catch (UnsupportedOperationException ex) {
      // this is expected if the provider doesn't support requested locations
      return;
    }

    // if supported, the location should be unchanged.
    Assert.assertNotNull("Descriptor should be returned", created);
    Assert.assertTrue("Descriptor should exist", provider.exists(NAMESPACE, NAME));
    Assert.assertEquals("Requested locations should match",
        requestedLocation, created.getLocation());
  }
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
  @Test
  public void testLoadSetsLocation() throws IOException {
    ensureCreated();

    DatasetDescriptor loaded = provider.load(NAMESPACE, NAME);
    Assert.assertNotNull("Loaded descriptor should have a location",
        loaded.getLocation());
    if (distributed) {
      // purposely call new Configuration() to test that the URI has HDFS info
      Assert.assertEquals(
          getDFS(),
          FileSystem.get(loaded.getLocation(), new Configuration()));
      Assert.assertEquals(
          "hdfs",
          loaded.getLocation().getScheme());
      Assert.assertEquals(
          getDFS().getUri().getAuthority(),
          loaded.getLocation().getAuthority());
    } else {
      // purposely call new Configuration() to test that the URI has FS info
      Assert.assertEquals(
          getFS(),
          FileSystem.get(loaded.getLocation(), new Configuration()));
      Assert.assertEquals(
          "file",
          loaded.getLocation().getScheme());
      Assert.assertEquals(
          getFS().getUri().getAuthority(),
          loaded.getLocation().getAuthority());
    }
  }
View Full Code Here
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
  @Test
  public void testLoad() {
    ensureCreated();

    DatasetDescriptor loaded = provider.load(NAMESPACE, NAME);

    Assert.assertNotNull("DatasetDescriptor should be returned", loaded);
    Assert.assertEquals("Schema should match",
        testDescriptor.getSchema(), loaded.getSchema());
    Assert.assertEquals("PartitionStrategy should match",
        testDescriptor.getPartitionStrategy(), loaded.getPartitionStrategy());
    Assert.assertEquals("Format should match",
        testDescriptor.getFormat(), loaded.getFormat());
  }
View Full Code Here
TOP

Related Classes of org.kitesdk.data.DatasetDescriptor$Builder

  • co.cask.tigon.sql.internal.DefaultGDATField
  • com.asakusafw.utils.java.model.syntax.Expression
  • com.asakusafw.utils.java.model.syntax.ModelFactory
  • com.asakusafw.utils.java.model.util.ImportBuilder
  • com.asakusafw.utils.java.model.util.TypeBuilder
  • com.fasterxml.jackson.databind.JsonNode
  • com.fasterxml.jackson.databind.ObjectMapper
  • com.google.gwt.autobean.shared.AutoBean.PropertyName
  • com.google.javascript.rhino.JSDocInfo
  • com.google.javascript.rhino.Node
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.