Package org.kitesdk.data

Examples of org.kitesdk.data.DatasetDescriptor$Builder

81
82
83
84
85
86
87
88
89
90
91
    Dataset<GenericData.Record> dataset = managed.load("ns", "external");
    Assert.assertNotNull("Should open external dataset with managed", dataset);
    Assert.assertEquals("Should match external dataset",
        external.load("ns", "external").getDescriptor(), dataset.getDescriptor());

    DatasetDescriptor updatedDescriptor =
        new DatasetDescriptor.Builder(dataset.getDescriptor())
            .property("kite.writer.cache-size", "34")
            .schemaLiteral("\"string\"")
            .build();
View Full Code Here
116
117
118
119
120
121
122
123
124
125
126
    Dataset<GenericData.Record> dataset = external.load("default", "managed");
    Assert.assertNotNull("Should open managed dataset with external", dataset);
    Assert.assertEquals("Should match managed dataset",
        managed.load("default", "managed").getDescriptor(), dataset.getDescriptor());

    DatasetDescriptor updatedDescriptor =
        new DatasetDescriptor.Builder(dataset.getDescriptor())
            .property("kite.writer.cache-size", "34")
            .schemaLiteral("\"string\"")
            .build();
View Full Code Here
60
61
62
63
64
65
66
67
68
69
  @Test
  public void testCreateAssignsCorrectLocation() {
    ensureCreated();

    DatasetDescriptor loaded = provider.load(NAMESPACE, NAME);
    Path assignedPath = new Path(loaded.getLocation().getPath());
    Assert.assertEquals("Path should be in the test directory",
        new Path(testDirectory, new Path(NAMESPACE, NAME)), assignedPath);
  }
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
        new Path(testDirectory, new Path(NAMESPACE, NAME)), assignedPath);
  }

  @Test
  public void testRejectsDuplicatePartitionNames() {
    final DatasetDescriptor descriptor = new DatasetDescriptor.Builder(testDescriptor)
        .partitionStrategy(new PartitionStrategy.Builder()
            .identity("timestamp", "timestamp")
            .build())
        .build();
    TestHelpers.assertThrows("Should reject duplicate field and partition name",
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
            "Invalid partition <field:type>: " + partition);
      }
    }

    // building the descriptor validates the schema and strategy
    DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
        .partitionStrategy(strategyBuilder.build())
        .schema(open(avroSchemaFile))
        .build();

    String strategy = descriptor.getPartitionStrategy().toString(!minimize);

    output(strategy, console, outputPath);

    return 0;
  }
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
            Iterators.getNext(parts, null),
            Iterators.getNext(parts, null));
      }
    }

    DatasetDescriptor descriptor = descriptorBuilder.build();

    if (isDataUri(dataset)) {
      Datasets.<GenericData.Record, Dataset<GenericData.Record>> update(dataset, descriptor, GenericData.Record.class);
    } else {
      getDatasetRepository().update(namespace, dataset, descriptor);
View Full Code Here
139
140
141
142
143
144
145
146
147
148
149
    // this used to be a relative external URI, but is now a managed URI
    String uri = "dataset:hive:data/ds";

    DatasetRepository repo = DatasetRepositories
        .repositoryFor("repo:hive:/tmp/data");
    DatasetDescriptor withLocation = new DatasetDescriptor.Builder(DESCRIPTOR)
        .location("file:/tmp/data/ds") // old location
        .build();
    Dataset<GenericRecord> expected = repo.create(
        "default", "ds", withLocation, GenericRecord.class);
View Full Code Here
210
211
212
213
214
215
216
217
218
219
220
    // this used to be a relative external URI, but is now a managed URI
    String uri = "dataset:hive:ds";

    DatasetRepository repo = DatasetRepositories
        .repositoryFor("repo:hive:/tmp/data");
    DatasetDescriptor withLocation = new DatasetDescriptor.Builder(DESCRIPTOR)
        .location("file:/tmp/data/ds") // old location
        .build();
    Dataset<GenericRecord> expected = repo.create(
        "default", "ds", withLocation, GenericRecord.class);
View Full Code Here
315
316
317
318
319
320
321
322
323
324
325
    // this used to be a relative external URI, but is now a managed URI
    String uri = "dataset:hive:/data/ds";

    DatasetRepository repo = DatasetRepositories
        .repositoryFor("repo:hive:/tmp/data");
    DatasetDescriptor withLocation = new DatasetDescriptor.Builder(DESCRIPTOR)
        .location("file:/tmp/data/ds") // old location
        .build();
    Dataset<GenericRecord> expected = repo.create(
        "default", "ds", withLocation, GenericRecord.class);
View Full Code Here
386
387
388
389
390
391
392
393
394
395
396
    // this used to be a relative external URI, but is now a managed URI
    String uri = "dataset:hive:/ds";

    DatasetRepository repo = DatasetRepositories
        .repositoryFor("repo:hive:/tmp/data");
    DatasetDescriptor withLocation = new DatasetDescriptor.Builder(DESCRIPTOR)
        .location("file:/tmp/data/ds") // old location
        .build();
    Dataset<GenericRecord> expected = repo.create(
        "default", "ds", withLocation, GenericRecord.class);
View Full Code Here
TOP

Related Classes of org.kitesdk.data.DatasetDescriptor$Builder

  • co.cask.tigon.sql.internal.DefaultGDATField
  • com.asakusafw.utils.java.model.syntax.Expression
  • com.asakusafw.utils.java.model.syntax.ModelFactory
  • com.asakusafw.utils.java.model.util.ImportBuilder
  • com.asakusafw.utils.java.model.util.TypeBuilder
  • com.fasterxml.jackson.databind.JsonNode
  • com.fasterxml.jackson.databind.ObjectMapper
  • com.google.gwt.autobean.shared.AutoBean.PropertyName
  • com.google.javascript.rhino.JSDocInfo
  • com.google.javascript.rhino.Node
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.