Package org.kitesdk.data

Examples of org.kitesdk.data.DatasetDescriptor

133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
        throw new ValidationException("Unknown mapping: " + mapping);
      }
    }

    // building the descriptor validates the schema and strategy
    DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
        .columnMapping(mappingBuilder.build())
        .schema(open(avroSchemaFile))
        .partitionStrategy(open(partitionStrategyFile))
        .build();

    String mapping = descriptor.getColumnMapping().toString(!minimize);

    output(mapping, console, outputPath);

    return 0;
  }
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
109
110
111
    org.junit.Assert.assertTrue("Repo should be a HCatalogExternalDatasetRepository",
        repo instanceof HiveExternalDatasetRepository);
    Assert.assertEquals("Repository URI", repoUri, repo.getUri());

    // verify location
    DatasetDescriptor created = repo.create("ns", "test",
        new DatasetDescriptor.Builder()
            .schemaLiteral("\"string\"")
            .build()).getDescriptor();
    Assert.assertEquals("Location should be in local FS",
        "file", created.getLocation().getScheme());
    Assert.assertTrue("Location should be in the repo path",
        created.getLocation().getPath().startsWith("/tmp/hive-repo"));
  }
View Full Code Here
119
120
121
122
123
124
125
126
127
128
129
130
131
132
    org.junit.Assert.assertTrue("Repo should be a HCatalogExternalDatasetRepository",
        repo instanceof HiveExternalDatasetRepository);
    Assert.assertEquals("Repository URI", repoUri, repo.getUri());

    // verify location
    DatasetDescriptor created = repo.create("ns", "test",
        new DatasetDescriptor.Builder()
            .schemaLiteral("\"string\"")
            .build()).getDescriptor();
    Assert.assertEquals("Location should be in local FS",
        "file", created.getLocation().getScheme());
    Assert.assertTrue("Location should be in the repo path",
        created.getLocation().getPath().endsWith("tmp/hive-repo/ns/test"));
  }
View Full Code Here
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
    org.junit.Assert.assertTrue("Repo should be a HCatalogExternalDatasetRepository",
        repo instanceof HiveExternalDatasetRepository);
    Assert.assertEquals("Repository URI", repoUri, repo.getUri());

    // verify location
    DatasetDescriptor created = repo.create("tmp", "test",
        new DatasetDescriptor.Builder()
        .schemaLiteral("\"string\"")
        .build()).getDescriptor();
    Assert.assertEquals("Location should be in HDFS",
        "hdfs", created.getLocation().getScheme());
    Assert.assertEquals("Location should have the correct HDFS host",
        hdfsUri.getHost(), created.getLocation().getHost());
    Assert.assertEquals("Location should have the correct HDFS port",
        hdfsUri.getPort(), created.getLocation().getPort());
    Assert.assertTrue("Location should be in the repo path",
        created.getLocation().getPath().startsWith("/tmp/hive-repo"));
  }
View Full Code Here
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
                                    @Nullable URI location) {
    if (getMetaStoreUtil().exists(namespace, name)) {
      return namespace;
    }
    try {
      DatasetDescriptor descriptor = HiveUtils.descriptorForTable(
          conf, getMetaStoreUtil().getTable(URIBuilder.NAMESPACE_DEFAULT, name));
      URI expectedLocation = location;
      if (location == null) {
        expectedLocation = expectedLocation(namespace, name);
      }
      if ((expectedLocation == null) ||
          pathsEquivalent(expectedLocation, descriptor.getLocation())) {
        // table in the default db has the location that would have been used
        return URIBuilder.NAMESPACE_DEFAULT;
      }
      // fall through and return null
    } catch (DatasetNotFoundException e) {
View Full Code Here
125
126
127
128
129
130
131
132
133
134
135
    reader.hasNext();
  }

  @Test(expected = UnknownFormatException.class)
  public void testUnknownFormat() throws IOException {
    final DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
        .schema(STRING_SCHEMA)
        .format(Accessor.getDefault().newFormat("explode!"))
        .build();

    MultiFileDatasetReader<Record> reader = new MultiFileDatasetReader<Record>(
View Full Code Here
53
54
55
56
57
58
59
60
61
62
63
  public void testBasicUse() throws Exception {
    command.avroSchemaFile = "resource:test-schemas/user.avsc";
    command.datasets = Lists.newArrayList("users");
    command.run();

    DatasetDescriptor expectedDescriptor = new DatasetDescriptor.Builder()
        .schemaUri("resource:test-schemas/user.avsc")
        .build();

    verify(repo).create("default", "users", expectedDescriptor);
    verify(console).debug(contains("Created"), eq("users"));
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
    command.avroSchemaFile = "resource:test-schemas/user.avsc";
    command.datasets = Lists.newArrayList("users");
    command.format = "parquet";
    command.run();

    DatasetDescriptor expectedDescriptor = new DatasetDescriptor.Builder()
        .schemaUri("resource:test-schemas/user.avsc")
        .format("parquet")
        .build();

    verify(repo).create("default", "users", expectedDescriptor);
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
    if (exists(namespace, name)) {
      throw new DatasetExistsException(
          "Dataset already exists for name:" + name);
    }

    DatasetDescriptor newDescriptor;
    if (descriptor.getLocation() == null) {
      newDescriptor = new DatasetDescriptor.Builder(descriptor)
          .location(fs.makeQualified(new Path(newLocation(name))))
          .build();
    } else {
View Full Code Here
85
86
87
88
89
90
91
92
93
94
95
    Dataset<Record> dataset = repo.create(NAMESPACE, NAME,
        new DatasetDescriptor.Builder(testDescriptor)
            .format(Formats.AVRO)
            .build());

    DatasetDescriptor changed =
        new DatasetDescriptor.Builder(dataset.getDescriptor())
        .format(Formats.PARQUET)
        .build();

    try {
View Full Code Here
TOP

Related Classes of org.kitesdk.data.DatasetDescriptor

  • org.apache.avro.Schema
  • org.apache.flume.sink.kite.DatasetSink
  • org.apache.flume.sink.kite.TestDatasetSink
  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.Path
  • org.apache.sqoop.mapreduce.ParquetJob
  • org.kitesdk.cli.commands.CreateColumnMappingCommand
  • org.kitesdk.cli.commands.CreateDatasetCommand
  • org.kitesdk.cli.commands.CreatePartitionStrategyCommand
  • org.kitesdk.cli.commands.CSVImportCommand
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.