Package com.cloudera.cdk.data

Examples of com.cloudera.cdk.data.Format

38
39
40
41
42
43
44
45
46
47
48
      fs.mkdirs(path);
    } catch (IOException ex) {
      throw new DatasetWriterException("Could not create path:" + path, ex);
    }

    final Format format = descriptor.getFormat();
    final Path file = new Path(path, uniqueFilename(descriptor.getFormat()));

    if (Formats.PARQUET.equals(format)) {
      return new ParquetFileSystemDatasetWriter(fs, file, descriptor.getSchema());
    } else if (Formats.AVRO.equals(format)) {
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
67
68
  @Override
  public void open() {
    Preconditions.checkState(state.equals(ReaderWriterState.NEW),
      "A reader may not be opened more than once - current state:%s", state);

    final Format format = descriptor.getFormat();
    if (!(Formats.AVRO.equals(format) || Formats.PARQUET.equals(format)
        || Formats.CSV.equals(format))) {
      throw new UnknownFormatException("Cannot open format:" + format.getName());
    }

    this.state = ReaderWriterState.OPEN;
  }
View Full Code Here
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
      table.setProperty(CUSTOM_PROPERTIES_PROPERTY_NAME,
          NAME_JOINER.join(descriptor.listProperties()));
    }

    // translate from Format to SerDe
    final Format format = descriptor.getFormat();
    if (FORMAT_TO_SERDE.containsKey(format)) {
      table.setSerializationLib(FORMAT_TO_SERDE.get(format));
      try {
        table.setInputFormatClass(FORMAT_TO_INPUT_FORMAT.get(format));
        table.setOutputFormatClass(FORMAT_TO_OUTPUT_FORMAT.get(format));
      } catch (HiveException ex) {
        throw new MetadataProviderException(
            "Failed to set input/output formats for format:" +
            format.getName(), ex);
      }
    } else {
      throw new UnknownFormatException(
          "No known serde for format:" + format.getName());
    }
    try {
    } catch (Exception e) {
      throw new MetadataProviderException("Error configuring Hive Avro table, " +
          "table creation failed", e);
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
      if (type.isAssignableFrom(GenericData.Record.class)) {
        avroType = (AvroType<E>) Avros.generics(dataset.getDescriptor().getSchema());
      } else {
        avroType = Avros.records(type);
      }
      final Format format = dataset.getDescriptor().getFormat();
      if (Formats.PARQUET.equals(format)) {
        return new AvroParquetFileSource<E>(paths, avroType);
      } else if (Formats.AVRO.equals(format)) {
        return new AvroFileSource<E>(paths, avroType);
      } else {
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
   * filesystem-based.
   */
  public static Target asTarget(Dataset dataset) {
    Path directory = Accessor.getDefault().getDirectory(dataset);
    if (directory != null) {
      final Format format = dataset.getDescriptor().getFormat();
      if (Formats.PARQUET.equals(format)) {
        return new AvroParquetFileTarget(directory);
      } else if (Formats.AVRO.equals(format)) {
        return new AvroFileTarget(directory);
      } else {
View Full Code Here
TOP

Related Classes of com.cloudera.cdk.data.Format

  • com.cloudera.cdk.data.crunch.CrunchDatasets
  • com.cloudera.cdk.data.filesystem.FileSystemWriters
  • com.cloudera.cdk.data.filesystem.MultiFileDatasetReader
  • com.cloudera.cdk.data.hcatalog.HiveUtils
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.