Package parquet.hadoop.metadata

Examples of parquet.hadoop.metadata.CompressionCodecName

227
228
229
230
231
232
233
234
235
236
237
            LOG.debug(MessageFormat.format(
                    "Parquet file output ({0}): {1}",
                    path,
                    conf));
        }
        CompressionCodecName compress = conf.getCompressionCodecName();
        if (compress != null) {
            options.setCompressionCodecName(compress);
        }
        Integer block = conf.getBlockSize();
        if (block != null) {
View Full Code Here
242
243
244
245
246
247
248
249
250
251
252
    }

    private void consumeCompression(
            DmdlSemantics environment, AstAttribute attribute,
            Map<String, AstAttributeElement> elements, ParquetFileTrait result) {
        CompressionCodecName option = consumeOption(
                environment, attribute, elements,
                ELEMENT_COMPRESSION, Messages.getString("ParquetFileDriver.labelCompression"), //$NON-NLS-1$
                CompressionCodecName.values());
        if (option != null) {
            result.configuration().withCompressionCodecName(option);
View Full Code Here
138
139
140
141
142
143
144
145
146
147
148
    // messageSchema = messageSchema.substring(schemaType, messageSchema.length() - 1);
    messageSchema += "}";

    MessageType schema = MessageTypeParser.parseMessageType(messageSchema);

    CompressionCodecName codec = CompressionCodecName.UNCOMPRESSED;
    ParquetFileWriter w = new ParquetFileWriter(configuration, schema, path);
    w.start();
    HashMap<String, Integer> columnValuesWritten = new HashMap();
    int valsWritten;
    for (int k = 0; k < numberRowGroups; k++) {
View Full Code Here
259
260
261
262
263
264
265
266
267
268
269
    //messageSchema = messageSchema.substring(schemaType, messageSchema.length() - 1);
    messageSchema += "}";

    MessageType schema = MessageTypeParser.parseMessageType(messageSchema);

    CompressionCodecName codec = CompressionCodecName.UNCOMPRESSED;
    ParquetFileWriter w = new ParquetFileWriter(configuration, schema, path);
    w.start();
    HashMap<String, Integer> columnValuesWritten = new HashMap();
    int valsWritten;
    for (int k = 0; k < props.numberRowGroups; k++){
View Full Code Here
156
157
158
159
160
161
162
163
164
165
166
    //messageSchema = messageSchema.substring(schemaType, messageSchema.length() - 1);
    messageSchema += "}";

    MessageType schema = MessageTypeParser.parseMessageType(messageSchema);

    CompressionCodecName codec = CompressionCodecName.UNCOMPRESSED;
    ParquetFileWriter w = new ParquetFileWriter(configuration, schema, path);
    w.start();
    HashMap<String, Integer> columnValuesWritten = new HashMap();
    int valsWritten;
    for (int k = 0; k < props.numberRowGroups; k++){
View Full Code Here
89
90
91
92
93
94
95
96
97
    String compressionName = tableProperties.getProperty(ParquetOutputFormat.COMPRESSION);
    if (compressionName != null && !compressionName.isEmpty()) {
      //get override compression properties via "tblproperties" clause if it is set
      LOG.debug("get override compression properties via tblproperties");
      CompressionCodecName codecName = CompressionCodecName.fromConf(compressionName);
      conf.set(ParquetOutputFormat.COMPRESSION, codecName.name());
    }
  }
View Full Code Here
71
72
73
74
75
76
77
78
79
80
81
    logger.debug(
      "Opening data file with pathTmp:{} (final path will be path:{})",
      pathTmp, path);

    try {
      CompressionCodecName codecName = CompressionCodecName.UNCOMPRESSED;
      if (enableCompression) {
         if (SnappyCodec.isNativeCodeLoaded()) {
           codecName = CompressionCodecName.SNAPPY;
         } else {
           logger.warn("Compression enabled, but Snappy native code not loaded. " +
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
    this.compressionType = compressionType;
  }

  @Override
  public void open() throws IOException {
    CompressionCodecName codecName = CompressionCodecName.UNCOMPRESSED;
    if (enableCompression) {
      codecName = getCompressionCodecName();
    }
    avroParquetWriter = new AvroParquetWriter<E>(fileSystem.makeQualified(path),
        schema, codecName, DEFAULT_BLOCK_SIZE,
View Full Code Here
157
158
159
160
161
162
163
164
165
166
167
    //messageSchema = messageSchema.substring(schemaType, messageSchema.length() - 1);
    messageSchema += "}";

    MessageType schema = MessageTypeParser.parseMessageType(messageSchema);

    CompressionCodecName codec = CompressionCodecName.UNCOMPRESSED;
    ParquetFileWriter w = new ParquetFileWriter(configuration, schema, path);
    w.start();
    HashMap<String, Integer> columnValuesWritten = new HashMap();
    int valsWritten;
    for (int k = 0; k < props.numberRowGroups; k++){
View Full Code Here
239
240
241
242
243
244
245
246
247
248
  public RecordWriter<Void, T> getRecordWriter(TaskAttemptContext taskAttemptContext)
      throws IOException, InterruptedException {

    final Configuration conf = getConfiguration(taskAttemptContext);

    CompressionCodecName codec = getCodec(taskAttemptContext);
    String extension = codec.getExtension() + ".parquet";
    Path file = getDefaultWorkFile(taskAttemptContext, extension);
    return getRecordWriter(conf, file, codec);
  }
View Full Code Here
TOP

Related Classes of parquet.hadoop.metadata.CompressionCodecName

  • com.asakusafw.directio.hive.parquet.AbstractParquetFileFormat
  • com.asakusafw.dmdl.directio.hive.parquet.ParquetFileDriver
  • com.cloudera.cdk.data.filesystem.ParquetFileSystemDatasetWriter
  • org.apache.drill.exec.store.parquet.TestFileGenerator
  • org.apache.drill.exec.store.ParquetRecordReaderTest
  • org.apache.hadoop.hive.ql.io.parquet.write.ParquetRecordWriterWrapper
  • org.kitesdk.data.spi.filesystem.ParquetAppender
  • parquet.hadoop.codec.CodecConfig
  • parquet.hadoop.mapred.DeprecatedParquetOutputFormat$RecordWriterWrapper
  • parquet.hadoop.ParquetInputSplit
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.