Package org.apache.avro.file

Examples of org.apache.avro.file.CodecFactory

63
64
65
66
67
68
69
70
71
72
73
    DataFileWriter<GenericRecord> dfw = new DataFileWriter<GenericRecord>(gdw);

    if (isCompressed) {
      int level = jobConf.getInt(DEFLATE_LEVEL_KEY, DEFAULT_DEFLATE_LEVEL);
      String codecName = jobConf.get(OUTPUT_CODEC, DEFLATE_CODEC);
      CodecFactory factory = codecName.equals(DEFLATE_CODEC)
          ? CodecFactory.deflateCodec(level)
          : CodecFactory.fromString(codecName);
      dfw.setCodec(factory);
    }
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
      JobConf job) throws UnsupportedEncodingException {
    if (FileOutputFormat.getCompressOutput(job)) {
      int level = job.getInt(DEFLATE_LEVEL_KEY,
          DEFAULT_DEFLATE_LEVEL);
      String codecName = job.get(AvroJob.OUTPUT_CODEC, DEFLATE_CODEC);
      CodecFactory factory = codecName.equals(DEFLATE_CODEC)
        ? CodecFactory.deflateCodec(level)
        : CodecFactory.fromString(codecName);
      writer.setCodec(factory);
    }
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
    if (FileOutputFormat.getCompressOutput(context)) {
      int level = context.getConfiguration()
        .getInt(DEFLATE_LEVEL_KEY, DEFAULT_DEFLATE_LEVEL);
      String codecName = context.getConfiguration()
        .get(org.apache.avro.mapred.AvroJob.OUTPUT_CODEC, DEFLATE_CODEC);
      CodecFactory factory =
        codecName.equals(DEFLATE_CODEC) ? CodecFactory.deflateCodec(level)
          : CodecFactory.fromString(codecName);
      writer.setCodec(factory);
    }
View Full Code Here
79
80
81
82
83
84
85
86
87
88
89
    }
 
    String codecName = opts.hasArgument(codec)
      ? codec.value(opts)
      : DEFLATE_CODEC;
    CodecFactory codecFactory = codecName.equals(DEFLATE_CODEC)
      ? CodecFactory.deflateCodec(compressionLevel)
      : CodecFactory.fromString(codecName);

    BufferedInputStream inStream = Util.fileOrStdin(nargs.get(0), stdin);
    BufferedOutputStream outStream = Util.fileOrStdout(nargs.get(1), out);
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
    DataFileStream<GenericRecord> reader = new DataFileStream<GenericRecord>(
        input, new GenericDatumReader<GenericRecord>());
    Schema schema = reader.getSchema();
    DataFileWriter<GenericRecord> writer = new DataFileWriter<GenericRecord>(
        new GenericDatumWriter<GenericRecord>());
    CodecFactory codec = opts.valueOf(codecOpt).equals("deflate")
        ? CodecFactory.deflateCodec(Integer.parseInt(levelOpt.value(opts)))
        : CodecFactory.fromString(codecOpt.value(opts));
    writer.setCodec(codec);
    for (String key : reader.getMetaKeys()) {
      if (!DataFileWriter.isReservedMeta(key)) {
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
    if (org.apache.hadoop.mapred.FileOutputFormat.getCompressOutput(jc)) {
      int level = conf.getInt(org.apache.avro.mapred.AvroOutputFormat.DEFLATE_LEVEL_KEY,
          org.apache.avro.mapred.AvroOutputFormat.DEFAULT_DEFLATE_LEVEL);
      String codecName = conf.get(AvroJob.OUTPUT_CODEC,
          org.apache.avro.file.DataFileConstants.DEFLATE_CODEC);
      CodecFactory codec = codecName.equals(org.apache.avro.file.DataFileConstants.DEFLATE_CODEC)
          ? CodecFactory.deflateCodec(level)
          : CodecFactory.fromString(codecName);
      WRITER.setCodec(codec);
    }
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
      new DataFileWriter<T>(new ReflectDatumWriter<T>());

    if (FileOutputFormat.getCompressOutput(job)) {
      int level = job.getInt(DEFLATE_LEVEL_KEY, DEFAULT_DEFLATE_LEVEL);
      String codecName = job.get(AvroJob.OUTPUT_CODEC, DEFLATE_CODEC);
      CodecFactory factory = codecName.equals(DEFLATE_CODEC)
        ? CodecFactory.deflateCodec(level)
        : CodecFactory.fromString(codecName);
      writer.setCodec(factory);
    }
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
    DataFileWriter<GenericRecord> dfw = new DataFileWriter<GenericRecord>(gdw);

    if (isCompressed) {
      int level = jobConf.getInt(DEFLATE_LEVEL_KEY, DEFAULT_DEFLATE_LEVEL);
      String codecName = jobConf.get(OUTPUT_CODEC, DEFLATE_CODEC);
      CodecFactory factory = codecName.equals(DEFLATE_CODEC)
          ? CodecFactory.deflateCodec(level)
          : CodecFactory.fromString(codecName);
      dfw.setCodec(factory);
    }
View Full Code Here
128
129
130
131
132
133
134
135
136
137
138
    dataFileWriter = new DataFileWriter<Object>(writer);

    dataFileWriter.setSyncInterval(syncIntervalBytes);

    try {
      CodecFactory codecFactory = CodecFactory.fromString(compressionCodec);
      dataFileWriter.setCodec(codecFactory);
    } catch (AvroRuntimeException e) {
      logger.warn("Unable to instantiate avro codec with name (" +
          compressionCodec + "). Compression disabled. Exception follows.", e);
    }
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
        DataFileWriter<Object> writer = new DataFileWriter<Object>(new PigAvroDatumWriter(schema));

        if (FileOutputFormat.getCompressOutput(context)) {
            int level = conf.getInt(DEFLATE_LEVEL_KEY, DEFAULT_DEFLATE_LEVEL);
            String codecName = conf.get(OUTPUT_CODEC, DEFLATE_CODEC);
            CodecFactory factory = codecName.equals(DEFLATE_CODEC)
                ? CodecFactory.deflateCodec(level)
                : CodecFactory.fromString(codecName);
            writer.setCodec(factory);
        }
View Full Code Here
TOP

Related Classes of org.apache.avro.file.CodecFactory

  • com.cloudera.flume.handlers.avro.AvroDataFileOutputFormat
  • com.datasalt.pangool.examples.avro.TestAvroTweetsJoin
  • com.datasalt.pangool.tuplemr.avro.AvroOutputFormat
  • com.datasalt.pangool.tuplemr.mapred.lib.output.TupleOutputFormat
  • com.linkedin.haivvreo.AvroContainerOutputFormat
  • org.apache.avro.hadoop.file.HadoopCodecFactory
  • org.apache.avro.hadoop.file.TestHadoopCodecFactory
  • org.apache.avro.mapred.AvroOutputFormat
  • org.apache.avro.mapred.TestAvroOutputFormat
  • org.apache.avro.mapreduce.lib.output.AvroOutputFormat
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.