Package org.apache.hadoop.io.compress

Examples of org.apache.hadoop.io.compress.CompressionCodec

300
301
302
303
304
305
306
307
308
309
310
    Configuration conf = new Configuration();
    List<Class<? extends CompressionCodec>> codecs = CompressionCodecFactory
        .getCodecClasses(conf);
    // Wish we could base this on DefaultCodec but appears not all codec's
    // extend DefaultCodec(Lzo)
    CompressionCodec codec = null;
    ArrayList<String> codecStrs = new ArrayList<String>();
    codecStrs.add("None");
    for (Class<? extends CompressionCodec> cls : codecs) {
      codecStrs.add(cls.getSimpleName());
      if (codecMatches(cls, codecName)) {
View Full Code Here
72
73
74
75
76
77
78
79
80
81
82
          Text.class, Text.class, CompressionType.NONE, (CompressionCodec) null);
      holder = new SequenceFileWriterHolder<SequenceFile.Writer>(writer, p);
    }
    else {
      Class<?> clazz = ClassUtils.resolveClassName(codecInfo.getCodecClass(), getClass().getClassLoader());
      CompressionCodec compressionCodec = (CompressionCodec) ReflectionUtils.newInstance(clazz,
          getConfiguration());
      writer = SequenceFile.createWriter(fs,
          getConfiguration(), getResolvedPath(),
          Text.class, Text.class, CompressionType.RECORD, compressionCodec);
      holder = new SequenceFileWriterHolder<SequenceFile.Writer>(writer, p);
View Full Code Here
115
116
117
118
119
120
121
122
123
124
125
126
127
    if (!isCompressed()) {
      holder.setStream(wout);
    } else {
      // TODO: will isCompressed() really guard for npe against getCodec()
      Class<?> clazz = ClassUtils.resolveClassName(getCodec().getCodecClass(), getClass().getClassLoader());
      CompressionCodec compressionCodec = (CompressionCodec) ReflectionUtils.newInstance(clazz,
          getConfiguration());
      OutputStream out = compressionCodec.createOutputStream(wout);
      holder.setWrappedStream(wout);
      holder.setStream(out);
    }
    return holder;
  }
View Full Code Here
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
        input.seek(getSplit().getStart());
        holder.setStream(input);
      }
    } else {
      Class<?> clazz = ClassUtils.resolveClassName(getCodec().getCodecClass(), getClass().getClassLoader());
      CompressionCodec compressionCodec = (CompressionCodec) ReflectionUtils.newInstance(clazz,
          getConfiguration());
      Decompressor decompressor = CodecPool.getDecompressor(compressionCodec);
      if (getSplit() == null) {
        // with codec, no split
        FSDataInputStream winput = fs.open(p);
        InputStream input = compressionCodec.createInputStream(winput, decompressor);
        holder.setWrappedStream(winput);
        holder.setStream(input);
      } else {
        // with codec, with split
        long start = getSplit().getStart();
View Full Code Here
331
332
333
334
335
336
337
338
339
340
341
    Configuration conf = new Configuration();
    List<Class<? extends CompressionCodec>> codecs = CompressionCodecFactory
        .getCodecClasses(conf);
    // Wish we could base this on DefaultCodec but appears not all codec's
    // extend DefaultCodec(Lzo)
    CompressionCodec codec = null;
    ArrayList<String> codecStrs = new ArrayList<String>();
    codecStrs.add("None");
    for (Class<? extends CompressionCodec> cls : codecs) {
      codecStrs.add(cls.getSimpleName());
      if (codecMatches(cls, codecName)) {
View Full Code Here
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
  public InputStream getInputStream() throws IOException {
    if (exists) {
      InputStream stream = fs.open(path);

      if (codecsFactory != null) {
        CompressionCodec codec = codecsFactory.getCodec(path);
        if (codec != null) {
          // the pool is not used since the returned inputstream needs to be decorated
          // to return the decompressor on close which can mask the actual stream
          // it's also unclear whether the pool is actually useful or not
          // Decompressor decompressor = CodecPool.getDecompressor(codec);
          // stream = (decompressor != null ? codec.createInputStream(stream, decompressor) : codec.createInputStream(stream));
          stream = codec.createInputStream(stream);
        }
      }

      return stream;
    }
View Full Code Here
590
591
592
593
594
595
596
597
598
            return new XMLFileRecordReader(recordIdentifier);
        } 
       
        @Override
        protected boolean isSplitable(JobContext context, Path filename) {
           CompressionCodec codec =
              new CompressionCodecFactory(context.getConfiguration()).getCodec(filename);
           return (!(codec == null)) ? isSplitable : true;
        }
View Full Code Here
641
642
643
644
645
646
647
648
649
650
651
652
653
654
               this.xmlLoaderBPIS = new XMLLoaderBufferedPositionedInputStream(in,start,end);
            }
            else if (file.toString().endsWith(".gz"))
            {
              CompressionCodecFactory compressionCodecs =  new CompressionCodecFactory(job);
              final CompressionCodec codec = compressionCodecs.getCodec(file);
               if (codec != null) {
                  end = Long.MAX_VALUE;
                    CompressionInputStream stream = codec.createInputStream(fileIn);
                    this.xmlLoaderBPIS = new XMLLoaderBufferedPositionedInputStream(stream,start,end);
                  }
            }
           
            else
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
        FileSystem fs = file.getFileSystem(job);
        CompressionType compressionType = CompressionType.BLOCK;
        // find the right codec
        Class<?> codecClass = getOutputCompressorClass(job, DefaultCodec.class);
        CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, job);

        // set the schema metadata
        /* begin jays code */
        SequenceFile.Metadata meta = new SequenceFile.Metadata();
        meta.set(new Text("key.schema"), new Text(getSchema("reducer.output.key.schema", job)));
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
        Integer.MAX_VALUE);
    start = split.getStart();
    end = start + split.getLength();
    final Path file = split.getPath();
    compressionCodecs = new CompressionCodecFactory(job);
    final CompressionCodec codec = compressionCodecs.getCodec(file);

    // open the file and seek to the start of the split
    FileSystem fs = file.getFileSystem(job);
    FSDataInputStream fileIn = fs.open(split.getPath());
    boolean skipFirstLine = false;
    if (codec != null) {
      in = new LineReader(codec.createInputStream(fileIn), job);
      end = Long.MAX_VALUE;
    } else {
      if (start != 0) {
        skipFirstLine = true;
        --start;
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.compress.CompressionCodec

  • com.cloudera.flume.handlers.hdfs.CustomDfsSink
  • com.cloudera.sqoop.TestExport
  • com.datasalt.pangool.tuplemr.mapred.lib.input.TupleTextInputFormat
  • com.datasalt.pangool.tuplemr.mapred.lib.input.TupleTextInputFormat$TupleTextInputReader
  • com.datasalt.pangool.tuplemr.mapred.lib.output.TupleOutputFormat
  • com.facebook.presto.hive.AbstractTestHiveFileFormats
  • com.hadoop.compression.lzo.LzoIndex
  • com.hadoop.compression.lzo.TestLzoCodec
  • com.hadoop.mapreduce.LzoLineRecordReader
  • com.hadoop.mapreduce.LzoSplitRecordReader
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.