Package org.apache.hadoop.io.compress

Examples of org.apache.hadoop.io.compress.CompressionCodecFactory

83
84
85
86
87
88
89
90
91
92
93
    // open the file and seek to the start of the split
    final FileSystem fs = file.getFileSystem(job);
    fileIn = fs.open(file);
   
    CompressionCodec codec = new CompressionCodecFactory(job).getCodec(file);
    if (null!=codec) {
      isCompressedInput = true; 
      decompressor = CodecPool.getDecompressor(codec);
      if (codec instanceof SplittableCompressionCodec) {
        final SplitCompressionInputStream cIn =
View Full Code Here
156
157
158
159
160
161
162
163
164
165
166
  }
 
  @Override
  protected boolean isSplitable(JobContext context, Path file) {
    final CompressionCodec codec =
      new CompressionCodecFactory(context.getConfiguration()).getCodec(file);
    if (null == codec) {
      return true;
    }
    return codec instanceof SplittableCompressionCodec;
  }
View Full Code Here
194
195
196
197
198
199
200
201
202
203
204
205
206
        boolean isCompressed = in.readBoolean();
        v.visit(ImageElement.IS_COMPRESSED, String.valueOf(isCompressed));
        if (isCompressed) {
          String codecClassName = Text.readString(in);
          v.visit(ImageElement.COMPRESS_CODEC, codecClassName);
          CompressionCodecFactory codecFac = new CompressionCodecFactory(
              new Configuration());
          CompressionCodec codec = codecFac.getCodecByClassName(codecClassName);
          if (codec == null) {
            throw new IOException("Image compression codec not supported: "
                + codecClassName);
          }
          in = new DataInputStream(codec.createInputStream(in));
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
    this.maxLineLength = job.getInt("mapred.linerecordreader.maxlength",
                                    Integer.MAX_VALUE);
    start = split.getStart();
    end = start + split.getLength();
    final Path file = split.getPath();
    compressionCodecs = new CompressionCodecFactory(job);
    final CompressionCodec codec = compressionCodecs.getCodec(file);

    // open the file and seek to the start of the split
    FileSystem fs = file.getFileSystem(job);
    FSDataInputStream fileIn = fs.open(split.getPath());
View Full Code Here
91
92
93
94
95
96
97
98
99
100
101
102
   */
  private static FSImageCompression createCompression(Configuration conf,
                                                      String codecClassName)
    throws IOException {

    CompressionCodecFactory factory = new CompressionCodecFactory(conf);
    CompressionCodec codec = factory.getCodecByClassName(codecClassName);
    if (codec == null) {
      throw new IOException("Not a supported codec: " + codecClassName);
    }

    return new FSImageCompression(codec);
View Full Code Here
166
167
168
169
170
171
172
173
174
175
176
177
178
        boolean isCompressed = in.readBoolean();
        v.visit(ImageElement.IS_COMPRESSED, imageVersion);
        if (isCompressed) {
          String codecClassName = Text.readString(in);
          v.visit(ImageElement.COMPRESS_CODEC, codecClassName);
          CompressionCodecFactory codecFac = new CompressionCodecFactory(
              new Configuration());
          CompressionCodec codec = codecFac.getCodecByClassName(codecClassName);
          if (codec == null) {
            throw new IOException("Image compression codec not supported: "
                + codecClassName);
          }
          in = new DataInputStream(codec.createInputStream(in));
View Full Code Here
145
146
147
148
149
150
151
152
153
154
155
  }
 
  @Override
  protected boolean isSplitable(JobContext context, Path file) {
    final CompressionCodec codec =
      new CompressionCodecFactory(context.getConfiguration()).getCodec(file);
    if (null == codec) {
      return true;
    }
    return codec instanceof SplittableCompressionCodec;
  }
View Full Code Here
165
166
167
168
169
170
171
172
173
174
175
176
177
        boolean isCompressed = in.readBoolean();
        v.visit(ImageElement.IS_COMPRESSED, imageVersion);
        if (isCompressed) {
          String codecClassName = Text.readString(in);
          v.visit(ImageElement.COMPRESS_CODEC, codecClassName);
          CompressionCodecFactory codecFac = new CompressionCodecFactory(
              new Configuration());
          CompressionCodec codec = codecFac.getCodecByClassName(codecClassName);
          if (codec == null) {
            throw new IOException("Image compression codec not supported: "
                + codecClassName);
          }
          in = new DataInputStream(codec.createInputStream(in));
View Full Code Here
74
75
76
77
78
79
80
81
  private enum InputType {CSV, FIXED_WIDTH};
 
  @Override
  protected boolean isSplitable(JobContext context, Path file) {
    CompressionCodec codec = new CompressionCodecFactory(context.getConfiguration()).getCodec(file);
    return codec == null;
  }
View Full Code Here
274
275
276
277
278
279
280
281
282
283
284
     
      start = split.getStart();
      end = start + split.getLength();
     
      final Path file = split.getPath();
      compressionCodecs = new CompressionCodecFactory(conf);
      final CompressionCodec codec = compressionCodecs.getCodec(file);

      // open the file and seek to the start of the split
      FileSystem fs = file.getFileSystem(conf);
      FSDataInputStream fileIn = fs.open(split.getPath());
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.compress.CompressionCodecFactory

  • com.cloudera.sqoop.TestExport
  • com.datasalt.pangool.tuplemr.mapred.lib.input.TupleTextInputFormat
  • com.datasalt.pangool.tuplemr.mapred.lib.input.TupleTextInputFormat$TupleTextInputReader
  • com.facebook.presto.hadoop.HadoopNative
  • com.facebook.presto.hive.AbstractTestHiveFileFormats
  • com.hadoop.compression.lzo.LzoIndex
  • com.hadoop.mapreduce.LzoLineRecordReader
  • com.hadoop.mapreduce.LzoSplitRecordReader
  • com.twitter.elephantbird.mapreduce.input.LzoRecordReader
  • com.twitter.elephantbird.mapreduce.input.MultiInputFormat
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.