Package org.apache.hadoop.io.compress

Examples of org.apache.hadoop.io.compress.CompressionCodecFactory

232
233
234
235
236
237
238
239
240
241
242
    FileSystem fs = FileSystem.get(tpch.getTestingCluster().getConfiguration());
    Path path = new Path("/tajo-data/testInsertOverwriteLocationWithCompression");
    assertTrue(fs.exists(path));
    assertEquals(1, fs.listStatus(path).length);

    CompressionCodecFactory factory = new CompressionCodecFactory(tpch.getTestingCluster().getConfiguration());
    for (FileStatus file : fs.listStatus(path)){
      CompressionCodec codec = factory.getCodec(file.getPath());
      assertTrue(codec instanceof DeflateCodec);
    }
  }
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
    // open the file and seek to the start of the split
    final FileSystem fs = file.getFileSystem(job);
    fileIn = fs.open(file);
   
    CompressionCodec codec = new CompressionCodecFactory(job).getCodec(file);
    if (null!=codec) {
      isCompressedInput = true; 
      decompressor = CodecPool.getDecompressor(codec);
      if (codec instanceof SplittableCompressionCodec) {
        final SplitCompressionInputStream cIn =
View Full Code Here
198
199
200
201
202
203
204
205
206
207
208
209
210
        boolean isCompressed = in.readBoolean();
        v.visit(ImageElement.IS_COMPRESSED, String.valueOf(isCompressed));
        if (isCompressed) {
          String codecClassName = Text.readString(in);
          v.visit(ImageElement.COMPRESS_CODEC, codecClassName);
          CompressionCodecFactory codecFac = new CompressionCodecFactory(
              new Configuration());
          CompressionCodec codec = codecFac.getCodecByClassName(codecClassName);
          if (codec == null) {
            throw new IOException("Image compression codec not supported: "
                + codecClassName);
          }
          in = new DataInputStream(codec.createInputStream(in));
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
    this.maxLineLength = job.getInt(org.apache.hadoop.mapreduce.lib.input.
      LineRecordReader.MAX_LINE_LENGTH, Integer.MAX_VALUE);
    start = split.getStart();
    end = start + split.getLength();
    final Path file = split.getPath();
    compressionCodecs = new CompressionCodecFactory(job);
    codec = compressionCodecs.getCodec(file);

    // open the file and seek to the start of the split
    final FileSystem fs = file.getFileSystem(job);
    fileIn = fs.open(file);
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
    this.maxLineLength = job.getInt("bsp.linerecordreader.maxlength",
        Integer.MAX_VALUE);
    start = split.getStart();
    end = start + split.getLength();
    final Path file = split.getPath();
    compressionCodecs = new CompressionCodecFactory(job);
    final CompressionCodec codec = compressionCodecs.getCodec(file);

    // open the file and seek to the start of the split
    FileSystem fs = file.getFileSystem(job);
    FSDataInputStream fileIn = fs.open(split.getPath());
View Full Code Here
29
30
31
32
33
34
35
36
37
*/
public class CompressionUtil {

    public static CompressionCodec createCompressionCodec(String className)
            throws Exception {
        CompressionCodecFactory ccf = new CompressionCodecFactory(
                new Configuration());
        return ccf.getCodecByClassName(className);
    }
View Full Code Here
38
39
40
41
42
43
44
        return reader;
    }

    @Override
    protected boolean isSplitable(final JobContext context, final Path file) {
        return null == new CompressionCodecFactory(context.getConfiguration()).getCodec(file);
    }
View Full Code Here
34
35
36
37
38
39
40
41
42
43
44
    public void initialize(final InputSplit genericSplit, final TaskAttemptContext context) throws IOException {
        final FileSplit split = (FileSplit) genericSplit;
        final Configuration job = context.getConfiguration();
        long start = split.getStart();
        final Path file = split.getPath();
        if (null != new CompressionCodecFactory(job).getCodec(file)) {
            throw new IllegalStateException("Compression is not supported for the (binary) Gremlin Kryo format");
        }
        // open the file and seek to the start of the split
        this.inputStream = file.getFileSystem(job).open(split.getPath());
        this.inputStream.seek(start);
View Full Code Here
28
29
30
31
32
33
34
        return reader;
    }

    @Override
    protected boolean isSplitable(final JobContext context, final Path file) {
        return null == new CompressionCodecFactory(context.getConfiguration()).getCodec(file);
    }
View Full Code Here
31
32
33
34
35
36
37
38
39
40
41
        final Path file = split.getPath();
        start = split.getStart();
        end = start + split.getLength();

        compressionCodecs = new CompressionCodecFactory(job);
        final CompressionCodec codec = compressionCodecs.getCodec(file);

        FileSystem fs = file.getFileSystem(job);
        FSDataInputStream fileIn = fs.open(split.getPath());
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.compress.CompressionCodecFactory

  • com.cloudera.sqoop.TestExport
  • com.datasalt.pangool.tuplemr.mapred.lib.input.TupleTextInputFormat
  • com.datasalt.pangool.tuplemr.mapred.lib.input.TupleTextInputFormat$TupleTextInputReader
  • com.facebook.presto.hadoop.HadoopNative
  • com.facebook.presto.hive.AbstractTestHiveFileFormats
  • com.hadoop.compression.lzo.LzoIndex
  • com.hadoop.mapreduce.LzoLineRecordReader
  • com.hadoop.mapreduce.LzoSplitRecordReader
  • com.twitter.elephantbird.mapreduce.input.LzoRecordReader
  • com.twitter.elephantbird.mapreduce.input.MultiInputFormat
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.