Package org.apache.hadoop.io.compress

Examples of org.apache.hadoop.io.compress.CompressionCodecFactory

383
384
385
386
387
388
389
390
391
392
393
    Compressor compressor;

    FileSystem fs = resultPath.getFileSystem(conf);
    CompressionCodec codec =
        new CompressionCodecFactory(conf).getCodec(resultPath);
    OutputStream output;
    if (codec != null) {
      compressor = CodecPool.getCompressor(codec);
      output = codec.createOutputStream(fs.create(resultPath), compressor);
    } else {
View Full Code Here
162
163
164
165
166
167
168
169
170
171
172
173
174
        boolean isCompressed = in.readBoolean();
        v.visit(ImageElement.IS_COMPRESSED, String.valueOf(isCompressed));
        if (isCompressed) {
          String codecClassName = Text.readString(in);
          v.visit(ImageElement.COMPRESS_CODEC, codecClassName);
          CompressionCodecFactory codecFac = new CompressionCodecFactory(
              new Configuration());
          CompressionCodec codec = codecFac.getCodecByClassName(codecClassName);
          if (codec == null) {
            throw new IOException("Image compression codec not supported: "
                + codecClassName);
          }
          in = new DataInputStream(codec.createInputStream(in));
View Full Code Here
145
146
147
148
149
150
151
152
153
154
155
  }
 
  @Override
  protected boolean isSplitable(JobContext context, Path file) {
    final CompressionCodec codec =
      new CompressionCodecFactory(context.getConfiguration()).getCodec(file);
    if (null == codec) {
      return true;
    }
    return codec instanceof SplittableCompressionCodec;
  }
View Full Code Here
65
66
67
68
69
70
71
    // local FS must be raw in order to be Syncable
    conf.set("fs.file.impl", "org.apache.hadoop.fs.RawLocalFileSystem");
    Path path = new Path(fileURI);
    path.getFileSystem(conf); // get FS with our conf cached

    this.factory = new CompressionCodecFactory(conf);
  }
View Full Code Here
110
111
112
113
114
115
116
117
118
119
120
      throw new FileNotFoundException(path.toString());
    }

    String codecName = this.meta.getOption(StorageConstants.COMPRESSION_CODEC);
    if(!StringUtils.isEmpty(codecName)){
      codecFactory = new CompressionCodecFactory(conf);
      codec = codecFactory.getCodecByClassName(codecName);
    } else {
      if (fs.exists(path)) {
        throw new AlreadyExistsStorageException(path);
      }
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
    Configuration conf = new Configuration();
    // local FS must be raw in order to be Syncable
    conf.set("fs.file.impl", "org.apache.hadoop.fs.RawLocalFileSystem");
    Path path = new Path(fileURI);
    FileSystem fs = path.getFileSystem(conf); // get FS with our conf cached
    CompressionCodecFactory factory = new CompressionCodecFactory(conf);

    HDFSCompressedDataStream writer = new HDFSCompressedDataStream();
    FlumeFormatter fmt = new HDFSTextFormatter();
    writer.open(fileURI, factory.getCodec(new Path(fileURI)),
        SequenceFile.CompressionType.BLOCK, fmt);
    String body = "yarf!";
    Event evt = EventBuilder.withBody(body, Charsets.UTF_8);
    writer.append(evt, fmt);
    writer.sync();
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
      this.maxLineLength = job.getInt("bsp.linerecordreader.maxlength",
          Integer.MAX_VALUE);
      start = split.getStart();
      end = start + split.getLength();
      final Path file = split.getPath();
      compressionCodecs = new CompressionCodecFactory(job);
      final CompressionCodec codec = compressionCodecs.getCodec(file);

      // open the file and seek to the start of the split
      FileSystem fs = file.getFileSystem(job);
      FSDataInputStream fileIn = fs.open(split.getPath());
View Full Code Here
237
238
239
240
241
242
243
244
245
246
247
    Compressor compressor;

    FileSystem fs = resultPath.getFileSystem(conf);
    CompressionCodec codec =
        new CompressionCodecFactory(conf).getCodec(resultPath);
    OutputStream output;
    if (codec != null) {
      compressor = CodecPool.getCompressor(codec);
      output = codec.createOutputStream(fs.create(resultPath), compressor);
    } else {
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
    Configuration job = context.getConfiguration();
    this.maxLineLength = job.getInt(MAX_LINE_LENGTH, Integer.MAX_VALUE);
    start = split.getStart();
    end = start + split.getLength();
    final Path file = split.getPath();
    compressionCodecs = new CompressionCodecFactory(job);
    codec = compressionCodecs.getCodec(file);

    // open the file and seek to the start of the split
    final FileSystem fs = file.getFileSystem(job);
    fileIn = fs.open(file);
View Full Code Here
42
43
44
45
46
47
48
  implements JobConfigurable {

  private CompressionCodecFactory compressionCodecs = null;
 
  public void configure(JobConf conf) {
    compressionCodecs = new CompressionCodecFactory(conf);
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.compress.CompressionCodecFactory

  • com.cloudera.sqoop.TestExport
  • com.datasalt.pangool.tuplemr.mapred.lib.input.TupleTextInputFormat
  • com.datasalt.pangool.tuplemr.mapred.lib.input.TupleTextInputFormat$TupleTextInputReader
  • com.facebook.presto.hadoop.HadoopNative
  • com.facebook.presto.hive.AbstractTestHiveFileFormats
  • com.hadoop.compression.lzo.LzoIndex
  • com.hadoop.mapreduce.LzoLineRecordReader
  • com.hadoop.mapreduce.LzoSplitRecordReader
  • com.twitter.elephantbird.mapreduce.input.LzoRecordReader
  • com.twitter.elephantbird.mapreduce.input.MultiInputFormat
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.