Package org.apache.hadoop.io.compress

Examples of org.apache.hadoop.io.compress.CompressionCodec

47
48
49
50
51
52
53
54
55
56
57
    if (!fs.exists(outputPath)) {
      throw new IOException("Output directory doesnt exist");
    }
    Path file = new Path(outputPath, name);
   
    CompressionCodec codec = null;
    CompressionType compressionType = CompressionType.NONE;
    if (getCompressOutput(job)) {
      // find the kind of compression to do
      compressionType = SequenceFileOutputFormat.getOutputCompressionType(job);
View Full Code Here
117
118
119
120
121
122
123
124
125
126
127
128
129
130
      return new LineRecordWriter<K, V>(fileOut);
    } else {
      Class<? extends CompressionCodec> codecClass =
        getOutputCompressorClass(job, GzipCodec.class);
      // create the named codec
      CompressionCodec codec = (CompressionCodec)
        ReflectionUtils.newInstance(codecClass, job);
      // build the filename including the extension
      Path filename = new Path(dir, name + codec.getDefaultExtension());
      FSDataOutputStream fileOut = fs.create(filename, progress);
      return new LineRecordWriter<K, V>(new DataOutputStream
                                        (codec.createOutputStream(fileOut)));
    }
  }     
View Full Code Here
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
                          FileSplit split) throws IOException {
    start = split.getStart();
    end = start + split.getLength();
    final Path file = split.getPath();
    compressionCodecs = new CompressionCodecFactory(job);
    final CompressionCodec codec = compressionCodecs.getCodec(file);

    // open the file and seek to the start of the split
    FileSystem fs = file.getFileSystem(job);
    FSDataInputStream fileIn = fs.open(split.getPath());
    boolean skipFirstLine = false;
    if (codec != null) {
      in = new LineReader(codec.createInputStream(fileIn), job);
      end = Long.MAX_VALUE;
    } else {
      if (start != 0) {
        skipFirstLine = true;
        --start;
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
    FileSystem fs = outputPath.getFileSystem(job);
    if (!fs.exists(outputPath)) {
      throw new IOException("Output directory doesnt exist");
    }
    Path file = new Path(outputPath, name);
    CompressionCodec codec = null;
    CompressionType compressionType = CompressionType.NONE;
    if (getCompressOutput(job)) {
      // find the kind of compression to do
      compressionType = getOutputCompressionType(job);
View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
  }
 
  public void testLzoSequenceFile() throws Exception {
    if (LzoCodec.isNativeLzoLoaded(conf)) {
      LOG.info("Testing SequenceFile with LzoCodec");
      CompressionCodec lzoCodec = null;
      try {
        lzoCodec = (CompressionCodec) ReflectionUtils.newInstance(
                                                                  conf.getClassByName(LzoCodec.class.getName()), conf);
      } catch (ClassNotFoundException cnfe) {
        throw new IOException("Cannot find LzoCodec!");
View Full Code Here
323
324
325
326
327
328
329
330
331
332
333
  public void testSequenceFileMetadata() throws Exception {
    LOG.info("Testing SequenceFile with metadata");
    int count = 1024 * 10;
    int megabytes = 1;
    int factor = 5;
    CompressionCodec codec = new DefaultCodec();
    Path file = new Path(System.getProperty("test.build.data",".")+"/test.seq.metadata");
    Path recordCompressedFile =
      new Path(System.getProperty("test.build.data",".")+"/test.rc.seq.metadata");
    Path blockCompressedFile =
      new Path(System.getProperty("test.build.data",".")+"/test.bc.seq.metadata");
View Full Code Here
483
484
485
486
487
488
489
490
491
492
493
        System.exit(-1);
      }

      CompressionType compressionType =
        CompressionType.valueOf(compressType);
      CompressionCodec codec = (CompressionCodec)ReflectionUtils.newInstance(
                                                                             conf.getClassByName(compressionCodec),
                                                                             conf);

      if (rwonly || (create && !merge)) {
        writeTest(fs, count, seed, file, compressionType, codec);
View Full Code Here
143
144
145
146
147
148
149
150
151
152
153
154
    FileSystem fs = FileSystem.get(conf);
    fs.mkdirs(tablePath);
    OutputStream os = fs.create(filePath);
    if (gzip) {
      CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
      CompressionCodec codec = ccf.getCodec(filePath);
      os = codec.createOutputStream(os);
    }
    BufferedWriter w = new BufferedWriter(new OutputStreamWriter(os));
    for (int i = 0; i < numRecords; i++) {
      w.write(getRecordLine(startId + i, extraCols));
    }
View Full Code Here
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
      conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
    }
    FileSystem fs = FileSystem.get(conf);
    InputStream is = fs.open(f);
    CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
    CompressionCodec codec = ccf.getCodec(f);
    LOG.info("gzip check codec is " + codec);
    Decompressor decompressor = CodecPool.getDecompressor(codec);
    if (null == decompressor) {
      LOG.info("Verifying gzip sanity with null decompressor");
    } else {
      LOG.info("Verifying gzip sanity with decompressor: "
          + decompressor.toString());
    }
    is = codec.createInputStream(is, decompressor);
    BufferedReader r = new BufferedReader(new InputStreamReader(is));
    int numLines = 0;
    while (true) {
      String ln = r.readLine();
      if (ln == null) {
View Full Code Here
150
151
152
153
154
155
156
157
158
159
160
    pools.add(multi);
  }

  @Override
  protected boolean isSplitable(JobContext context, Path file) {
    final CompressionCodec codec =
      new CompressionCodecFactory(context.getConfiguration()).getCodec(file);
    if (null == codec) {
      return true;
    }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.compress.CompressionCodec

  • com.cloudera.flume.handlers.hdfs.CustomDfsSink
  • com.cloudera.sqoop.TestExport
  • com.datasalt.pangool.tuplemr.mapred.lib.input.TupleTextInputFormat
  • com.datasalt.pangool.tuplemr.mapred.lib.input.TupleTextInputFormat$TupleTextInputReader
  • com.datasalt.pangool.tuplemr.mapred.lib.output.TupleOutputFormat
  • com.facebook.presto.hive.AbstractTestHiveFileFormats
  • com.hadoop.compression.lzo.LzoIndex
  • com.hadoop.compression.lzo.TestLzoCodec
  • com.hadoop.mapreduce.LzoLineRecordReader
  • com.hadoop.mapreduce.LzoSplitRecordReader
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.