Examples of org.apache.hadoop.io.compress.CompressionCodec

org.apache.hadoop.io.compress.CompressionCodec
This class encapsulates a streaming compression/decompression pair. @author Owen O'Malley

   *          Path to be created
   * @return output stream over the created rcfile
   */
  public static RCFile.Writer createRCFileWriter(JobConf jc, FileSystem fs, Path file,
      boolean isCompressed) throws IOException {
    CompressionCodec codec = null;
    Class<?> codecClass = null;
    if (isCompressed) {
      codecClass = FileOutputFormat.getOutputCompressorClass(jc, DefaultCodec.class);
      codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, jc);
    }

View Full Code Here

   * Test using the gzip codec for reading
   */
  @Test(timeout=10000)
  public void testGzip() throws IOException, InterruptedException {
    Configuration conf = new Configuration(defaultConf);
    CompressionCodec gzip = new GzipCodec();
    ReflectionUtils.setConf(gzip, conf);
    localFs.delete(workDir, true);
    writeFile(localFs, new Path(workDir, "part1.txt.gz"), gzip,
              "the quick\nbrown\nfox jumped\nover\n the lazy\n dog\n");
    writeFile(localFs, new Path(workDir, "part2.txt.gz"), gzip,

View Full Code Here

          splits.length);


      CompressionType compressionType = getOutputCompressionType(job);
      Class<? extends CompressionCodec> outputCompressorClass = getOutputCompressorClass(
          job, null);
      CompressionCodec codec = null;
      if (outputCompressorClass != null) {
        codec = ReflectionUtils.newInstance(outputCompressorClass,
            job.getConf());
      }

View Full Code Here

        if (isCompressed) {
          String codecClassName = Text.readString(in);
          v.visit(ImageElement.COMPRESS_CODEC, codecClassName);
          CompressionCodecFactory codecFac = new CompressionCodecFactory(
              new Configuration());
          CompressionCodec codec = codecFac.getCodecByClassName(codecClassName);
          if (codec == null) {
            throw new IOException("Image compression codec not supported: "
                + codecClassName);
          }
          in = new DataInputStream(codec.createInputStream(in));
        }
      }
      processINodes(in, v, numInodes, skipBlocks, supportSnapshot);
      subtreeMap.clear();
      dirNodeMap.clear();

View Full Code Here

          Arrays.equals(outbytes, out.toByteArray()));
      out.reset();


      // Test a simple codec. Extension based detection. We use
      // Bzip2 cause its non-native.
      CompressionCodec codec = ReflectionUtils.newInstance(BZip2Codec.class, conf);
      String extension = codec.getDefaultExtension();
      Path p = new Path(root, "file." + extension);
      OutputStream fout = new DataOutputStream(codec.createOutputStream(
          fs.create(p, true)));
      byte[] writebytes = "foo".getBytes();
      fout.write(writebytes);
      fout.close();
      out = new ByteArrayOutputStream();

View Full Code Here

      Class<?> valueClass = valueClassOption == null ?
          Object.class : valueClassOption.getValue();
      Metadata metadata = metadataOption == null ?
          new Metadata() : metadataOption.getValue();
      this.compress = compressionTypeOption.getValue();
      final CompressionCodec codec = compressionTypeOption.getCodec();
      if (codec != null &&
          (codec instanceof GzipCodec) &&
          !NativeCodeLoader.isNativeCodeLoaded() &&
          !ZlibFactory.isNativeZlibLoaded(conf)) {
        throw new IllegalArgumentException("SequenceFile doesn't work with " +

View Full Code Here

                                      Progressable prog) throws IOException {
      Reader reader = new Reader(conf,
                                 Reader.file(inputFile),
                                 new Reader.OnlyHeaderOption());
      CompressionType compress = reader.getCompressionType();
      CompressionCodec codec = reader.getCompressionCodec();
      reader.close();


      Writer writer = createWriter(conf, 
                                   Writer.file(outputFile), 
                                   Writer.keyClass(keyClass),

View Full Code Here

      public int run(boolean deleteInput) throws IOException {
        int segments = 0;
        int currentFile = 0;
        boolean atEof = (currentFile >= inFiles.length);
        CompressionType compressionType;
        CompressionCodec codec = null;
        segmentLengths.clear();
        if (atEof) {
          return 0;
        }

View Full Code Here

        Integer.MAX_VALUE);
    start = split.getStart();
    end = start + split.getLength();
    final Path file = split.getPath();
    compressionCodecs = new CompressionCodecFactory(job);
    final CompressionCodec codec = compressionCodecs.getCodec(file);


    // open the file and seek to the start of the split
    FileSystem fs = file.getFileSystem(job);
    FSDataInputStream fileIn = fs.open(split.getPath());
    boolean skipFirstLine = false;
    if (codec != null) {
      in = new LineReader(codec.createInputStream(fileIn), job);
      end = Long.MAX_VALUE;
    } else {
      if (start != 0) {
        skipFirstLine = true;
        --start;

View Full Code Here

      return new LineRecordWriter<K, V>(fileOut, keyValueSeparator);
    } else {
      Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(
          job, GzipCodec.class);
      // create the named codec
      CompressionCodec codec = ReflectionUtils.newInstance(codecClass, job
          .getConf());
      // build the filename including the extension
      Path file = FileOutputFormat.getTaskOutputPath(job, name
          + codec.getDefaultExtension());
      FileSystem fs = file.getFileSystem(job.getConf());
      FSDataOutputStream fileOut = fs.create(file);
      return new LineRecordWriter<K, V>(new DataOutputStream(codec
          .createOutputStream(fileOut)), keyValueSeparator);
    }
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.io.compress.CompressionCodec

com.cloudera.flume.handlers.hdfs.CustomDfsSink

com.cloudera.sqoop.TestExport

com.datasalt.pangool.tuplemr.mapred.lib.input.TupleTextInputFormat

com.datasalt.pangool.tuplemr.mapred.lib.input.TupleTextInputFormat$TupleTextInputReader

com.datasalt.pangool.tuplemr.mapred.lib.output.TupleOutputFormat

com.facebook.presto.hive.AbstractTestHiveFileFormats

com.hadoop.compression.lzo.LzoIndex

com.hadoop.compression.lzo.TestLzoCodec

com.hadoop.mapreduce.LzoLineRecordReader

com.hadoop.mapreduce.LzoSplitRecordReader

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.