Examples of org.apache.hadoop.io.compress.CompressionCodec

org.apache.hadoop.io.compress.CompressionCodec
This class encapsulates a streaming compression/decompression pair. @author Owen O'Malley

    FileSystem fs = file.getFileSystem(config);




    //WT2G collection has incorrectly named extensions. Terrier can deal with this,
    //Hadoop cant
    CompressionCodec codec = compressionCodecs.getCodec(
      new Path(file.toString().replaceAll("\\.GZ$", ".gz")));
    
    length = fs.getFileStatus(file).getLen();
    FSDataInputStream _input = fs.open(file); //TODO: we could use utility.Files here if
    //no codec was found  
    InputStream internalInputStream = null;
    start = offset;
    
    if (codec !=null)
    {
      start = 0;
      inputStream = new CountingInputStream(_input);
      internalInputStream = codec.createInputStream(inputStream);
    } 
    else 
    {
      if (start != 0) //TODO: start is always zero? 
      {

View Full Code Here

    }.globAndProcess(srcPattern, srcPattern.getFileSystem(getConf()));
  }


  private InputStream decompress(Path p, FileSystem srcFs) throws IOException {
    CompressionCodecFactory factory = new CompressionCodecFactory(getConf());
    CompressionCodec codec = factory.getCodec(p);
    InputStream in = srcFs.open(p);
    if (codec == null) {
      throw new IOException("Cannot find codec for " + p);
    }
    return codec.createInputStream(in);
  }

View Full Code Here

    Path dstPath = new Path(dstf);
    FileSystem dstFs = dstPath.getFileSystem(getConf());


    // Create codec
    CompressionCodecFactory factory = new CompressionCodecFactory(conf);
    CompressionCodec codec = factory.getCodec(dstPath);
    if (codec == null) {
      System.err.println(cmd.substring(1) + ": cannot find compression codec for "
          + dstf);
      return 1;
    }


    // open input stream
    InputStream in = srcFs.open(srcPath);


    // Create compression stream
    OutputStream out = dstFs.create(dstPath);
    out = codec.createOutputStream(out);


    IOUtils.copyBytes(in, out, conf, true);


    return 0;
  }

View Full Code Here

    throws IOException {
      FileSystem srcFileSys = inputFile.getFileSystem(conf);
      Reader reader = new Reader(srcFileSys, inputFile, 4096, conf, true);
      boolean compress = reader.isCompressed();
      boolean blockCompress = reader.isBlockCompressed();
      CompressionCodec codec = reader.getCompressionCodec();
      reader.close();


      Writer writer = createWriter(outputFile.getFileSystem(conf), conf, 
                                   outputFile, keyClass, valClass, compress, 
                                   blockCompress, codec, prog,

View Full Code Here

        int segments = 0;
        int currentFile = 0;
        boolean atEof = (currentFile >= inFiles.length);
        boolean isCompressed = false;
        boolean isBlockCompressed = false;
        CompressionCodec codec = null;
        segmentLengths.clear();
        if (atEof) {
          return 0;
        }

View Full Code Here

  private static FSImageCompression createCompression(Configuration conf,
                                                      String codecClassName)
    throws IOException {


    CompressionCodecFactory factory = new CompressionCodecFactory(conf);
    CompressionCodec codec = factory.getCodecByClassName(codecClassName);
    if (codec == null) {
      throw new IOException("Not a supported codec: " + codecClassName);
    }


    return new FSImageCompression(codec);

View Full Code Here

        if (isCompressed) {
          String codecClassName = Text.readString(in);
          v.visit(ImageElement.COMPRESS_CODEC, codecClassName);
          CompressionCodecFactory codecFac = new CompressionCodecFactory(
              new Configuration());
          CompressionCodec codec = codecFac.getCodecByClassName(codecClassName);
          if (codec == null) {
            throw new IOException("Image compression codec not supported: "
                + codecClassName);
          }
          in = new DataInputStream(codec.createInputStream(in));
        }
      }
      processINodes(in, v, numInodes, skipBlocks);


      processINodesUC(in, v, skipBlocks);

View Full Code Here

    public Writer cloneFileAttributes(Path inputFile, Path outputFile, Progressable prog) throws IOException {
      FileSystem srcFileSys = inputFile.getFileSystem(conf);
      Reader reader = new Reader(srcFileSys, inputFile, 4096, conf, true);
      boolean compress = reader.isCompressed();
      boolean blockCompress = reader.isBlockCompressed();
      CompressionCodec codec = reader.getCompressionCodec();
      reader.close();
      
      Writer writer = createWriter(outputFile.getFileSystem(conf), conf, outputFile, keyClass, valClass, compress, blockCompress, codec, prog, new Metadata());
      return writer;
    }

View Full Code Here

        int segments = 0;
        int currentFile = 0;
        boolean atEof = (currentFile >= inFiles.length);
        boolean isCompressed = false;
        boolean isBlockCompressed = false;
        CompressionCodec codec = null;
        segmentLengths.clear();
        if (atEof) {
          return 0;
        }

View Full Code Here

    public RecordWriter<WritableComparable<?>,Writable> getRecordWriter(final TaskAttemptContext job) throws IOException, InterruptedException {
      // get the path of the temporary output file
      Path file = getDefaultWorkFile(job, "");


      FileSystem fs = file.getFileSystem(InputFormatBase.getConfiguration(job));
      CompressionCodec codec = null;
      CompressionType compressionType = CompressionType.NONE;
      if (getCompressOutput(job)) {
        // find the kind of compression to do
        compressionType = SequenceFileOutputFormat.getOutputCompressionType(job);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.io.compress.CompressionCodec

com.cloudera.flume.handlers.hdfs.CustomDfsSink

com.cloudera.sqoop.TestExport

com.datasalt.pangool.tuplemr.mapred.lib.input.TupleTextInputFormat

com.datasalt.pangool.tuplemr.mapred.lib.input.TupleTextInputFormat$TupleTextInputReader

com.datasalt.pangool.tuplemr.mapred.lib.output.TupleOutputFormat

com.facebook.presto.hive.AbstractTestHiveFileFormats

com.hadoop.compression.lzo.LzoIndex

com.hadoop.compression.lzo.TestLzoCodec

com.hadoop.mapreduce.LzoLineRecordReader

com.hadoop.mapreduce.LzoSplitRecordReader

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.