Examples of CompressionInputStream

com.netflix.aegisthus.io.sstable.compression.CompressionInputStream
This class implements an input stream for reading Snappy compressed data of the format produced by {@link SnappyOutputStream}.
org.apache.hadoop.io.compress.CompressionInputStream
A compression input stream.
Implementations are assumed to be buffered. This permits clients to reposition the underlying input stream then call {@link #resetState()}, without having to also synchronize client buffers. @author Arun C Murthy

Examples of org.apache.hadoop.io.compress.CompressionInputStream

    InputStream is = null;
    if (compressor != null) {
      // GZIPCodec fails w/ NPE if no configuration.
      if (compressor instanceof Configurable) ((Configurable)compressor).setConf(this.conf);
      Decompressor poolDecompressor = CodecPool.getDecompressor(compressor);
      CompressionInputStream cis =
        compressor.createInputStream(new ByteArrayInputStream(cellBlock, offset, length),
        poolDecompressor);
      ByteBufferOutputStream bbos = null;
      try {
        // TODO: This is ugly.  The buffer will be resized on us if we guess wrong.

View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionInputStream


   DefaultCodec codec = new DefaultCodec();
   codec.setConf(job);
   Path expectedFile = new Path(workDir, file + codec.getDefaultExtension());
   final FileInputStream istream = new FileInputStream(expectedFile.toString());
   CompressionInputStream cistream = codec.createInputStream(istream);
   LineReader reader = new LineReader(cistream);


   String output = "";
   Text out = new Text();
   while (reader.readLine(out) > 0) {

View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionInputStream

            {
              CompressionCodecFactory compressionCodecs =  new CompressionCodecFactory(job);
              final CompressionCodec codec = compressionCodecs.getCodec(file);
               if (codec != null) {
                  end = Long.MAX_VALUE;
                    CompressionInputStream stream = codec.createInputStream(fileIn);
                    this.xmlLoaderBPIS = new XMLLoaderBufferedPositionedInputStream(stream,start,end);
                  }
            }
            
            else

View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionInputStream

            else if (type.equals("gz")) {
               codec = new GzipCodec();
               ((GzipCodec)codec).setConf(new Configuration());
            }


            CompressionInputStream createInputStream = codec
                  .createInputStream(new FileInputStream(file));
            int b;
            StringBuffer sb = new StringBuffer();
            while ((b = createInputStream.read()) != -1) {
               sb.append((char) b);
            }
            createInputStream.close();


            // Assert for the number of fields and keys.
            String[] fields = sb.toString().split("\\t");
            assertEquals(3, fields.length);
            String id = indexFolder.substring(1,2);

View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionInputStream

   * @return The result after decompressing BSPMessageBundle.
   */
  @Override
  public byte[] decompress(byte[] compressedBytes) {
    ByteArrayInputStream bis = null;
    CompressionInputStream sis = null;
    DataInputStream dis = null;
    byte[] bytes = null;


    try {
      bis = new ByteArrayInputStream(compressedBytes);
      sis = codec.createInputStream(bis);
      dis = new DataInputStream(sis);
      bytes = IOUtils.toByteArray(dis);
    } catch (IOException ioe) {
      LOG.error("Unable to decompress.", ioe);
    } finally {
      try {
        dis.close();
        sis.close();
        bis.close();
      } catch (IOException e) {
        LOG.warn("Failed to close decompression streams.", e);
      }
    }

View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionInputStream

        new CompressionCodecFactory(conf);
      CompressionCodec codec = compressionCodecs.getCodec(file);
      if (codec != null) {
        Decompressor decompressor = CodecPool.getDecompressor(codec);
        if (decompressor != null) {
          CompressionInputStream in = 
            codec.createInputStream(fs.open(file), decompressor);
          //TODO Seek doesnt work with compressed input stream. 
          //     Use SplittableCompressionCodec?
          return (InputStream)in;
        }
      }
    }
    FSDataInputStream in = fs.open(file);
    in.seek(offset);
    return (InputStream)in;
  }

View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionInputStream

        new CompressionCodecFactory(conf);
      CompressionCodec codec = compressionCodecs.getCodec(file);
      if (codec != null) {
        Decompressor decompressor = CodecPool.getDecompressor(codec);
        if (decompressor != null) {
          CompressionInputStream in = 
            codec.createInputStream(fs.open(file), decompressor);
          //TODO Seek doesnt work with compressed input stream. 
          //     Use SplittableCompressionCodec?
          return (InputStream)in;
        }
      }
    }
    FSDataInputStream in = fs.open(file);
    in.seek(offset);
    return (InputStream)in;
  }

View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionInputStream


      DataInputBuffer deCompressedDataBuffer = new DataInputBuffer();
      deCompressedDataBuffer.reset(compressedDataBuffer.getData(), 0,
          compressedDataBuffer.getLength());


      CompressionInputStream inflateFilter = new BlockDecompressorStream(
          deCompressedDataBuffer, new SnappyDecompressor(bufferSize),
          bufferSize);


      inflateIn = new DataInputStream(new BufferedInputStream(inflateFilter));

View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionInputStream


      DataInputBuffer deCompressedDataBuffer = new DataInputBuffer();
      deCompressedDataBuffer.reset(compressedDataBuffer.getData(), 0,
          compressedDataBuffer.getLength());


      CompressionInputStream inflateFilter = new BlockDecompressorStream(
          deCompressedDataBuffer, new Lz4Decompressor(bufferSize), bufferSize);


      inflateIn = new DataInputStream(new BufferedInputStream(inflateFilter));


      byte[] result = new byte[BYTE_SIZE];

View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionInputStream

      compressedKeyLen = in.readInt();
      if (decompress) {
        keyTempBuffer.reset();
        keyTempBuffer.write(in, compressedKeyLen);
        keyDecompressBuffer.reset(keyTempBuffer.getData(), compressedKeyLen);
        CompressionInputStream deflatFilter = codec.createInputStream(
            keyDecompressBuffer, keyDecompressor);
        DataInputStream compressedIn = new DataInputStream(deflatFilter);
        deflatFilter.resetState();
        keyDecompressedData.reset();
        keyDecompressedData.write(compressedIn, currentKeyLength);
        keyDataIn.reset(keyDecompressedData.getData(), currentKeyLength);
        currentKey.readFields(keyDataIn);
      } else {

View Full Code Here

0 1 2 3 4

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.