Examples of org.apache.hadoop.io.compress.Decompressor

org.apache.hadoop.io.compress.Decompressor
Specification of a stream-based 'de-compressor' which can be plugged into a {@link CompressionInputStream} to compress data.This is modelled after {@link java.util.zip.Inflater} @author Arun C Murthy

     * @throws IOException
     */
    private ByteBuffer decompress(final long offset, final int compressedSize,
      final int decompressedSize) 
    throws IOException {
      Decompressor decompressor = null;
      ByteBuffer buf = null;
      try {
        decompressor = this.compressAlgo.getDecompressor();
        // My guess is that the bounded range fis is needed to stop the 
        // decompressor reading into next block -- IIRC, it just grabs a

View Full Code Here

      }
    } else {
      Class<?> clazz = ClassUtils.resolveClassName(getCodec().getCodecClass(), getClass().getClassLoader());
      CompressionCodec compressionCodec = (CompressionCodec) ReflectionUtils.newInstance(clazz,
          getConfiguration());
      Decompressor decompressor = CodecPool.getDecompressor(compressionCodec);
      if (getSplit() == null) {
        // with codec, no split
        FSDataInputStream winput = fs.open(p);
        InputStream input = compressionCodec.createInputStream(winput, decompressor);
        holder.setWrappedStream(winput);

View Full Code Here

    // resources if the stream is not closed properly after we let it out.
    InputStream is = null;
    if (compressor != null) {
      // GZIPCodec fails w/ NPE if no configuration.
      if (compressor instanceof Configurable) ((Configurable)compressor).setConf(this.conf);
      Decompressor poolDecompressor = CodecPool.getDecompressor(compressor);
      CompressionInputStream cis =
        compressor.createInputStream(new ByteArrayInputStream(cellBlock, offset, length),
        poolDecompressor);
      try {
        // TODO: This is ugly.  The buffer will be resized on us if we guess wrong.

View Full Code Here

      this.end = in.getPos() + length;
      init(tempReader);
    }
    
    private Decompressor getPooledOrNewDecompressor() {
      Decompressor decompressor = null;
      decompressor = decompressorPool.getCodec(codec.getDecompressorType());
      if (decompressor == null) {
        decompressor = codec.createDecompressor();
      }
      return decompressor;

View Full Code Here

     * @throws IOException
     */
    private ByteBuffer decompress(final long offset, final int compressedSize,
      final int decompressedSize) 
    throws IOException {
      Decompressor decompressor = null;
      ByteBuffer buf = null;
      try {
        decompressor = this.compressAlgo.getDecompressor();
        // My guess is that the bounded range fis is needed to stop the 
        // decompressor reading into next block -- IIRC, it just grabs a

View Full Code Here

    FileSystem fs = FileSystem.get(conf);
    InputStream is = fs.open(f);
    CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
    CompressionCodec codec = ccf.getCodec(f);
    LOG.info("gzip check codec is " + codec);
    Decompressor decompressor = CodecPool.getDecompressor(codec);
    if (null == decompressor) {
      LOG.info("Verifying gzip sanity with null decompressor");
    } else {
      LOG.info("Verifying gzip sanity with decompressor: "
          + decompressor.toString());
    }
    is = codec.createInputStream(is, decompressor);
    BufferedReader r = new BufferedReader(new InputStreamReader(is));
    int numLines = 0;
    while (true) {

View Full Code Here

   *          <code>Decompressor</code>
   * @return <code>Decompressor</code> for the given
   *         <code>CompressionCodec</code> the pool or a new one
   */
  public static Decompressor getDecompressor(CompressionCodec codec) {
    Decompressor decompressor = borrow(DECOMPRESSOR_POOL, codec
        .getDecompressorType());
    if (decompressor == null) {
      decompressor = codec.createDecompressor();
      LOG.info("Got brand-new decompressor ["+codec.getDefaultExtension()+"]");
    } else {

View Full Code Here

     * @throws IOException
     */
    protected void decompress(byte[] dest, int destOffset,
        InputStream bufferedBoundedStream,
        int uncompressedSize) throws IOException {
      Decompressor decompressor = null;
      try {
        decompressor = compressAlgo.getDecompressor();
        InputStream is = compressAlgo.createDecompressionStream(
            bufferedBoundedStream, decompressor, 0);

View Full Code Here

     * @throws IOException
     */
    private ByteBuffer decompress(final long offset, final int compressedSize,
      final int decompressedSize, final boolean pread)
    throws IOException {
      Decompressor decompressor = null;
      ByteBuffer buf = null;
      try {
        decompressor = this.compressAlgo.getDecompressor();
        // My guess is that the bounded range fis is needed to stop the
        // decompressor reading into next block -- IIRC, it just grabs a

View Full Code Here

     * @throws IOException
     */
    private ByteBuffer decompress(final long offset, final int compressedSize,
      final int decompressedSize) 
    throws IOException {
      Decompressor decompressor = null;
      ByteBuffer buf = null;
      try {
        decompressor = this.compressAlgo.getDecompressor();
        // My guess is that the bounded range fis is needed to stop the 
        // decompressor reading into next block -- IIRC, it just grabs a

View Full Code Here

0 1 2 3 4 5

TOP

Related Classes of org.apache.hadoop.io.compress.Decompressor

cascading.tuple.hadoop.collect.HadoopSpillableTupleList

com.cloudera.sqoop.TestExport

hivemall.utils.hadoop.HadoopUtils

org.apache.hadoop.hbase.io.compress.Compression

org.apache.hadoop.hbase.io.hfile.HFile$Reader

org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader

org.apache.hadoop.hbase.ipc.IPCUtil

org.apache.hadoop.hive.ql.io.CodecPool

org.apache.hadoop.io.compress.zlib.TestZlibCompressorDecompressor

org.apache.hadoop.io.SequenceFile$Reader

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.