Examples of org.apache.hadoop.io.compress.CompressionOutputStream

org.apache.hadoop.io.compress.CompressionOutputStream
A compression output stream. @author Arun C Murthy

      out.write(b, off, len);
    }


    @Override
    public void flush() throws IOException {
      CompressionOutputStream cout = (CompressionOutputStream) out;
      cout.finish();
      cout.flush();
      cout.resetState();
    }

View Full Code Here

            Configuration conf = new Configuration();
            CompressionCodec codec = (CompressionCodec)
              ReflectionUtils.newInstance(conf.getClassByName(codecClassName), conf);


            DataOutputBuffer compressedDataBuffer = new DataOutputBuffer();
            CompressionOutputStream deflateFilter = codec.createOutputStream(compressedDataBuffer);


            byte[] data = new byte[1024];
            DataOutputStream deflateOut = new DataOutputStream(new BufferedOutputStream(deflateFilter));
            deflateOut.write(data, 0, data.length);
            deflateOut.flush();
            deflateFilter.finish();


            // Codec class, codec nativelib and Hadoop nativelib with codec JNIs are present
            assertTrue(CompressionTest.testCompression(codecName));
        } catch (UnsatisfiedLinkError e) {
          // Hadoop nativelib does not have codec JNIs.

View Full Code Here

    System.out.println(name + ":");


    // compress it
    List<Long> compressDurations = new ArrayList<Long>();
    ByteArrayOutputStream compressedStream = new ByteArrayOutputStream();
    CompressionOutputStream compressingStream =
        algorithm.createPlainCompressionStream(compressedStream, compressor);
    try {
      for (int itTime = 0; itTime < benchmarkNTimes; ++itTime) {
        final long startTime = System.nanoTime();
        compressingStream.resetState();
        compressedStream.reset();
        compressingStream.write(buffer, offset, length);
        compressingStream.flush();
        compressedStream.toByteArray();


        final long finishTime = System.nanoTime();


        // add time record
        if (itTime >= benchmarkNOmit) {
          compressDurations.add(finishTime - startTime);
        }
      }
    } catch (IOException e) {
      throw new RuntimeException(String.format(
          "Benchmark, or encoding algorithm '%s' cause some stream problems",
          name), e);
    }
    compressingStream.close();
    printBenchmarkResult(length, compressDurations, Manipulation.COMPRESSION);


    byte[] compBuffer = compressedStream.toByteArray();


    // uncompress it several times and measure performance

View Full Code Here


      key.numberRows = bufferedRecords;


      Compressor compressor = null;
      NonSyncDataOutputBuffer valueBuffer = null;
      CompressionOutputStream deflateFilter = null;
      DataOutputStream deflateOut = null;
      boolean isCompressed = isCompressed();
      int valueLength = 0;
      if (isCompressed) {
        ReflectionUtils.setConf(codec, this.conf);
        compressor = CodecPool.getCompressor(codec);
        valueBuffer = new NonSyncDataOutputBuffer();
        deflateFilter = codec.createOutputStream(valueBuffer, compressor);
        deflateOut = new DataOutputStream(deflateFilter);
      }


      for (int columnIndex = 0; columnIndex < columnNumber; columnIndex++) {
        ColumnBuffer currentBuf = columnBuffers[columnIndex];
        currentBuf.flushGroup();


        NonSyncDataOutputBuffer columnValue = currentBuf.columnValBuffer;
        int colLen;
        int plainLen = columnValuePlainLength[columnIndex];


        if (isCompressed) {
          if (deflateFilter instanceof SchemaAwareCompressionOutputStream) {
            ((SchemaAwareCompressionOutputStream)deflateFilter).
              setColumnIndex(columnIndex);
          }
          deflateFilter.resetState();
          deflateOut.write(columnValue.getData(), 0, columnValue.getLength());
          deflateOut.flush();
          deflateFilter.finish();
          // find how much compressed data was added for this column
          colLen = valueBuffer.getLength() - valueLength;
        } else {
          colLen = columnValuePlainLength[columnIndex];
        }

View Full Code Here


      if(this.isCompressed()) {
        Compressor compressor = CodecPool.getCompressor(codec);
        NonSyncDataOutputBuffer compressionBuffer =
          new NonSyncDataOutputBuffer();
        CompressionOutputStream deflateFilter =
          codec.createOutputStream(compressionBuffer, compressor);
        DataOutputStream deflateOut = new DataOutputStream(deflateFilter);
        //compress key and write key out
        compressionBuffer.reset();
        deflateFilter.resetState();
        keyBuffer.write(deflateOut);
        deflateOut.flush();
        deflateFilter.finish();
        int compressedKeyLen = compressionBuffer.getLength();
        out.writeInt(compressedKeyLen);
        out.write(compressionBuffer.getData(), 0, compressedKeyLen);
        CodecPool.returnCompressor(compressor);
      } else {

View Full Code Here

      out.write(b, off, len);
    }


    @Override
    public void flush() throws IOException {
      CompressionOutputStream cout = (CompressionOutputStream) out;
      cout.finish();
      cout.flush();
      cout.resetState();
    }

View Full Code Here

    System.out.println(name + ":");


    // compress it
    List<Long> compressDurations = new ArrayList<Long>();
    ByteArrayOutputStream compressedStream = new ByteArrayOutputStream();
    CompressionOutputStream compressingStream =
        algorithm.createPlainCompressionStream(compressedStream, compressor);
    try {
      for (int itTime = 0; itTime < benchmarkNTimes; ++itTime) {
        final long startTime = System.nanoTime();
        compressingStream.resetState();
        compressedStream.reset();
        compressingStream.write(buffer, offset, length);
        compressingStream.flush();
        compressedStream.toByteArray();


        final long finishTime = System.nanoTime();


        // add time record
        if (itTime >= benchmarkNOmit) {
          compressDurations.add(finishTime - startTime);
        }
      }
    } catch (IOException e) {
      throw new RuntimeException(String.format(
          "Benchmark, or encoding algorithm '%s' cause some stream problems",
          name), e);
    }
    compressingStream.close();
    printBenchmarkResult(length, compressDurations, Manipulation.COMPRESSION);


    byte[] compBuffer = compressedStream.toByteArray();


    // uncompress it several times and measure performance

View Full Code Here

      out.write(b, off, len);
    }


    @Override
    public void flush() throws IOException {
      CompressionOutputStream cout = (CompressionOutputStream) out;
      cout.finish();
      cout.flush();
      cout.resetState();
    }

View Full Code Here

  private final BZip2Codec codec = new BZip2Codec();


  @Override
  public byte[] compress(byte[] bytes) {
    ByteArrayOutputStream bos = null;
    CompressionOutputStream sos = null;
    DataOutputStream dos = null;
    byte[] compressedBytes = null;


    try {
      bos = new ByteArrayOutputStream();
      sos = codec.createOutputStream(bos);
      dos = new DataOutputStream(sos);
      dos.close(); // Flush the stream as no more data will be sent.


      compressedBytes = bos.toByteArray();
    } catch (IOException ioe) {
      LOG.error("Unable to compress", ioe);
    } finally {
      try {
        sos.close();
        bos.close();
      } catch (IOException e) {
        LOG.warn("Failed to close compression streams.", e);
      }
    }

View Full Code Here

      out.write(b, off, len);
    }


    @Override
    public void flush() throws IOException {
      CompressionOutputStream cout = (CompressionOutputStream) out;
      cout.finish();
      cout.flush();
      cout.resetState();
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.io.compress.CompressionOutputStream

com.dianping.cosmos.hive.server.queryengine.jdbc.DataFileStore

com.hadoop.compression.snappy.CompressionTests

com.pinterest.secor.io.FileReaderWriterTest

org.apache.accumulo.core.file.rfile.bcfile.Compression$FinishOnFlushCompressionStream

org.apache.hadoop.chukwa.datacollection.sender.ChukwaHttpSender$BuffersRequestEntity

org.apache.hadoop.hbase.io.compress.Compression$FinishOnFlushCompressionStream

org.apache.hadoop.hbase.io.hfile.Compression$FinishOnFlushCompressionStream

org.apache.hadoop.hbase.regionserver.DataBlockEncodingTool

org.apache.hadoop.hbase.util.TestCompressionTest

org.apache.hadoop.hdfs.server.namenode.ha.TestStandbyCheckpoints$SlowCodec

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.