Examples of org.apache.hadoop.io.compress.Compressor

org.apache.hadoop.io.compress.Compressor
Specification of a stream-based 'compressor' which can be plugged into a {@link CompressionOutputStream} to compress data.This is modelled after {@link java.util.zip.Deflater}

    fs = (HFileSystem)HFileSystem.get(TEST_UTIL.getConfiguration());
  }


  public byte[] createTestV1Block(Compression.Algorithm algo)
      throws IOException {
    Compressor compressor = algo.getCompressor();
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    OutputStream os = algo.createCompressionStream(baos, compressor, 0);
    DataOutputStream dos = new DataOutputStream(os);
    BlockType.META.write(dos); // Let's make this a meta block.
    TestHFileBlock.writeTestBlockContents(dos);

View Full Code Here

        " previously failed test.");
      }
    }


    try {
      Compressor c = algo.getCompressor();
      algo.returnCompressor(c);
      compressionTestResults[algo.ordinal()] = true; // passes
    } catch (Throwable t) {
      compressionTestResults[algo.ordinal()] = false; // failure
      throw new IOException(t);

View Full Code Here

        " previously failed test.");
      }
    }


    try {
      Compressor c = algo.getCompressor();
      algo.returnCompressor(c);
      compressionTestResults[algo.ordinal()] = true; // passes
    } catch (Throwable t) {
      compressionTestResults[algo.ordinal()] = false; // failure
      throw new IOException(t);

View Full Code Here

   *              <code>Compressor</code>
   * @return <code>Compressor</code> for the given 
   *         <code>CompressionCodec</code> from the pool or a new one
   */
  public static Compressor getCompressor(CompressionCodec codec) {
    Compressor compressor = (Compressor) borrow(compressorPool, codec.getCompressorType());
    if (compressor == null) {
      compressor = codec.createCompressor();
      LOG.info("Got brand-new compressor");
    } else {
      LOG.debug("Got recycled compressor");

View Full Code Here

  @Test
  public void testZlibCompressorDecompressorSetDictionary() {
    Configuration conf = new Configuration();
    conf.setBoolean(CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY, true);
    if (ZlibFactory.isNativeZlibLoaded(conf)) {
      Compressor zlibCompressor = ZlibFactory.getZlibCompressor(conf);
      Decompressor zlibDecompressor = ZlibFactory.getZlibDecompressor(conf);


      checkSetDictionaryNullPointerException(zlibCompressor);
      checkSetDictionaryNullPointerException(zlibDecompressor);

View Full Code Here

    conf.setBoolean(CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY, true);
    if (ZlibFactory.isNativeZlibLoaded(conf)) {
      byte[] rawData;
      int tryNumber = 5;
      int BYTE_SIZE = 10 * 1024;
      Compressor zlibCompressor = ZlibFactory.getZlibCompressor(conf);
      Decompressor zlibDecompressor = ZlibFactory.getZlibDecompressor(conf);
      rawData = generate(BYTE_SIZE);
      try {
        for (int i = 0; i < tryNumber; i++)
          compressDecompressZlib(rawData, (ZlibCompressor) zlibCompressor,
              (ZlibDecompressor) zlibDecompressor);
        zlibCompressor.reinit(conf);
      } catch (Exception ex) {
        fail("testZlibCompressorDecompressorWithConfiguration ex error " + ex);
      }
    } else {
      assertTrue("ZlibFactory is using native libs against request",

View Full Code Here

    // Presizing doesn't work because can't tell what size will be when serialized.
    // BBOS will resize itself.
    ByteBufferOutputStream baos =
      new ByteBufferOutputStream(this.cellBlockBuildingInitialBufferSize);
    OutputStream os = baos;
    Compressor poolCompressor = null;
    try {
      if (compressor != null) {
        if (compressor instanceof Configurable) ((Configurable)compressor).setConf(this.conf);
        poolCompressor = CodecPool.getCompressor(compressor);
        os = compressor.createOutputStream(os, poolCompressor);

View Full Code Here

    return totalSize;
  }


  public byte[] createTestV1Block(Compression.Algorithm algo)
      throws IOException {
    Compressor compressor = algo.getCompressor();
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    OutputStream os = algo.createCompressionStream(baos, compressor, 0);
    DataOutputStream dos = new DataOutputStream(os);
    BlockType.META.write(dos); // Let's make this a meta block.
    writeTestBlockContents(dos);

View Full Code Here

    fs = (HFileSystem)HFileSystem.get(TEST_UTIL.getConfiguration());
  }


  public byte[] createTestV1Block(Compression.Algorithm algo)
      throws IOException {
    Compressor compressor = algo.getCompressor();
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    OutputStream os = algo.createCompressionStream(baos, compressor, 0);
    DataOutputStream dos = new DataOutputStream(os);
    BlockType.META.write(dos); // Let's make this a meta block.
    TestHFileBlock.writeTestBlockContents(dos);

View Full Code Here


    final Path resultPath = new Path(tempDir, "result.text");


    System.out.println("testHadoop20JHParser sent its output to " + resultPath);


    Compressor compressor;


    FileSystem fs = resultPath.getFileSystem(conf);
    CompressionCodec codec =
        new CompressionCodecFactory(conf).getCodec(resultPath);
    OutputStream output;

View Full Code Here

0 1 2 3 4 5 6 7

TOP

Related Classes of org.apache.hadoop.io.compress.Compressor

cascading.tuple.hadoop.collect.HadoopSpillableTupleList

com.cloudera.flume.handlers.hdfs.CustomDfsSink

com.hadoop.compression.lzo.TestLzoCodec

org.apache.hadoop.hbase.io.hfile.TestHFileBlock

org.apache.hadoop.hbase.io.hfile.TestHFileBlockCompatibility

org.apache.hadoop.hbase.ipc.IPCUtil

org.apache.hadoop.hbase.util.CompressionTest

org.apache.hadoop.hive.ql.io.CodecPool

org.apache.hadoop.hive.ql.io.RCFile$Writer

org.apache.hadoop.io.compress.zlib.TestZlibCompressorDecompressor

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.