Package org.apache.hadoop.io.compress

Examples of org.apache.hadoop.io.compress.Compressor

105
106
107
108
109
110
111
112
113
114
115
    // within the estimated buffer size, then the cost is not worth it.  If we are often well
    // outside the guesstimated buffer size, the processing can be done in half the time if we
    // go w/ the estimated size rather than let the buffer resize.
    ByteBufferOutputStream baos = new ByteBufferOutputStream(bufferSize);
    OutputStream os = baos;
    Compressor poolCompressor = null;
    try {
      if (compressor != null) {
        if (compressor instanceof Configurable) ((Configurable)compressor).setConf(this.conf);
        poolCompressor = CodecPool.getCompressor(compressor);
        os = compressor.createOutputStream(os, poolCompressor);
View Full Code Here
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
    Configuration conf = new Configuration();
    CompressionCodec codec = ReflectionUtils.newInstance(
      LzoCodec.class, conf);

    // Put a codec in the pool
    Compressor c1 = CodecPool.getCompressor(codec);
    assertEquals(LzoCompressor.CompressionStrategy.LZO1X_1,
                 ((LzoCompressor)c1).getStrategy());
    CodecPool.returnCompressor(c1);

    // Set compression strategy
    LzoCodec.setCompressionStrategy(conf, LzoCompressor.CompressionStrategy.LZO1C_BEST_COMPRESSION);

    Compressor c2 = CodecPool.getCompressor(codec, conf);
    assertSame(c1, c2);

    assertEquals(LzoCompressor.CompressionStrategy.LZO1C_BEST_COMPRESSION,
                 ((LzoCompressor)c2).getStrategy());
View Full Code Here
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
    Configuration conf = new Configuration();
    CompressionCodec codec = ReflectionUtils.newInstance(
      LzoCodec.class, conf);

    // Put a codec in the pool
    Compressor c1 = CodecPool.getCompressor(codec);
    assertEquals(LzoCompressor.CompressionStrategy.LZO1X_1,
                 ((LzoCompressor)c1).getStrategy());
    CodecPool.returnCompressor(c1);

    // Set compression strategy
    int newBufSize = LzoCodec.DEFAULT_LZO_BUFFER_SIZE * 2;
    LzoCodec.setBufferSize(conf, newBufSize);

    Compressor c2 = CodecPool.getCompressor(codec, conf);
    assertSame(c1, c2);

    assertEquals(newBufSize, ((LzoCompressor)c2).getDirectBufferSize());

  }
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
    // Set compression strategy
    LzoCodec.setCompressionStrategy(conf, LzoCompressor.CompressionStrategy.LZO1C_BEST_COMPRESSION);

    // Put a codec in the pool with BEST_COMPRESSION strategy
    Compressor c1 = CodecPool.getCompressor(codec, conf);
    assertEquals(LzoCompressor.CompressionStrategy.LZO1C_BEST_COMPRESSION,
                 ((LzoCompressor)c1).getStrategy());
    CodecPool.returnCompressor(c1);

    // Get it back from the pool without specifying any configuration,
    // it should return to default compression
    Compressor c2 = CodecPool.getCompressor(codec);
    assertSame(c1, c2);

    assertEquals(LzoCompressor.CompressionStrategy.LZO1X_1,
                 ((LzoCompressor)c2).getStrategy());
  }
View Full Code Here
201
202
203
204
205
206
207
208
209
210
211
    return totalSize;
  }

  public byte[] createTestV1Block(Compression.Algorithm algo)
      throws IOException {
    Compressor compressor = algo.getCompressor();
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    OutputStream os = algo.createCompressionStream(baos, compressor, 0);
    DataOutputStream dos = new DataOutputStream(os);
    BlockType.META.write(dos); // Let's make this a meta block.
    writeTestBlockContents(dos);
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
      }
    }

    Configuration conf = HBaseConfiguration.create();
    try {
      Compressor c = algo.getCompressor();
      algo.returnCompressor(c);
      compressionTestResults[algo.ordinal()] = true; // passes
    } catch (Throwable t) {
      compressionTestResults[algo.ordinal()] = false; // failure
      throw new IOException(t);
View Full Code Here
108
109
110
111
112
113
114
115
116
117
118
    fs = (HFileSystem)HFileSystem.get(TEST_UTIL.getConfiguration());
  }

  public byte[] createTestV1Block(Compression.Algorithm algo)
      throws IOException {
    Compressor compressor = algo.getCompressor();
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    OutputStream os = algo.createCompressionStream(baos, compressor, 0);
    DataOutputStream dos = new DataOutputStream(os);
    BlockType.META.write(dos); // Let's make this a meta block.
    TestHFileBlock.writeTestBlockContents(dos);
View Full Code Here
179
180
181
182
183
184
185
186
187
188
189
    return totalSize;
  }

  public byte[] createTestV1Block(Compression.Algorithm algo)
      throws IOException {
    Compressor compressor = algo.getCompressor();
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    OutputStream os = algo.createCompressionStream(baos, compressor, 0);
    DataOutputStream dos = new DataOutputStream(os);
    BlockType.META.write(dos); // Let's make this a meta block.
    writeTestBlockContents(dos);
View Full Code Here
239
240
241
242
243
244
245
246
247
248
249
    final Path resultPath = new Path(tempDir, "result.text");

    System.out.println("testHadoop20JHParser sent its output to " + resultPath);

    Compressor compressor;

    FileSystem fs = resultPath.getFileSystem(conf);
    CompressionCodec codec =
        new CompressionCodecFactory(conf).getCodec(resultPath);
    OutputStream output;
View Full Code Here
379
380
381
382
383
384
385
386
387
388
389
    final Path resultPath = new Path(tempDir, "result.text");

    System.out.println("testHadoop20JHParser sent its output to " + resultPath);

    Compressor compressor;

    FileSystem fs = resultPath.getFileSystem(conf);
    CompressionCodec codec =
        new CompressionCodecFactory(conf).getCodec(resultPath);
    OutputStream output;
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.compress.Compressor

  • cascading.tuple.hadoop.collect.HadoopSpillableTupleList
  • com.cloudera.flume.handlers.hdfs.CustomDfsSink
  • com.hadoop.compression.lzo.TestLzoCodec
  • org.apache.hadoop.hbase.io.hfile.TestHFileBlock
  • org.apache.hadoop.hbase.io.hfile.TestHFileBlockCompatibility
  • org.apache.hadoop.hbase.ipc.IPCUtil
  • org.apache.hadoop.hbase.util.CompressionTest
  • org.apache.hadoop.hive.ql.io.CodecPool
  • org.apache.hadoop.hive.ql.io.RCFile$Writer
  • org.apache.hadoop.io.compress.zlib.TestZlibCompressorDecompressor
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.