Package org.apache.hadoop.io.compress

Examples of org.apache.hadoop.io.compress.CompressionInputStream

644
645
646
647
648
649
650
651
652
653
654
            {
              CompressionCodecFactory compressionCodecs =  new CompressionCodecFactory(job);
              final CompressionCodec codec = compressionCodecs.getCodec(file);
               if (codec != null) {
                  end = Long.MAX_VALUE;
                    CompressionInputStream stream = codec.createInputStream(fileIn);
                    this.xmlLoaderBPIS = new XMLLoaderBufferedPositionedInputStream(stream,start,end);
                  }
            }
           
            else
View Full Code Here
162
163
164
165
166
167
168
169
170
171
172
    InputStream is = null;
    if (compressor != null) {
      // GZIPCodec fails w/ NPE if no configuration.
      if (compressor instanceof Configurable) ((Configurable)compressor).setConf(this.conf);
      Decompressor poolDecompressor = CodecPool.getDecompressor(compressor);
      CompressionInputStream cis =
        compressor.createInputStream(new ByteArrayInputStream(cellBlock, offset, length),
        poolDecompressor);
      try {
        // TODO: This is ugly.  The buffer will be resized on us if we guess wrong.
        // TODO: Reuse buffers.
View Full Code Here
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
      compressedKeyLen = in.readInt();
      if (decompress) {
        keyTempBuffer.reset();
        keyTempBuffer.write(in, compressedKeyLen);
        keyDecompressBuffer.reset(keyTempBuffer.getData(), compressedKeyLen);
        CompressionInputStream deflatFilter = codec.createInputStream(
            keyDecompressBuffer, keyDecompressor);
        DataInputStream compressedIn = new DataInputStream(deflatFilter);
        deflatFilter.resetState();
        keyDecompressedData.reset();
        keyDecompressedData.write(compressedIn, currentKeyLength);
        keyDataIn.reset(keyDecompressedData.getData(), currentKeyLength);
        currentKey.readFields(keyDataIn);
      } else {
View Full Code Here
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
      compressedKeyLen = in.readInt();
      if (decompress) {
        keyTempBuffer.reset();
        keyTempBuffer.write(in, compressedKeyLen);
        keyDecompressBuffer.reset(keyTempBuffer.getData(), compressedKeyLen);
        CompressionInputStream deflatFilter = codec.createInputStream(
            keyDecompressBuffer, keyDecompressor);
        DataInputStream compressedIn = new DataInputStream(deflatFilter);
        deflatFilter.resetState();
        keyDecompressedData.reset();
        keyDecompressedData.write(compressedIn, currentKeyLength);
        keyDataIn.reset(keyDecompressedData.getData(), currentKeyLength);
        currentKey.readFields(keyDataIn);
      } else {
View Full Code Here
99
100
101
102
103
104
105
106
107
108
109
                .thenReturn(fileInputStream);

        PowerMockito.whenNew(FileOutputStream.class).withAnyArguments()
                .thenReturn(fileOutputStream);

        CompressionInputStream inputStream = Mockito
                .mock(CompressionInputStream.class);
        CompressionOutputStream outputStream = Mockito
                .mock(CompressionOutputStream.class);
        Mockito.when(codec.createInputStream(Mockito.any(InputStream.class)))
                .thenReturn(inputStream);
View Full Code Here
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
      compressedKeyLen = in.readInt();
      if (decompress) {
        keyTempBuffer.reset();
        keyTempBuffer.write(in, compressedKeyLen);
        keyDecompressBuffer.reset(keyTempBuffer.getData(), compressedKeyLen);
        CompressionInputStream deflatFilter = codec.createInputStream(
            keyDecompressBuffer, keyDecompressor);
        DataInputStream compressedIn = new DataInputStream(deflatFilter);
        deflatFilter.resetState();
        keyDecompressedData.reset();
        keyDecompressedData.write(compressedIn, currentKeyLength);
        keyDataIn.reset(keyDecompressedData.getData(), currentKeyLength);
        currentKey.readFields(keyDataIn);
      } else {
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
   * @return
   */
  @Override
  public byte[] decompress(byte[] compressedBytes) {
    ByteArrayInputStream bis = null;
    CompressionInputStream sis = null;
    DataInputStream dis = null;
    byte[] bytes = null;

    try {
      bis = new ByteArrayInputStream(compressedBytes);
      sis = codec.createInputStream(bis);
      dis = new DataInputStream(sis);
      bytes = IOUtils.toByteArray(dis);
    } catch (IOException ioe) {
      LOG.error("Unable to decompress.", ioe);
    } finally {
      try {
        dis.close();
        sis.close();
        bis.close();
      } catch (IOException e) {
        LOG.warn("Failed to close decompression streams.", e);
      }
    }
View Full Code Here
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
      compressedKeyLen = in.readInt();
      if (decompress) {
        keyTempBuffer.reset();
        keyTempBuffer.write(in, compressedKeyLen);
        keyDecompressBuffer.reset(keyTempBuffer.getData(), compressedKeyLen);
        CompressionInputStream deflatFilter = codec.createInputStream(
            keyDecompressBuffer, keyDecompressor);
        DataInputStream compressedIn = new DataInputStream(deflatFilter);
        deflatFilter.resetState();
        keyDecompressedData.reset();
        keyDecompressedData.write(compressedIn, currentKeyLength);
        keyDataIn.reset(keyDecompressedData.getData(), currentKeyLength);
        currentKey.readFields(keyDataIn);
      } else {
View Full Code Here
155
156
157
158
159
160
161
162
163
164
165
    InputStream is = null;
    if (compressor != null) {
      // GZIPCodec fails w/ NPE if no configuration.
      if (compressor instanceof Configurable) ((Configurable)compressor).setConf(this.conf);
      Decompressor poolDecompressor = CodecPool.getDecompressor(compressor);
      CompressionInputStream cis =
        compressor.createInputStream(new ByteArrayInputStream(cellBlock, offset, length),
        poolDecompressor);
      try {
        // TODO: This is ugly.  The buffer will be resized on us if we guess wrong.
        // TODO: Reuse buffers.
View Full Code Here
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
      compressedKeyLen = in.readInt();
      if (decompress) {
        keyTempBuffer.reset();
        keyTempBuffer.write(in, compressedKeyLen);
        keyDecompressBuffer.reset(keyTempBuffer.getData(), compressedKeyLen);
        CompressionInputStream deflatFilter = codec.createInputStream(
            keyDecompressBuffer, keyDecompressor);
        DataInputStream compressedIn = new DataInputStream(deflatFilter);
        deflatFilter.resetState();
        keyDecompressedData.reset();
        keyDecompressedData.write(compressedIn, currentKeyLength);
        keyDataIn.reset(keyDecompressedData.getData(), currentKeyLength);
        currentKey.readFields(keyDataIn);
      } else {
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.compress.CompressionInputStream

  • com.dp.nebula.wormhole.plugins.reader.hivereader.HiveReader
  • com.hadoop.compression.snappy.CompressionTests
  • com.pinterest.secor.io.FileReaderWriterTest
  • hivemall.utils.hadoop.HadoopUtils
  • org.apache.hadoop.hbase.ipc.IPCUtil
  • org.apache.hadoop.hive.ql.io.RCFile$Reader
  • org.apache.hadoop.io.compress.lz4.TestLz4CompressorDecompressor
  • org.apache.hadoop.io.compress.snappy.TestSnappyCodec
  • org.apache.hadoop.io.compress.snappy.TestSnappyCompressorDecompressor
  • org.apache.hadoop.mapred.gridmix.CompressionEmulationUtil
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.