Package org.apache.hadoop.io.compress

Examples of org.apache.hadoop.io.compress.Compressor

1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
    private void flushRecords() throws IOException {

      key.numberRows = bufferedRecords;

      Compressor compressor = null;
      NonSyncDataOutputBuffer valueBuffer = null;
      CompressionOutputStream deflateFilter = null;
      DataOutputStream deflateOut = null;
      boolean isCompressed = isCompressed();
      int valueLength = 0;
View Full Code Here
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
      checkAndWriteSync(); // sync
      out.writeInt(recordLen); // total record length
      out.writeInt(keyLength); // key portion length

      if(this.isCompressed()) {
        Compressor compressor = CodecPool.getCompressor(codec);
        NonSyncDataOutputBuffer compressionBuffer =
          new NonSyncDataOutputBuffer();
        CompressionOutputStream deflateFilter =
          codec.createOutputStream(compressionBuffer, compressor);
        DataOutputStream deflateOut = new DataOutputStream(deflateFilter);
View Full Code Here
927
928
929
930
931
932
933
934
935
936
937
    private void flushRecords() throws IOException {

      key.numberRows = bufferedRecords;

      Compressor compressor = null;
      NonSyncDataOutputBuffer valueBuffer = null;
      CompressionOutputStream deflateFilter = null;
      DataOutputStream deflateOut = null;
      boolean isCompressed = isCompressed();
      int valueLength = 0;
View Full Code Here
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
      checkAndWriteSync(); // sync
      out.writeInt(recordLen); // total record length
      out.writeInt(keyLength); // key portion length

      if(this.isCompressed()) {
        Compressor compressor = CodecPool.getCompressor(codec);
        NonSyncDataOutputBuffer compressionBuffer =
          new NonSyncDataOutputBuffer();
        CompressionOutputStream deflateFilter =
          codec.createOutputStream(compressionBuffer, compressor);
        DataOutputStream deflateOut = new DataOutputStream(deflateFilter);
View Full Code Here
176
177
178
179
180
181
182
183
184
185
186
      FileSystem outFS = outPath.getFileSystem(conf);

      CompressionCodec codec =
          new CompressionCodecFactory(conf).getCodec(outPath);
      OutputStream output;
      Compressor compressor = null;
      if (codec != null) {
        compressor = CodecPool.getCompressor(codec);
        output = codec.createOutputStream(outFS.create(outPath), compressor);
      } else {
        output = outFS.create(outPath);
View Full Code Here
343
344
345
346
347
348
349
350
351
352
353
            tempPaths.add(nextSegment);

            CompressionCodec codec =
                new CompressionCodecFactory(conf).getCodec(nextSegment);
            OutputStream output;
            Compressor compressor = null;
            if (codec != null) {
              compressor = CodecPool.getCompressor(codec);
              output = codec.createOutputStream(tempUncompOut, compressor);
            } else {
              output = tempUncompOut;
View Full Code Here
115
116
117
118
119
120
121
122
123
124
125
      writer = pathManager.open();
      LOG.info("Creating HDFS file: " + pathManager.getOpenPath());
      return;
    }

    Compressor cmp = codec.createCompressor();
    dstPath = new Path(path + codec.getDefaultExtension());
    hdfs = dstPath.getFileSystem(conf);
    pathManager = new PathManager(hdfs, dstPath.getParent(), dstPath.getName());
    writer = pathManager.open();
    try {
View Full Code Here
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
    private void flushRecords() throws IOException {

      key.numberRows = bufferedRecords;

      Compressor compressor = null;
      NonSyncDataOutputBuffer valueBuffer = null;
      CompressionOutputStream deflateFilter = null;
      DataOutputStream deflateOut = null;
      boolean isCompressed = isCompressed();
      int valueLength = 0;
View Full Code Here
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
      checkAndWriteSync(); // sync
      out.writeInt(recordLen); // total record length
      out.writeInt(keyLength); // key portion length

      if(this.isCompressed()) {
        Compressor compressor = CodecPool.getCompressor(codec);
        NonSyncDataOutputBuffer compressionBuffer =
          new NonSyncDataOutputBuffer();
        CompressionOutputStream deflateFilter =
          codec.createOutputStream(compressionBuffer, compressor);
        DataOutputStream deflateOut = new DataOutputStream(deflateFilter);
View Full Code Here
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
    try
      {
      outputStream = new FileOutputStream( file );

      Compressor compressor = null;

      if( codec != null )
        {
        compressor = getCompressor();
        outputStream = codec.createOutputStream( outputStream, compressor );
        }

      final Compressor finalCompressor = compressor;

      return new HadoopTupleOutputStream( outputStream, tupleSerialization.getElementWriter() )
      {
      @Override
      public void close() throws IOException
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.compress.Compressor

  • cascading.tuple.hadoop.collect.HadoopSpillableTupleList
  • com.cloudera.flume.handlers.hdfs.CustomDfsSink
  • com.hadoop.compression.lzo.TestLzoCodec
  • org.apache.hadoop.hbase.io.hfile.TestHFileBlock
  • org.apache.hadoop.hbase.io.hfile.TestHFileBlockCompatibility
  • org.apache.hadoop.hbase.ipc.IPCUtil
  • org.apache.hadoop.hbase.util.CompressionTest
  • org.apache.hadoop.hive.ql.io.CodecPool
  • org.apache.hadoop.hive.ql.io.RCFile$Writer
  • org.apache.hadoop.io.compress.zlib.TestZlibCompressorDecompressor
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.