Package org.apache.hadoop.io.compress

Examples of org.apache.hadoop.io.compress.GzipCodec$GzipZlibCompressor

49
50
51
52
53
54
55
56
57
58
      this.outputDir = outputDir;
      fs = getPath(currentChunkID).getFileSystem(conf);
      currentChunkID = 0;
      conf.setClass("mapred.output.compression.codec", GzipCodec.class,
          CompressionCodec.class);
      CompressionCodec codec = new GzipCodec();
      writer = SequenceFile.createWriter(fs, conf,
          getPath(currentChunkID), Text.class, Text.class,
          CompressionType.BLOCK, codec);
    }
View Full Code Here
169
170
171
172
173
174
175
176
177
178
179
         }
        writer.close();
    } else if ("SequenceFile".equals(fileType)){
        CompressionCodec codec = null;
        if ("gz".equals(codecName))
          codec = new GzipCodec();
        else if (!"none".equals(codecName))
          throw new IOException("Codec not supported.");

        SequenceFile.Writer writer;
View Full Code Here
182
183
184
185
186
187
188
189
190
191
192
         }
        writer.close();
    } else if ("SequenceFile".equals(fileType)){
        CompressionCodec codec = null;
        if ("gz".equals(codecName))
          codec = new GzipCodec();
        else if (!"none".equals(codecName))
          throw new IOException("Codec not supported.");

        SequenceFile.Writer writer;
View Full Code Here
172
173
174
175
176
177
178
179
180
181
182
         }
        writer.close();
    } else if ("SequenceFile".equals(fileType)){
        CompressionCodec codec = null;
        if ("gz".equals(codecName))
          codec = new GzipCodec();
        else if (!"none".equals(codecName))
          throw new IOException("Codec not supported.");

        SequenceFile.Writer writer;
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
  private static CompressionCodec getCodec(Configuration conf,
      SqoopOptions options) throws IOException {
    if (options.shouldUseCompression()) {
      if (options.getCompressionCodec() == null) {
        return new GzipCodec();
      } else {
        return CodecMap.getCodec(options.getCompressionCodec(), conf);
      }
    }
    return null;
View Full Code Here
112
113
114
115
116
117
118
119
120
121
122
          getTableName());

      reader = SeqFileReader.getSeqFileReader(getDataFilePath().toString());

      if (codec == null) {
        codec = new GzipCodec();
      }
      assertTrue("Block compressed", reader.isBlockCompressed());
      assertEquals(codec.getClass(), reader.getCompressionCodec().getClass());

      // here we can actually instantiate (k, v) pairs.
View Full Code Here
160
161
162
163
164
165
166
167
168
169
170
      conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
    }
    FileSystem fs = FileSystem.get(conf);

    if (codec == null) {
      codec = new GzipCodec();
    }
    ReflectionUtils.setConf(codec, getConf());
    Path p = new Path(getDataFilePath().toString()
        + codec.getDefaultExtension());
    InputStream is = codec.createInputStream(fs.open(p));
View Full Code Here
177
178
179
180
181
182
183
184
185
186
187
         }
        writer.close();
    } else if ("SequenceFile".equals(fileType)){
        CompressionCodec codec = null;
        if ("gz".equals(codecName))
          codec = new GzipCodec();
        else if (!"none".equals(codecName))
          throw new IOException("Codec not supported.");

        SequenceFile.Writer writer;
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
  private static InputStream createStream(FSDataInputStream in, JobConf job)
    throws IOException{
    InputStream finalStream = in ;
    boolean gzipped = StreamInputFormat.isGzippedInput(job);
    if ( gzipped ) {
      GzipCodec codec = new GzipCodec();
      codec.setConf(job);
      finalStream = codec.createInputStream(in);
    }
    return finalStream;
  }
View Full Code Here
224
225
226
227
228
229
230
231
232
233
234
   * Test using the gzip codec for reading
   */
  //@Test(timeout=10000)
  public void testGzip() throws IOException {
    JobConf job = new JobConf(defaultConf);
    CompressionCodec gzip = new GzipCodec();
    ReflectionUtils.setConf(gzip, job);
    localFs.delete(workDir, true);
    writeFile(localFs, new Path(workDir, "part1.txt.gz"), gzip,
              "the quick\nbrown\nfox jumped\nover\n the lazy\n dog\n");
    writeFile(localFs, new Path(workDir, "part2.txt.gz"), gzip,
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.compress.GzipCodec$GzipZlibCompressor

  • cascading.tuple.hadoop.SpillableTupleHadoopTest
  • com.cloudera.flume.handlers.hdfs.CustomDfsSink
  • com.cloudera.flume.handlers.hdfs.TestEscapedCustomOutputDfs
  • com.cloudera.sqoop.io.TestSplittableBufferedWriter
  • com.cloudera.sqoop.TestCompression
  • com.fb2pdf.hadoop.cluster.SequenceFilesFromDirectory$ChunkedWriter
  • com.pinterest.secor.common.FileRegistryTest
  • com.pinterest.secor.io.FileReaderWriterTest
  • org.apache.flume.sink.hdfs.TestUseRawLocalFileSystem
  • org.apache.hadoop.hbase.io.hfile.TestHFilePerformance
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.