Package org.apache.hadoop.io.compress

Examples of org.apache.hadoop.io.compress.GzipCodec$GzipZlibDecompressor

224
225
226
227
228
229
230
231
232
233
234
   * Test using the gzip codec for reading
   */
  //@Test(timeout=10000)
  public void testGzip() throws IOException {
    JobConf job = new JobConf(defaultConf);
    CompressionCodec gzip = new GzipCodec();
    ReflectionUtils.setConf(gzip, job);
    localFs.delete(workDir, true);
    writeFile(localFs, new Path(workDir, "part1.txt.gz"), gzip,
              "the quick\nbrown\nfox jumped\nover\n the lazy\n dog\n");
    writeFile(localFs, new Path(workDir, "part2.txt.gz"), gzip,
View Full Code Here
170
171
172
173
174
175
176
177
178
179
180
         }
        writer.close();
    } else if ("SequenceFile".equals(fileType)){
        CompressionCodec codec = null;
        if ("gz".equals(codecName))
          codec = new GzipCodec();
        else if (!"none".equals(codecName))
          throw new IOException("Codec not supported.");

        SequenceFile.Writer writer;
View Full Code Here
37
38
39
40
41
42
43
44
45
46
47
48
  public void testIFileWriterWithCodec() throws Exception {
    Configuration conf = new Configuration();
    FileSystem localFs = FileSystem.getLocal(conf);
    FileSystem rfs = ((LocalFileSystem)localFs).getRaw();
    Path path = new Path(new Path("build/test.ifile"), "data");
    DefaultCodec codec = new GzipCodec();
    codec.setConf(conf);
    IFile.Writer<Text, Text> writer =
      new IFile.Writer<Text, Text>(conf, rfs, path, Text.class, Text.class,
                                   codec, null);
    writer.close();
  }
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
63
  public void testIFileReaderWithCodec() throws Exception {
    Configuration conf = new Configuration();
    FileSystem localFs = FileSystem.getLocal(conf);
    FileSystem rfs = ((LocalFileSystem)localFs).getRaw();
    Path path = new Path(new Path("build/test.ifile"), "data");
    DefaultCodec codec = new GzipCodec();
    codec.setConf(conf);
    IFile.Reader<Text, Text> reader =
      new IFile.Reader<Text, Text>(conf, rfs, path, codec, null);
    reader.close();
   
    // test check sum
View Full Code Here
60
61
62
63
64
65
66
 
  @Test
  public void testBuildCellBlock() throws IOException {
    doBuildCellBlockUndoCellBlock(this.util, new KeyValueCodec(), null);
    doBuildCellBlockUndoCellBlock(this.util, new KeyValueCodec(), new DefaultCodec());
    doBuildCellBlockUndoCellBlock(this.util, new KeyValueCodec(), new GzipCodec());
  }
View Full Code Here
189
190
191
192
193
194
195
    }
    IPCUtil util = new IPCUtil(HBaseConfiguration.create());
    ((Log4JLogger)IPCUtil.LOG).getLogger().setLevel(Level.ALL);
    timerTests(util, count, size,  new KeyValueCodec(), null);
    timerTests(util, count, size,  new KeyValueCodec(), new DefaultCodec());
    timerTests(util, count, size,  new KeyValueCodec(), new GzipCodec());
  }
View Full Code Here
208
209
210
211
212
213
214
215
216
217
218
   * Test using the gzip codec for reading
   */
  @Test(timeout=10000)
  public void testGzip() throws IOException {
    JobConf job = new JobConf(defaultConf);
    CompressionCodec gzip = new GzipCodec();
    ReflectionUtils.setConf(gzip, job);
    localFs.delete(workDir, true);
    writeFile(localFs, new Path(workDir, "part1.txt.gz"), gzip,
              "the quick\nbrown\nfox jumped\nover\n the lazy\n dog\n");
    writeFile(localFs, new Path(workDir, "part2.txt.gz"), gzip,
View Full Code Here
37
38
39
40
41
42
43
44
45
46
47
48
  public void testIFileWriterWithCodec() throws Exception {
    Configuration conf = new Configuration();
    FileSystem localFs = FileSystem.getLocal(conf);
    FileSystem rfs = ((LocalFileSystem)localFs).getRaw();
    Path path = new Path(new Path("build/test.ifile"), "data");
    DefaultCodec codec = new GzipCodec();
    codec.setConf(conf);
    IFile.Writer<Text, Text> writer =
      new IFile.Writer<Text, Text>(conf, rfs, path, Text.class, Text.class,
                                   codec, null);
    writer.close();
  }
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
63
  public void testIFileReaderWithCodec() throws Exception {
    Configuration conf = new Configuration();
    FileSystem localFs = FileSystem.getLocal(conf);
    FileSystem rfs = ((LocalFileSystem)localFs).getRaw();
    Path path = new Path(new Path("build/test.ifile"), "data");
    DefaultCodec codec = new GzipCodec();
    codec.setConf(conf);
    IFile.Writer<Text, Text> writer =
        new IFile.Writer<Text, Text>(conf, rfs, path, Text.class, Text.class,
                                     codec, null);
    writer.close();
    IFile.Reader<Text, Text> reader =
View Full Code Here
224
225
226
227
228
229
230
231
232
233
234
   * Test using the gzip codec for reading
   */
  @Test(timeout=10000)
  public void testGzip() throws IOException, InterruptedException {
    Configuration conf = new Configuration(defaultConf);
    CompressionCodec gzip = new GzipCodec();
    ReflectionUtils.setConf(gzip, conf);
    localFs.delete(workDir, true);
    writeFile(localFs, new Path(workDir, "part1.txt.gz"), gzip,
              "the quick\nbrown\nfox jumped\nover\n the lazy\n dog\n");
    writeFile(localFs, new Path(workDir, "part2.txt.gz"), gzip,
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.compress.GzipCodec$GzipZlibDecompressor

  • cascading.tuple.hadoop.SpillableTupleHadoopTest
  • com.cloudera.flume.handlers.hdfs.CustomDfsSink
  • com.cloudera.flume.handlers.hdfs.TestEscapedCustomOutputDfs
  • com.cloudera.sqoop.io.TestSplittableBufferedWriter
  • com.cloudera.sqoop.TestCompression
  • com.fb2pdf.hadoop.cluster.SequenceFilesFromDirectory$ChunkedWriter
  • com.pinterest.secor.common.FileRegistryTest
  • com.pinterest.secor.io.FileReaderWriterTest
  • org.apache.flume.sink.hdfs.TestUseRawLocalFileSystem
  • org.apache.hadoop.hbase.io.hfile.TestHFilePerformance
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.