Package org.apache.hadoop.io.compress

Examples of org.apache.hadoop.io.compress.GzipCodec$GzipOutputStream$ResetableGZIPOutputStream

112
113
114
115
116
117
118
119
120
121
122
          getTableName());

      reader = SeqFileReader.getSeqFileReader(getDataFilePath().toString());

      if (codec == null) {
        codec = new GzipCodec();
      }
      assertTrue("Block compressed", reader.isBlockCompressed());
      assertEquals(codec.getClass(), reader.getCompressionCodec().getClass());

      // here we can actually instantiate (k, v) pairs.
View Full Code Here
160
161
162
163
164
165
166
167
168
169
170
      conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
    }
    FileSystem fs = FileSystem.get(conf);

    if (codec == null) {
      codec = new GzipCodec();
    }
    Path p = new Path(getDataFilePath().toString()
        + codec.getDefaultExtension());
    InputStream is = codec.createInputStream(fs.open(p));
    BufferedReader r = new BufferedReader(new InputStreamReader(is));
View Full Code Here
130
131
132
133
134
135
136
137
138
139
140
           
            // Use the codec according to the test case
            if (type.equals("bz2"))
               codec = new BZip2Codec();
            else if (type.equals("gz")) {
               codec = new GzipCodec();
               ((GzipCodec)codec).setConf(new Configuration());
            }

            CompressionInputStream createInputStream = codec
                  .createInputStream(new FileInputStream(file));
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
  private static CompressionCodec getCodec(Configuration conf,
      SqoopOptions options) throws IOException {
    if (options.shouldUseCompression()) {
      if (options.getCompressionCodec() == null) {
        return new GzipCodec();
      } else {
        return CodecMap.getCodec(options.getCompressionCodec(), conf);
      }
    }
    return null;
View Full Code Here
112
113
114
115
116
117
118
119
120
121
122
          getTableName());

      reader = SeqFileReader.getSeqFileReader(getDataFilePath().toString());

      if (codec == null) {
        codec = new GzipCodec();
      }
      assertTrue("Block compressed", reader.isBlockCompressed());
      assertEquals(codec.getClass(), reader.getCompressionCodec().getClass());

      // here we can actually instantiate (k, v) pairs.
View Full Code Here
160
161
162
163
164
165
166
167
168
169
170
      conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
    }
    FileSystem fs = FileSystem.get(conf);

    if (codec == null) {
      codec = new GzipCodec();
      ReflectionUtils.setConf(codec, getConf());
    }
    Path p = new Path(getDataFilePath().toString()
        + codec.getDefaultExtension());
    InputStream is = codec.createInputStream(fs.open(p));
View Full Code Here
177
178
179
180
181
182
183
184
185
186
187
    }
  }

  public void testNonSplittingGzipFile() throws IOException {
    SplittingOutputStream os  = new SplittingOutputStream(getConf(),
        getWritePath(), "nonsplit-", 0, new GzipCodec());
    SplittableBufferedWriter w = new SplittableBufferedWriter(os, true);
    try {
      w.allowSplit();
      w.write("This is a string!");
      w.newLine();
View Full Code Here
267
268
269
270
271
272
273
274
275
276
277
    }
  }

  public void testSplittingGzipFile() throws IOException {
    SplittingOutputStream os = new SplittingOutputStream(getConf(),
        getWritePath(), "splitz-", 3, new GzipCodec());
    SplittableBufferedWriter w = new SplittableBufferedWriter(os, true);
    try {
      w.write("This is a string!");
      w.newLine();
      w.write("This is another string!");
View Full Code Here
37
38
39
40
41
42
43
44
45
46
47
48
  public void testIFileWriterWithCodec() throws Exception {
    Configuration conf = new Configuration();
    FileSystem localFs = FileSystem.getLocal(conf);
    FileSystem rfs = ((LocalFileSystem)localFs).getRaw();
    Path path = new Path(new Path("build/test.ifile"), "data");
    DefaultCodec codec = new GzipCodec();
    codec.setConf(conf);
    IFile.Writer<Text, Text> writer =
      new IFile.Writer<Text, Text>(conf, rfs, path, Text.class, Text.class,
                                   codec, null);
    writer.close();
  }
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
63
  public void testIFileReaderWithCodec() throws Exception {
    Configuration conf = new Configuration();
    FileSystem localFs = FileSystem.getLocal(conf);
    FileSystem rfs = ((LocalFileSystem)localFs).getRaw();
    Path path = new Path(new Path("build/test.ifile"), "data");
    DefaultCodec codec = new GzipCodec();
    codec.setConf(conf);
    IFile.Writer<Text, Text> writer =
        new IFile.Writer<Text, Text>(conf, rfs, path, Text.class, Text.class,
                                     codec, null);
    writer.close();
    IFile.Reader<Text, Text> reader =
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.compress.GzipCodec$GzipOutputStream$ResetableGZIPOutputStream

  • cascading.tuple.hadoop.SpillableTupleHadoopTest
  • com.cloudera.flume.handlers.hdfs.CustomDfsSink
  • com.cloudera.flume.handlers.hdfs.TestEscapedCustomOutputDfs
  • com.cloudera.sqoop.io.TestSplittableBufferedWriter
  • com.cloudera.sqoop.TestCompression
  • com.fb2pdf.hadoop.cluster.SequenceFilesFromDirectory$ChunkedWriter
  • com.pinterest.secor.common.FileRegistryTest
  • com.pinterest.secor.io.FileReaderWriterTest
  • org.apache.flume.sink.hdfs.TestUseRawLocalFileSystem
  • org.apache.hadoop.hbase.io.hfile.TestHFilePerformance
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.