Examples of org.apache.hadoop.io.compress.GzipCodec$GzipOutputStream$ResetableGZIPOutputStream

org.apache.hadoop.io.compress.GzipCodec
A bridge that wraps around a DeflaterOutputStream to make it a CompressionOutputStream.

  public void testCompressedFile() throws Exception {
    String file = testFile.getCanonicalPath();
    HDFSCompressedDataStream stream = new HDFSCompressedDataStream();
    context.put("hdfs.useRawLocalFileSystem", "true");
    stream.configure(context);
    stream.open(file, new GzipCodec(), CompressionType.RECORD);
    stream.append(event);
    stream.sync();
    Assert.assertTrue(testFile.length() > 0);
  }

View Full Code Here

         }
        writer.close();
    } else if ("SequenceFile".equals(fileType)){
        CompressionCodec codec = null;
        if ("gz".equals(codecName))
          codec = new GzipCodec();
        else if (!"none".equals(codecName))
          throw new IOException("Codec not supported.");


        SequenceFile.Writer writer;

View Full Code Here

   * Test using the gzip codec for reading
   */
  @Test(timeout=10000)
  public void testGzip() throws IOException {
    JobConf job = new JobConf(defaultConf);
    CompressionCodec gzip = new GzipCodec();
    ReflectionUtils.setConf(gzip, job);
    localFs.delete(workDir, true);
    writeFile(localFs, new Path(workDir, "part1.txt.gz"), gzip,
              "the quick\nbrown\nfox jumped\nover\n the lazy\n dog\n");
    writeFile(localFs, new Path(workDir, "part2.txt.gz"), gzip,

View Full Code Here

   * Test using the gzip codec for reading
   */
  @Test(timeout=10000)
  public void testGzip() throws IOException, InterruptedException {
    Configuration conf = new Configuration(defaultConf);
    CompressionCodec gzip = new GzipCodec();
    ReflectionUtils.setConf(gzip, conf);
    localFs.delete(workDir, true);
    writeFile(localFs, new Path(workDir, "part1.txt.gz"), gzip,
              "the quick\nbrown\nfox jumped\nover\n the lazy\n dog\n");
    writeFile(localFs, new Path(workDir, "part2.txt.gz"), gzip,

View Full Code Here

                .thenReturn(writer);


        Mockito.when(writer.getLength()).thenReturn(123L);


        FileReaderWriter createdWriter = mRegistry.getOrCreateWriter(
                mLogFilePathGz, new GzipCodec());
        assertTrue(createdWriter == writer);
    }

View Full Code Here

    }


    public void testGetOrCreateWriterCompressed() throws Exception {
        createCompressedWriter();


        mRegistry.getOrCreateWriter(mLogFilePathGz, new GzipCodec());


        // Verify that the method has been called exactly once (the default).
        PowerMockito.verifyStatic();
        FileUtil.delete(PATH_GZ);
        PowerMockito.verifyStatic();

View Full Code Here

                "com.pinterest.secor.io.impl.DelimitedTextFileReaderWriter");
        mConfig = new SecorConfig(properties);
    }


    private void mockGzipFileReaderWriter() throws Exception {
        GzipCodec codec = PowerMockito.mock(GzipCodec.class);
        PowerMockito.whenNew(GzipCodec.class).withNoArguments()
                .thenReturn(codec);


        FileInputStream fileInputStream = Mockito.mock(FileInputStream.class);
        FileOutputStream fileOutputStream = Mockito.mock(FileOutputStream.class);


        PowerMockito.whenNew(FileInputStream.class).withAnyArguments()
                .thenReturn(fileInputStream);


        PowerMockito.whenNew(FileOutputStream.class).withAnyArguments()
                .thenReturn(fileOutputStream);


        CompressionInputStream inputStream = Mockito
                .mock(CompressionInputStream.class);
        CompressionOutputStream outputStream = Mockito
                .mock(CompressionOutputStream.class);
        Mockito.when(codec.createInputStream(Mockito.any(InputStream.class)))
                .thenReturn(inputStream);


        Mockito.when(codec.createOutputStream(Mockito.any(OutputStream.class)))
                .thenReturn(outputStream);
    }

View Full Code Here

        assert writer.getLength() == 123L;


        mockSequenceFileReaderWriter(true);


        ReflectionUtil.createFileReaderWriter(mConfig.getFileReaderWriter(),
                mLogFilePathGz, new GzipCodec(), FileReaderWriter.Type.Writer);


        // Verify that the method has been called exactly once (the default).
        PowerMockito.verifyStatic();
        FileSystem.get(Mockito.any(Configuration.class));

View Full Code Here

    public void testGzipFileWriter() throws Exception {
        setupGzipFileReaderConfig();
        mockGzipFileReaderWriter();
        FileReaderWriter writer = (FileReaderWriter) ReflectionUtil
                .createFileReaderWriter(mConfig.getFileReaderWriter(),
                        mLogFilePathGz, new GzipCodec(), FileReaderWriter.Type.Writer);
        assert writer.getLength() == 0L;
    }

View Full Code Here


    public void testGzipFileReader() throws Exception {
        setupGzipFileReaderConfig();
        mockGzipFileReaderWriter();
        ReflectionUtil.createFileReaderWriter(mConfig.getFileReaderWriter(),
                mLogFilePathGz, new GzipCodec(), FileReaderWriter.Type.Reader);
    }

View Full Code Here

0 1 2 3 4 5 6

TOP

Related Classes of org.apache.hadoop.io.compress.GzipCodec$GzipOutputStream$ResetableGZIPOutputStream

cascading.tuple.hadoop.SpillableTupleHadoopTest

com.cloudera.flume.handlers.hdfs.CustomDfsSink

com.cloudera.flume.handlers.hdfs.TestEscapedCustomOutputDfs

com.cloudera.sqoop.io.TestSplittableBufferedWriter

com.cloudera.sqoop.TestCompression

com.fb2pdf.hadoop.cluster.SequenceFilesFromDirectory$ChunkedWriter

com.pinterest.secor.common.FileRegistryTest

com.pinterest.secor.io.FileReaderWriterTest

org.apache.flume.sink.hdfs.TestUseRawLocalFileSystem

org.apache.hadoop.hbase.io.hfile.TestHFilePerformance

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.