Examples of org.apache.hadoop.io.compress.CompressionCodec

org.apache.hadoop.io.compress.CompressionCodec
This class encapsulates a streaming compression/decompression pair. @author Owen O'Malley

    NumberFormat decimalFormatter = new DecimalFormat("0000");
    File dumpFile = new File(dumpFilePath);
    FileLineIterator it;
    if (dumpFilePath.endsWith(".bz2")) {
      // default compression format from http://download.wikimedia.org
      CompressionCodec codec = new BZip2Codec();
      it = new FileLineIterator(codec.createInputStream(new FileInputStream(dumpFile)));
    } else {
      // assume the user has previously de-compressed the dump file
      it = new FileLineIterator(dumpFile);
    }
    int filenumber = 0;

View Full Code Here

    public SeqFileAppendable(FileSystem fs, Path path, int osBufferSize,
        String compress, int minBlkSize) throws IOException {
      Configuration conf = new Configuration();
      conf.setBoolean("hadoop.native.lib", true);


      CompressionCodec codec = null;
      if ("lzo".equals(compress)) {
        codec = Compression.Algorithm.LZO.getCodec();
      }
      else if ("gz".equals(compress)) {
        codec = Compression.Algorithm.GZ.getCodec();

View Full Code Here

          totalBytesWritten += key.length;
          totalBytesWritten += value.length;
         }
        writer.close();
    } else if ("SequenceFile".equals(fileType)){
        CompressionCodec codec = null;
        if ("gz".equals(codecName))
          codec = new GzipCodec();
        else if (!"none".equals(codecName))
          throw new IOException("Codec not supported.");

View Full Code Here

                                      String name, Progressable progress)
    throws IOException {


    Path file = new Path(job.getOutputPath(), name);
    FileSystem fs = file.getFileSystem(job);
    CompressionCodec codec = null;
    CompressionType compressionType = CompressionType.NONE;
    if (getCompressOutput(job)) {
      // find the kind of compression to do
      compressionType = SequenceFile.getCompressionType(job);

View Full Code Here

                                      Progressable prog) throws IOException {
      FileSystem srcFileSys = inputFile.getFileSystem(conf);
      Reader reader = new Reader(srcFileSys, inputFile, 4096, conf);
      boolean compress = reader.isCompressed();
      boolean blockCompress = reader.isBlockCompressed();
      CompressionCodec codec = reader.getCompressionCodec();
      reader.close();
      
      FileSystem dstFileSys = outputFile.getFileSystem(conf);
      FSDataOutputStream out;
      if (prog != null)

View Full Code Here

        int segments = 0;
        int currentFile = 0;
        boolean atEof = (currentFile >= inFiles.length);
        boolean isCompressed = false;
        boolean isBlockCompressed = false;
        CompressionCodec codec = null;
        segmentLengths.clear();
        if (atEof) {
          return 0;
        }

View Full Code Here

  }
  
  public void testLzoSequenceFile() throws Exception {
    if (LzoCodec.isNativeLzoLoaded()) {
      LOG.info("Testing SequenceFile with LzoCodec");
      CompressionCodec lzoCodec = null;
      try {
        lzoCodec = (CompressionCodec) ReflectionUtils.newInstance(
                                                                  conf.getClassByName(LzoCodec.class.getName()), conf);
      } catch (ClassNotFoundException cnfe) {
        throw new IOException("Cannot find LzoCodec!");

View Full Code Here

  public void testSequenceFileMetadata() throws Exception {
    LOG.info("Testing SequenceFile with metadata");
    int count = 1024 * 10;
    int megabytes = 1;
    int factor = 5;
    CompressionCodec codec = new DefaultCodec();
    Path file = new Path(System.getProperty("test.build.data",".")+"/test.seq.metadata");
    Path recordCompressedFile = 
      new Path(System.getProperty("test.build.data",".")+"/test.rc.seq.metadata");
    Path blockCompressedFile = 
      new Path(System.getProperty("test.build.data",".")+"/test.bc.seq.metadata");

View Full Code Here

        System.exit(-1);
      }


      CompressionType compressionType = 
        CompressionType.valueOf(compressType);
      CompressionCodec codec = (CompressionCodec)ReflectionUtils.newInstance(
                                                                             conf.getClassByName(compressionCodec), 
                                                                             conf);


      if (rwonly || (create && !merge)) {
        writeTest(fs, count, seed, file, compressionType, codec);

View Full Code Here

    throws IOException {
    long start = split.getStart();
    long end = start + split.getLength();
    final Path file = split.getPath();
    compressionCodecs = new CompressionCodecFactory(job);
    final CompressionCodec codec = compressionCodecs.getCodec(file);


    // open the file and seek to the start of the split
    FileSystem fs = FileSystem.get(job);
    FSDataInputStream fileIn = fs.open(split.getPath());
    InputStream in = fileIn;
    if (codec != null) {
      in = codec.createInputStream(fileIn);
      end = Long.MAX_VALUE;
    } else if (start != 0) {
      fileIn.seek(start - 1);
      LineRecordReader.readLine(fileIn, null);
      start = fileIn.getPos();

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.io.compress.CompressionCodec

com.cloudera.flume.handlers.hdfs.CustomDfsSink

com.cloudera.sqoop.TestExport

com.datasalt.pangool.tuplemr.mapred.lib.input.TupleTextInputFormat

com.datasalt.pangool.tuplemr.mapred.lib.input.TupleTextInputFormat$TupleTextInputReader

com.datasalt.pangool.tuplemr.mapred.lib.output.TupleOutputFormat

com.facebook.presto.hive.AbstractTestHiveFileFormats

com.hadoop.compression.lzo.LzoIndex

com.hadoop.compression.lzo.TestLzoCodec

com.hadoop.mapreduce.LzoLineRecordReader

com.hadoop.mapreduce.LzoSplitRecordReader

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.