Examples of CompressionCodec

com.facebook.hive.orc.compression.CompressionCodec
org.apache.abdera.i18n.text.io.CompressionUtil.CompressionCodec
org.apache.abdera.util.CompressionUtil.CompressionCodec
org.apache.hadoop.io.compress.CompressionCodec
This class encapsulates a streaming compression/decompression pair. @author Owen O'Malley
org.apache.oozie.compression.CompressionCodec

Examples of org.apache.hadoop.io.compress.CompressionCodec

    if (!isCompressed()) {
      holder.setStream(wout);
    } else {
      // TODO: will isCompressed() really guard for npe against getCodec()
      Class<?> clazz = ClassUtils.resolveClassName(getCodec().getCodecClass(), getClass().getClassLoader());
      CompressionCodec compressionCodec = (CompressionCodec) ReflectionUtils.newInstance(clazz,
          getConfiguration());
      OutputStream out = compressionCodec.createOutputStream(wout);
      holder.setWrappedStream(wout);
      holder.setStream(out);
    }
    return holder;
  }

View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionCodec

        input.seek(getSplit().getStart());
        holder.setStream(input);
      }
    } else {
      Class<?> clazz = ClassUtils.resolveClassName(getCodec().getCodecClass(), getClass().getClassLoader());
      CompressionCodec compressionCodec = (CompressionCodec) ReflectionUtils.newInstance(clazz,
          getConfiguration());
      Decompressor decompressor = CodecPool.getDecompressor(compressionCodec);
      if (getSplit() == null) {
        // with codec, no split
        FSDataInputStream winput = fs.open(p);
        InputStream input = compressionCodec.createInputStream(winput, decompressor);
        holder.setWrappedStream(winput);
        holder.setStream(input);
      } else {
        // with codec, with split
        long start = getSplit().getStart();

View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionCodec

    Configuration conf = new Configuration();
    List<Class<? extends CompressionCodec>> codecs = CompressionCodecFactory
        .getCodecClasses(conf);
    // Wish we could base this on DefaultCodec but appears not all codec's
    // extend DefaultCodec(Lzo)
    CompressionCodec codec = null;
    ArrayList<String> codecStrs = new ArrayList<String>();
    codecStrs.add("None");
    for (Class<? extends CompressionCodec> cls : codecs) {
      codecStrs.add(cls.getSimpleName());
      if (codecMatches(cls, codecName)) {

View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionCodec

  public InputStream getInputStream() throws IOException {
    if (exists) {
      InputStream stream = fs.open(path);


      if (codecsFactory != null) {
        CompressionCodec codec = codecsFactory.getCodec(path);
        if (codec != null) {
          // the pool is not used since the returned inputstream needs to be decorated
          // to return the decompressor on close which can mask the actual stream
          // it's also unclear whether the pool is actually useful or not
          // Decompressor decompressor = CodecPool.getDecompressor(codec);
          // stream = (decompressor != null ? codec.createInputStream(stream, decompressor) : codec.createInputStream(stream));
          stream = codec.createInputStream(stream);
        }
      }


      return stream;
    }

View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionCodec

            return new XMLFileRecordReader(recordIdentifier);
        }  
        
        @Override
        protected boolean isSplitable(JobContext context, Path filename) {
           CompressionCodec codec = 
              new CompressionCodecFactory(context.getConfiguration()).getCodec(filename);
           return (!(codec == null)) ? isSplitable : true;
        }

View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionCodec

               this.xmlLoaderBPIS = new XMLLoaderBufferedPositionedInputStream(in,start,end);
            }
            else if (file.toString().endsWith(".gz"))
            {
              CompressionCodecFactory compressionCodecs =  new CompressionCodecFactory(job);
              final CompressionCodec codec = compressionCodecs.getCodec(file);
               if (codec != null) {
                  end = Long.MAX_VALUE;
                    CompressionInputStream stream = codec.createInputStream(fileIn);
                    this.xmlLoaderBPIS = new XMLLoaderBufferedPositionedInputStream(stream,start,end);
                  }
            }
            
            else

View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionCodec


        FileSystem fs = file.getFileSystem(job);
        CompressionType compressionType = CompressionType.BLOCK;
        // find the right codec
        Class<?> codecClass = getOutputCompressorClass(job, DefaultCodec.class);
        CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, job);


        // set the schema metadata
        /* begin jays code */
        SequenceFile.Metadata meta = new SequenceFile.Metadata();
        meta.set(new Text("key.schema"), new Text(getSchema("reducer.output.key.schema", job)));

View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionCodec

        Integer.MAX_VALUE);
    start = split.getStart();
    end = start + split.getLength();
    final Path file = split.getPath();
    compressionCodecs = new CompressionCodecFactory(job);
    final CompressionCodec codec = compressionCodecs.getCodec(file);


    // open the file and seek to the start of the split
    FileSystem fs = file.getFileSystem(job);
    FSDataInputStream fileIn = fs.open(split.getPath());
    boolean skipFirstLine = false;
    if (codec != null) {
      in = new LineReader(codec.createInputStream(fileIn), job);
      end = Long.MAX_VALUE;
    } else {
      if (start != 0) {
        skipFirstLine = true;
        --start;

View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionCodec

  public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context)
      throws IOException {
    boolean isCompressed = getCompressOutput(context);
    Configuration conf = context.getConfiguration();
    String ext = "";
    CompressionCodec codec = null;


    if (isCompressed) {
      // create the named codec
      Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(
          context, GzipCodec.class);
      codec = ReflectionUtils.newInstance(codecClass, conf);


      ext = codec.getDefaultExtension();
    }


    Path file = getDefaultWorkFile(context, ext);
    FileSystem fs = file.getFileSystem(conf);
    FSDataOutputStream fileOut = fs.create(file, false);
    DataOutputStream ostream = fileOut;


    if (isCompressed) {
      ostream = new DataOutputStream(codec.createOutputStream(fileOut));
    }


    return new NoKeyRecordWriter<K, V>(ostream);
  }

View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionCodec

    f.login(usr, pwd);
    f.changeWorkingDirectory(dir);
    f.setFileType(FTP.BINARY_FILE_TYPE);


    boolean isCompressed = getCompressOutput(job);
    CompressionCodec codec = null;
    String extension = "";
    if (isCompressed) {
      Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(
          job, GzipCodec.class);
      codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass,
          conf);
      extension = codec.getDefaultExtension();
    }
    Path file = getDefaultWorkFile(job, extension);
    FileSystem fs = file.getFileSystem(conf);
    String filename = file.getName();
    if (!isCompressed) {
      // FSDataOutputStream fileOut = fs.create(file, false);
      OutputStream os = f.appendFileStream(filename);
      DataOutputStream fileOut = new DataOutputStream(os);
      return new FTPLineRecordWriter<K, V>(fileOut, new String(
          keyValueSeparator), f);


    } else {
      // FSDataOutputStream fileOut = fs.create(file, false);
      OutputStream os = f.appendFileStream(filename);
      DataOutputStream fileOut = new DataOutputStream(os);
      return new FTPLineRecordWriter<K, V>(new DataOutputStream(
          codec.createOutputStream(fileOut)), keyValueSeparator, f);
    }
  }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.