Examples of org.apache.hadoop.io.compress.CompressionCodec.createInputStream()

Class org.apache.hadoop.io.compress.CompressionCodec

Examples of org.apache.hadoop.io.compress.CompressionCodec.createInputStream()

org.apache.hadoop.io.compress.CompressionCodec.createInputStream()
Create a stream decompressor that will read from the given input stream. @param in the stream to read compressed bytes from @return a stream to read uncompressed bytes from


      // open the file and seek to the start of the split
      FileSystem fs = file.getFileSystem(conf);
      FSDataInputStream fileIn = fs.open(split.getPath());
      if(codec != null) {
        in = new LineReader(codec.createInputStream(fileIn), conf);
        end = Long.MAX_VALUE;
      } else {
        if(start != 0) {
          // Skipping first line because we are not the first split, so start could not be
          // the start of the record

View Full Code Here

    if (inputCodec == null) {
      decompressor = null;
      coreInputStream = fileIn;
    } else {
      decompressor = CodecPool.getDecompressor(inputCodec);
      coreInputStream = inputCodec.createInputStream(fileIn, decompressor);
    }
  }


  @Override
  public int read() throws IOException {

View Full Code Here


    // check codecs
    CompressionCodecFactory cf = new CompressionCodecFactory(getConf());
    CompressionCodec codec = cf.getCodec(p);
    if (codec != null) {
      return codec.createInputStream(i);
    }


    switch(i.readShort()) {
      case 0x1f8b: // RFC 1952
        i.seek(0);

View Full Code Here

        reporter.setStatus("Opened " + filePath);
        // get a compression filter if specified
        if(compressionClass != null) {
          CompressionCodec codec = (CompressionCodec)
            ReflectionUtils.newInstance(compressionClass, new Configuration());
          in = codec.createInputStream(stm);
          LOG.info("Codec created " + filePath);
          reporter.setStatus("Codec created " + filePath);
        }
        BufferedReader reader = new BufferedReader(new InputStreamReader(in));
        LOG.info("Reader created " + filePath);

View Full Code Here

          CompressionCodec codec = codecFac.getCodecByClassName(codecClassName);
          if (codec == null) {
            throw new IOException("Image compression codec not supported: "
                + codecClassName);
          }
          in = new DataInputStream(codec.createInputStream(in));
        }
      }
      processINodes(in, v, numInodes, skipBlocks);


      processINodesUC(in, v, skipBlocks);

View Full Code Here

    File dumpFile = new File(dumpFilePath);
    FileLineIterator it;
    if (dumpFilePath.endsWith(".bz2")) {
      // default compression format from http://download.wikimedia.org
      CompressionCodec codec = new BZip2Codec();
      it = new FileLineIterator(codec.createInputStream(new FileInputStream(dumpFile)));
    } else {
      // assume the user has previously de-compressed the dump file
      it = new FileLineIterator(dumpFile);
    }
    int filenumber = 0;

View Full Code Here


      // open the file and seek to the start of the split
      FileSystem fs = file.getFileSystem(conf);
      FSDataInputStream fileIn = fs.open(split.getPath());
      if(codec != null) {
        in = new LineReader(codec.createInputStream(fileIn), conf);
        end = Long.MAX_VALUE;
      } else {
        if(start != 0) {
          // Skipping first line because we are not the first split, so start could not be
          // the start of the record

View Full Code Here

              FSDataInputStream compressedStream =
                  getFs().open(path);


              DataInputStream stream = codec == null ? compressedStream :
                  new DataInputStream(
                      codec.createInputStream(compressedStream));


              Partition<I, V, E> partition =
                  getConfiguration().createPartition(partitionId, getContext());


              partition.readFields(stream);

View Full Code Here

    CompressionCodec codec = factory.getCodec(p);
    InputStream in = srcFs.open(p);
    if (codec == null) {
      throw new IOException("Cannot find codec for " + p);
    }
    return codec.createInputStream(in);
  }


  void decompress(String srcf) throws IOException {
    Path srcPattern = new Path(srcf);
    new DelayedExceptionThrowing() {

View Full Code Here


    Class<?> codecClass = Class.forName(args[1]);
    CompressionCodec codec = (CompressionCodec)
        ReflectionUtils.newInstance(codecClass, config);


    InputStream cis = codec.createInputStream(is);


    IOUtils.copyBytes(cis, System.out, config, true);


    IOUtils.closeStream(is);
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.