Examples of org.apache.hadoop.io.compress.CompressionCodec.createInputStream()

Class org.apache.hadoop.io.compress.CompressionCodec

Examples of org.apache.hadoop.io.compress.CompressionCodec.createInputStream()

org.apache.hadoop.io.compress.CompressionCodec.createInputStream()
Create a stream decompressor that will read from the given input stream. @param in the stream to read compressed bytes from @return a stream to read uncompressed bytes from

      
      // check codecs
      CompressionCodecFactory cf = new CompressionCodecFactory(getConf());
      CompressionCodec codec = cf.getCodec(item.path);
      if (codec != null) {
        return codec.createInputStream(i);
      }


      switch(i.readShort()) {
        case 0x1f8b: { // RFC 1952
          i.seek(0);

View Full Code Here

        reporter.setStatus("Opened " + filePath);
        // get a compression filter if specified
        if(compressionClass != null) {
          CompressionCodec codec = (CompressionCodec)
            ReflectionUtils.newInstance(compressionClass, new Configuration());
          in = codec.createInputStream(stm);
          LOG.info("Codec created " + filePath);
          reporter.setStatus("Codec created " + filePath);
        }
        BufferedReader reader = new BufferedReader(new InputStreamReader(in));
        LOG.info("Reader created " + filePath);

View Full Code Here

    File dumpFile = new File(dumpFilePath);
    FileLineIterator it;
    if (dumpFilePath.endsWith(".bz2")) {
      // default compression format from http://download.wikimedia.org
      CompressionCodec codec = new BZip2Codec();
      it = new FileLineIterator(codec.createInputStream(new FileInputStream(dumpFile)));
    } else {
      // assume the user has previously de-compressed the dump file
      it = new FileLineIterator(dumpFile);
    }
    int filenumber = 0;

View Full Code Here

      CompressionCodec codec = compressionCodecs.getCodec(file);
      if (codec != null) {
        Decompressor decompressor = CodecPool.getDecompressor(codec);
        if (decompressor != null) {
          CompressionInputStream in = 
            codec.createInputStream(fs.open(file), decompressor);
          //TODO Seek doesnt work with compressed input stream. 
          //     Use SplittableCompressionCodec?
          return (InputStream)in;
        }
      }

View Full Code Here

    // open the file and seek to the start of the split
    FileSystem fs = file.getFileSystem(job);
    FSDataInputStream fileIn = fs.open(split.getPath());
    boolean skipFirstLine = false;
    if (codec != null) {
      in = new LineReader(codec.createInputStream(fileIn), job);
      end = Long.MAX_VALUE;
    } else {
      if (start != 0) {
        skipFirstLine = true;
        --start;

View Full Code Here

          CompressionCodec codec = codecFac.getCodecByClassName(codecClassName);
          if (codec == null) {
            throw new IOException("Image compression codec not supported: "
                + codecClassName);
          }
          in = new DataInputStream(codec.createInputStream(in));
        }
      }
      processINodes(in, v, numInodes, skipBlocks);


      processINodesUC(in, v, skipBlocks);

View Full Code Here

      final CompressionCodec codec = compressionCodecs.getCodec(path);
      this.conf = conf;


      fsin = fileSys.open(path);
      if (codec != null) {
        dcin = codec.createInputStream(fsin);
        in = new DataInputStream(dcin);
      } else {
        dcin = null;
        in = fsin;
      }

View Full Code Here

            else if (type.equals("gz")) {
               codec = new GzipCodec();
               ((GzipCodec)codec).setConf(new Configuration());
            }


            CompressionInputStream createInputStream = codec
                  .createInputStream(new FileInputStream(file));
            int b;
            StringBuffer sb = new StringBuffer();
            while ((b = createInputStream.read()) != -1) {
               sb.append((char) b);

View Full Code Here

        if(codec == null) {
            return new BufferedReader(new FileReader(file));
        } else {
            Decompressor decompressor = CodecPool.getDecompressor(codec);
            FileInputStream fis = new FileInputStream(file);
            CompressionInputStream cis = codec.createInputStream(fis, decompressor);
            BufferedReader br = new BufferedReaderExt(new InputStreamReader(cis), decompressor);
            return br;
        }
    }

View Full Code Here

          CompressionCodec codec = codecFac.getCodecByClassName(codecClassName);
          if (codec == null) {
            throw new IOException("Image compression codec not supported: "
                + codecClassName);
          }
          in = new DataInputStream(codec.createInputStream(in));
        }
      }
      processINodes(in, v, numInodes, skipBlocks);


      processINodesUC(in, v, skipBlocks);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.