Examples of org.apache.hadoop.io.compress.CompressionCodec.createInputStream()

Class org.apache.hadoop.io.compress.CompressionCodec

Examples of org.apache.hadoop.io.compress.CompressionCodec.createInputStream()

org.apache.hadoop.io.compress.CompressionCodec.createInputStream()
Create a stream decompressor that will read from the given input stream. @param in the stream to read compressed bytes from @return a stream to read uncompressed bytes from

            {
              CompressionCodecFactory compressionCodecs =  new CompressionCodecFactory(job);
              final CompressionCodec codec = compressionCodecs.getCodec(file);
               if (codec != null) {
                  end = Long.MAX_VALUE;
                    CompressionInputStream stream = codec.createInputStream(fileIn);
                    this.xmlLoaderBPIS = new XMLLoaderBufferedPositionedInputStream(stream,start,end);
                  }
            }
            
            else

View Full Code Here

          CompressionCodec codec = codecFac.getCodecByClassName(codecClassName);
          if (codec == null) {
            throw new IOException("Image compression codec not supported: "
                + codecClassName);
          }
          in = new DataInputStream(codec.createInputStream(in));
        }
      }
      processINodes(in, v, numInodes, skipBlocks);


      processINodesUC(in, v, skipBlocks);

View Full Code Here

        // open the file and seek to the start of the split
        FileSystem fs = file.getFileSystem(conf);
        _fileInputStream = fs.open(file);
        _fileInputStream.seek(_start);
        if (codec != null) {
          _bufferedInputStream = new BufferedInputStream(codec.createInputStream(_fileInputStream));
        } else {
          _bufferedInputStream = new BufferedInputStream(_fileInputStream);
        }
        seekNextRecordBoundary();
    }

View Full Code Here

    // open the file and seek to the start of the split
    FileSystem fs = file.getFileSystem(job);
    FSDataInputStream fileIn = fs.open(split.getPath());
    boolean skipFirstLine = false;
    if (codec != null) {
      in = new LineReader(codec.createInputStream(fileIn), job);
      end = Long.MAX_VALUE;
    } else {
      if (start != 0) {
        skipFirstLine = true;
        --start;

View Full Code Here

    // we drop support for Hadoop 1.0.
    if (codec == null) {
      filestream.seek(start);
      filereader = new LineReader(filestream);
    } else {
      filereader = new LineReader(codec.createInputStream(filestream,
          codec.createDecompressor()), conf);
      fileseeker = filestream;
    }
    if (start != 0) {
      // always throw away first record because

View Full Code Here

    Class<? extends CompressionCodec> codecClass = conf.getClass(
        JobConstants.HADOOP_COMPRESS_CODEC, SqoopFileOutputFormat.DEFAULT_CODEC)
        .asSubclass(CompressionCodec.class);
    CompressionCodec codec = ReflectionUtils.newInstance(codecClass, conf);
    String fileName = outdir + "/" +  OUTPUT_FILE + codec.getDefaultExtension();
    InputStream filestream = codec.createInputStream(FileUtils.open(fileName));
    BufferedReader filereader = new BufferedReader(new InputStreamReader(
        filestream, Data.CHARSET_NAME));
    verifyOutputText(filereader);
  }

View Full Code Here

          // own detection methods, based on the provided path.
          CompressionCodecFactory cf = new CompressionCodecFactory(getConf());
          CompressionCodec codec = cf.getCodec(item.path);
          if (codec != null) {
            i.seek(0);
            return codec.createInputStream(i);
          }
          break;
        }
      }

View Full Code Here

          CompressionCodec codec = codecFac.getCodecByClassName(codecClassName);
          if (codec == null) {
            throw new IOException("Image compression codec not supported: "
                + codecClassName);
          }
          in = new DataInputStream(codec.createInputStream(in));
        }
      }
      processINodes(in, v, numInodes, skipBlocks);


      processINodesUC(in, v, skipBlocks);

View Full Code Here


    FileLineIterator it;
    if (dumpFilePath.endsWith(".bz2")) {
      // default compression format from http://download.wikimedia.org
      CompressionCodec codec = new BZip2Codec();
      it = new FileLineIterator(codec.createInputStream(new FileInputStream(dumpFile)));
    } else {
      // assume the user has previously de-compressed the dump file
      it = new FileLineIterator(dumpFile);
    }
    int fileNumber = 0;

View Full Code Here


      // open the file and seek to the start of the split
      FileSystem fs = file.getFileSystem(conf);
      FSDataInputStream fileIn = fs.open(split.getPath());
      if(codec != null) {
        in = new LineReader(codec.createInputStream(fileIn), conf);
        end = Long.MAX_VALUE;
      } else {
        if(start != 0) {
          // Skipping first line because we are not the first split, so start could not be
          // the start of the record

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.