Examples of org.apache.hadoop.hbase.io.FSDataInputStreamWrapper

org.apache.hadoop.hbase.io.FSDataInputStreamWrapper
Wrapper for input stream(s) that takes care of the interaction of FS and HBase checksums, as well as closing streams. Initialization is not thread-safe, but normal operation is; see method comments.

          totalSize += hbw.getOnDiskSizeWithHeader();
        }
        os.close();


        FSDataInputStream is = fs.open(path);
        HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(new FSDataInputStreamWrapper(is),
            algo, totalSize, MINOR_VERSION, fs, path);
        HFileBlock b = hbr.readBlockData(0, -1, -1, pread);
        is.close();


        b.sanityCheck();
        assertEquals(4936, b.getUncompressedSizeWithoutHeader());
        assertEquals(algo == GZ ? 2173 : 4936, 
                     b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes());
        String blockStr = b.toString();


        if (algo == GZ) {
          is = fs.open(path);
          hbr = new HFileBlock.FSReaderV2(new FSDataInputStreamWrapper(is),
              algo, totalSize, MINOR_VERSION, fs, path);
          b = hbr.readBlockData(0, 2173 + HConstants.HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM +
                                b.totalChecksumBytes(), -1, pread);
          assertEquals(blockStr, b.toString());
          int wrongCompressedSize = 2172;

View Full Code Here

            totalSize += hbw.getOnDiskSizeWithHeader();
          }
          os.close();


          FSDataInputStream is = fs.open(path);
          HFileBlock.FSReaderV2 hbr = new HFileBlock.FSReaderV2(new FSDataInputStreamWrapper(is),
              algo, totalSize, MINOR_VERSION, fs, path);
          hbr.setDataBlockEncoder(dataBlockEncoder);
          hbr.setIncludesMemstoreTS(includesMemstoreTS);


          HFileBlock b;

View Full Code Here

   * @param dataBlockEncoding data block encoding algorithm.
   * @return The StoreFile.Reader for the file
   */
  public StoreFile.Reader open(final FileSystem fs, final CacheConfig cacheConf,
      final DataBlockEncoding dataBlockEncoding) throws IOException {
    FSDataInputStreamWrapper in;
    FileStatus status;


    if (this.link != null) {
      // HFileLink
      in = new FSDataInputStreamWrapper(fs, this.link);
      status = this.link.getFileStatus(fs);
    } else if (this.reference != null) {
      // HFile Reference
      Path referencePath = getReferredToFile(this.getPath());
      in = new FSDataInputStreamWrapper(fs, referencePath);
      status = fs.getFileStatus(referencePath);
    } else {
      in = new FSDataInputStreamWrapper(fs, this.getPath());
      status = fileStatus;
    }
    long length = status.getLen();
    if (this.reference != null) {
      hdfsBlocksDistribution = computeRefFileHDFSBlockDistribution(fs, reference, status);

View Full Code Here

   * @throws IOException Will throw a CorruptHFileException (DoNotRetryIOException subtype) if hfile is corrupt/invalid.
   */
  public static Reader createReader(
      FileSystem fs, Path path, CacheConfig cacheConf, Configuration conf) throws IOException {
    Preconditions.checkNotNull(cacheConf, "Cannot create Reader with null CacheConf");
    FSDataInputStreamWrapper stream = new FSDataInputStreamWrapper(fs, path);
    return pickReaderVersion(path, stream, fs.getFileStatus(path).getLen(),
      cacheConf, stream.getHfs(), conf);
  }

View Full Code Here

   * This factory method is used only by unit tests
   */
  static Reader createReaderFromStream(Path path,
      FSDataInputStream fsdis, long size, CacheConfig cacheConf, Configuration conf)
      throws IOException {
    FSDataInputStreamWrapper wrapper = new FSDataInputStreamWrapper(fsdis);
    return pickReaderVersion(path, wrapper, size, cacheConf, null, conf);
  }

View Full Code Here

   * @throws IOException Will throw a CorruptHFileException (DoNotRetryIOException subtype) if hfile is corrupt/invalid.
   */
  public static Reader createReader(
      FileSystem fs, Path path, CacheConfig cacheConf, Configuration conf) throws IOException {
    Preconditions.checkNotNull(cacheConf, "Cannot create Reader with null CacheConf");
    FSDataInputStreamWrapper stream = new FSDataInputStreamWrapper(fs, path);
    return pickReaderVersion(path, stream, fs.getFileStatus(path).getLen(),
      cacheConf, stream.getHfs(), conf);
  }

View Full Code Here

   * This factory method is used only by unit tests
   */
  static Reader createReaderFromStream(Path path,
      FSDataInputStream fsdis, long size, CacheConfig cacheConf, Configuration conf)
      throws IOException {
    FSDataInputStreamWrapper wrapper = new FSDataInputStreamWrapper(fsdis);
    return pickReaderVersion(path, wrapper, size, cacheConf, null, conf);
  }

View Full Code Here

   * @param cacheConf The cache configuration and block cache reference.
   * @return The StoreFile.Reader for the file
   */
  public StoreFile.Reader open(final FileSystem fs,
      final CacheConfig cacheConf) throws IOException {
    FSDataInputStreamWrapper in;
    FileStatus status;


    if (this.link != null) {
      // HFileLink
      in = new FSDataInputStreamWrapper(fs, this.link);
      status = this.link.getFileStatus(fs);
    } else if (this.reference != null) {
      // HFile Reference
      Path referencePath = getReferredToFile(this.getPath());
      in = new FSDataInputStreamWrapper(fs, referencePath);
      status = fs.getFileStatus(referencePath);
    } else {
      in = new FSDataInputStreamWrapper(fs, this.getPath());
      status = fileStatus;
    }
    long length = status.getLen();
    hdfsBlocksDistribution = computeHDFSBlocksDistribution(fs);

View Full Code Here

      System.out.println("Block Headers:");
      /*
       * TODO: this same/similar block iteration logic is used in HFileBlock#blockRange and
       * TestLazyDataBlockDecompression. Refactor?
       */
      FSDataInputStreamWrapper fsdis = new FSDataInputStreamWrapper(fs, file);
      long fileSize = fs.getFileStatus(file).getLen();
      FixedFileTrailer trailer =
        FixedFileTrailer.readFromStream(fsdis.getStream(false), fileSize);
      long offset = trailer.getFirstDataBlockOffset(),
        max = trailer.getLastDataBlockOffset();
      HFileBlock block;
      while (offset <= max) {
        block = reader.readBlock(offset, -1, /* cacheBlock */ false, /* pread */ false,

View Full Code Here

    /**
     * A constructor that reads files with the latest minor version.
     * This is used by unit tests only.
     */
    FSReaderV2(FSDataInputStream istream, long fileSize, HFileContext fileContext) throws IOException {
      this(new FSDataInputStreamWrapper(istream), fileSize, null, null, fileContext);
    }

View Full Code Here

0 1 2 3 4 5

TOP

Related Classes of org.apache.hadoop.hbase.io.FSDataInputStreamWrapper

org.apache.hadoop.fs.FileSystem

org.apache.hadoop.hbase.fs.HFileSystem

org.apache.hadoop.hbase.io.hfile.HFile

org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl

org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderV2

org.apache.hadoop.hbase.io.hfile.HFilePrettyPrinter

org.apache.hadoop.hbase.io.hfile.TestChecksum

org.apache.hadoop.hbase.io.hfile.TestHFileBlockCompatibility

org.apache.hadoop.hbase.io.hfile.TestLazyDataBlockDecompression

org.apache.hadoop.hbase.regionserver.StoreFileInfo

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.