Examples of org.apache.hadoop.hbase.io.encoding.DataBlockEncoder

Package org.apache.hadoop.hbase.io.encoding

Examples of org.apache.hadoop.hbase.io.encoding.DataBlockEncoder

org.apache.hadoop.hbase.io.encoding.DataBlockEncoder
Encoding of KeyValue. It aims to be fast and efficient using assumptions:
- the KeyValues are stored sorted by key
- we know the structure of KeyValue
- the values are always iterated forward from beginning of block
- knowledge of Key Value format
It is designed to work fast enough to be feasible as in memory compression.


    for (DataBlockEncoding encoding : encodings) {
      if (encoding == DataBlockEncoding.NONE) {
        continue;
      }
      DataBlockEncoder d = encoding.getEncoder();
      codecs.add(new EncodedDataBlock(d, includesMemstoreTS, encoding, rawKVs));
    }
  }

View Full Code Here

  private ByteBuffer encodeBufferToHFileBlockBuffer(ByteBuffer in,
      DataBlockEncoding algo, boolean includesMemstoreTS,
      byte[] dummyHeader) {
    ByteArrayOutputStream encodedStream = new ByteArrayOutputStream();
    DataOutputStream dataOut = new DataOutputStream(encodedStream);
    DataBlockEncoder encoder = algo.getEncoder();
    try {
      encodedStream.write(dummyHeader);
      algo.writeIdInBytes(dataOut);
      encoder.compressKeyValues(dataOut, in,
          includesMemstoreTS);
    } catch (IOException e) {
      throw new RuntimeException(String.format("Bug in data block encoder " +
          "'%s', it probably requested too much data", algo.toString()), e);
    }

View Full Code Here

        new DoubleOutputStream(dos, baos);
    writeTestKeyValues(doubleOutputStream, blockId, includesMemstoreTS, useTag);
    ByteBuffer rawBuf = ByteBuffer.wrap(baos.toByteArray());
    rawBuf.rewind();


    DataBlockEncoder encoder = encoding.getEncoder();
    int headerLen = dummyHeader.length;
    byte[] encodedResultWithHeader = null;
    HFileContext meta = new HFileContextBuilder()
                        .withCompression(algo)
                        .withIncludesMvcc(includesMemstoreTS)
                        .withIncludesTags(useTag)
                        .build();
    if (encoder != null) {
      HFileBlockEncodingContext encodingCtx = encoder.newDataBlockEncodingContext(encoding,
          dummyHeader, meta);
      encoder.encodeKeyValues(rawBuf, encodingCtx);
      encodedResultWithHeader =
          encodingCtx.getUncompressedBytesWithHeader();
    } else {
      HFileBlockDefaultEncodingContext defaultEncodingCtx = new HFileBlockDefaultEncodingContext(
          encoding, dummyHeader, meta);

View Full Code Here

    boolean useTag = (currentKV.getTagsLengthUnsigned() > 0);
    for (DataBlockEncoding encoding : encodings) {
      if (encoding == DataBlockEncoding.NONE) {
        continue;
      }
      DataBlockEncoder d = encoding.getEncoder();
      HFileContext meta = new HFileContextBuilder()
                          .withCompression(Compression.Algorithm.NONE)
                          .withIncludesMvcc(includesMemstoreTS)
                          .withIncludesTags(useTag).build();
      codecs.add(new EncodedDataBlock(d, encoding, rawKVs, meta ));

View Full Code Here

  private ByteBuffer encodeBufferToHFileBlockBuffer(ByteBuffer in,
      DataBlockEncoding algo, boolean includesMemstoreTS,
      byte[] dummyHeader) {
    ByteArrayOutputStream encodedStream = new ByteArrayOutputStream();
    DataOutputStream dataOut = new DataOutputStream(encodedStream);
    DataBlockEncoder encoder = algo.getEncoder();
    try {
      encodedStream.write(dummyHeader);
      algo.writeIdInBytes(dataOut);
      encoder.compressKeyValues(dataOut, in,
          includesMemstoreTS);
    } catch (IOException e) {
      throw new RuntimeException(String.format("Bug in data block encoder " +
          "'%s', it probably requested too much data", algo.toString()), e);
    }

View Full Code Here


      short dataBlockEncoderId = block.getDataBlockEncodingId();
      if (dataBlockEncoder == null ||
          !DataBlockEncoding.isCorrectEncoder(dataBlockEncoder,
              dataBlockEncoderId)) {
        DataBlockEncoder encoder =
            DataBlockEncoding.getDataBlockEncoderById(dataBlockEncoderId);
        setDataBlockEncoder(encoder);
      }


      seeker.setCurrentBuffer(getEncodedBuffer(newBlock));

View Full Code Here

  }


  @Override
  public HFileBlockEncodingContext newDataBlockEncodingContext(
      byte[] dummyHeader, HFileContext fileContext) {
    DataBlockEncoder encoder = encoding.getEncoder();
    if (encoder != null) {
      return encoder.newDataBlockEncodingContext(encoding, dummyHeader, fileContext);
    }
    return new HFileBlockDefaultEncodingContext(null, dummyHeader, fileContext);
  }

View Full Code Here

    return new HFileBlockDefaultEncodingContext(null, dummyHeader, fileContext);
  }


  @Override
  public HFileBlockDecodingContext newDataBlockDecodingContext(HFileContext fileContext) {
    DataBlockEncoder encoder = encoding.getEncoder();
    if (encoder != null) {
      return encoder.newDataBlockDecodingContext(fileContext);
    }
    return new HFileBlockDefaultDecodingContext(fileContext);
  }

View Full Code Here


      short dataBlockEncoderId = block.getDataBlockEncodingId();
      if (dataBlockEncoder == null ||
          !DataBlockEncoding.isCorrectEncoder(dataBlockEncoder,
              dataBlockEncoderId)) {
        DataBlockEncoder encoder =
            DataBlockEncoding.getDataBlockEncoderById(dataBlockEncoderId);
        setDataBlockEncoder(encoder);
      }


      seeker.setCurrentBuffer(getEncodedBuffer(newBlock));

View Full Code Here

        new DoubleOutputStream(dos, baos);
    writeTestKeyValues(doubleOutputStream, blockId, includesMemstoreTS, useTag);
    ByteBuffer rawBuf = ByteBuffer.wrap(baos.toByteArray());
    rawBuf.rewind();


    DataBlockEncoder encoder = encoding.getEncoder();
    int headerLen = dummyHeader.length;
    byte[] encodedResultWithHeader = null;
    HFileContext meta = new HFileContextBuilder()
                        .withCompression(algo)
                        .withIncludesMvcc(includesMemstoreTS)
                        .withIncludesTags(useTag)
                        .build();
    if (encoder != null) {
      HFileBlockEncodingContext encodingCtx = encoder.newDataBlockEncodingContext(encoding,
          dummyHeader, meta);
      encoder.encodeKeyValues(rawBuf, encodingCtx);
      encodedResultWithHeader =
          encodingCtx.getUncompressedBytesWithHeader();
    } else {
      HFileBlockDefaultEncodingContext defaultEncodingCtx = new HFileBlockDefaultEncodingContext(
          encoding, dummyHeader, meta);

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.hadoop.hbase.io.encoding.DataBlockEncoder

org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl

org.apache.hadoop.hbase.io.hfile.HFileReaderV2$EncodedScannerV2

org.apache.hadoop.hbase.io.hfile.TestHFileBlock

org.apache.hadoop.hbase.regionserver.DataBlockEncodingTool

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.