Package org.apache.hadoop.hbase.io.encoding

Examples of org.apache.hadoop.hbase.io.encoding.DataBlockEncoding

184
185
186
187
188
189
190
191
192
193
194
  private void createTable() throws Exception {
    deleteTable();
    LOG.info("Creating table");
    Configuration conf = util.getConfiguration();
    String encodingKey = String.format(ENCODING_KEY, this.getClass().getSimpleName());
    DataBlockEncoding blockEncoding = DataBlockEncoding.valueOf(conf.get(encodingKey, "FAST_DIFF"));
    HTableDescriptor htd = new HTableDescriptor(TABLE_NAME);
    for (byte[] cf : dataGen.getColumnFamilies()) {
      HColumnDescriptor hcd = new HColumnDescriptor(cf);
      hcd.setDataBlockEncoding(blockEncoding);
      htd.addFamily(hcd);
View Full Code Here
233
234
235
236
237
238
239
240
241
242
243
    long offset = 0;
    HFileBlock prevBlock = null;
    EnumMap<BlockType, Integer> blockCountByType =
        new EnumMap<BlockType, Integer>(BlockType.class);

    DataBlockEncoding encodingInCache =
        encoderType.getEncoder().getDataBlockEncoding();
    while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) {
      long onDiskSize = -1;
      if (prevBlock != null) {
         onDiskSize = prevBlock.getNextBlockOnDiskSizeWithHeader();
View Full Code Here
925
926
927
928
929
930
931
932
933
934
935
  public Map<DataBlockEncoding, Integer> getEncodingCountsForTest() {
    Map<DataBlockEncoding, Integer> counts =
        new EnumMap<DataBlockEncoding, Integer>(DataBlockEncoding.class);
    for (BlockCacheKey cacheKey : map.keySet()) {
      DataBlockEncoding encoding = cacheKey.getDataBlockEncoding();
      Integer count = counts.get(encoding);
      counts.put(encoding, (count == null ? 0 : count) + 1);
    }
    return counts;
  }
View Full Code Here
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
    byte[] dataBlockEncodingType = fileInfo.get(DATA_BLOCK_ENCODING);
    if (dataBlockEncodingType == null && !hasPreferredCacheEncoding) {
      return NoOpDataBlockEncoder.INSTANCE;
    }

    DataBlockEncoding onDisk;
    if (dataBlockEncodingType == null) {
      onDisk = DataBlockEncoding.NONE;
    }else {
      String dataBlockEncodingStr = Bytes.toString(dataBlockEncodingType);
      try {
        onDisk = DataBlockEncoding.valueOf(dataBlockEncodingStr);
      } catch (IllegalArgumentException ex) {
        throw new IOException("Invalid data block encoding type in file info: "
            + dataBlockEncodingStr, ex);
      }
    }

    DataBlockEncoding inCache;
    if (onDisk == DataBlockEncoding.NONE) {
      // This is an "in-cache-only" encoding or fully-unencoded scenario.
      // Either way, we use the given encoding (possibly NONE) specified by
      // the column family in cache.
      inCache = preferredEncodingInCache;
View Full Code Here
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
   */
  public void testDataBlockEncodingMetaData() throws IOException {
    Path dir = new Path(new Path(this.testDir, "regionname"), "familyname");
    Path path = new Path(dir, "1234567890");

    DataBlockEncoding dataBlockEncoderAlgo =
        DataBlockEncoding.FAST_DIFF;
    HFileDataBlockEncoder dataBlockEncoder =
        new HFileDataBlockEncoderImpl(
            dataBlockEncoderAlgo,
            dataBlockEncoderAlgo);
    cacheConf = new CacheConfig(conf);
    StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, fs,
        HFile.DEFAULT_BLOCKSIZE)
            .withFilePath(path)
            .withDataBlockEncoder(dataBlockEncoder)
            .withMaxKeyCount(2000)
            .withChecksumType(CKTYPE)
            .withBytesPerChecksum(CKBYTES)
            .build();
    writer.close();
   
    StoreFile storeFile = new StoreFile(fs, writer.getPath(), conf,
        cacheConf, BloomType.NONE, dataBlockEncoder);
    StoreFile.Reader reader = storeFile.createReader();
   
    Map<byte[], byte[]> fileInfo = reader.loadFileInfo();
    byte[] value = fileInfo.get(HFileDataBlockEncoder.DATA_BLOCK_ENCODING);

    assertEquals(dataBlockEncoderAlgo.getNameInBytes(), value);
  }
View Full Code Here
773
774
775
776
777
778
779
780
781
782
783
  public Map<DataBlockEncoding, Integer> getEncodingCountsForTest() {
    Map<DataBlockEncoding, Integer> counts =
        new EnumMap<DataBlockEncoding, Integer>(DataBlockEncoding.class);
    for (BlockCacheKey cacheKey : map.keySet()) {
      DataBlockEncoding encoding = cacheKey.getDataBlockEncoding();
      Integer count = counts.get(encoding);
      counts.put(encoding, (count == null ? 0 : count) + 1);
    }
    return counts;
  }
View Full Code Here
225
226
227
228
229
230
231
232
233
234
235
    long offset = 0;
    HFileBlock prevBlock = null;
    EnumMap<BlockType, Integer> blockCountByType =
        new EnumMap<BlockType, Integer>(BlockType.class);

    DataBlockEncoding encodingInCache =
        encoderType.getEncoder().getEncodingInCache();
    while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) {
      long onDiskSize = -1;
      if (prevBlock != null) {
         onDiskSize = prevBlock.getNextBlockOnDiskSizeWithHeader();
View Full Code Here
176
177
178
179
180
181
182
183
184
185
186
187
        new HashMap<Store, HFileDataBlockEncoder>();
    for (Entry<byte[], Store> pair : r.getStores().entrySet()) {
      Store store = pair.getValue();
      HFileDataBlockEncoder blockEncoder = store.getDataBlockEncoder();
      replaceBlockCache.put(pair.getValue(), blockEncoder);
      final DataBlockEncoding inCache = DataBlockEncoding.PREFIX;
      final DataBlockEncoding onDisk = inCacheOnly ? DataBlockEncoding.NONE :
          inCache;
      store.setDataBlockEncoderInTest(new HFileDataBlockEncoderImpl(
          onDisk, inCache));
    }
View Full Code Here
245
246
247
248
249
250
251
252
253
254
255
    long offset = 0;
    HFileBlock prevBlock = null;
    EnumMap<BlockType, Integer> blockCountByType =
        new EnumMap<BlockType, Integer>(BlockType.class);

    DataBlockEncoding encodingInCache =
        encoderType.getEncoder().getDataBlockEncoding();
    while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) {
      long onDiskSize = -1;
      if (prevBlock != null) {
         onDiskSize = prevBlock.getNextBlockOnDiskSizeWithHeader();
View Full Code Here
113
114
115
116
117
118
119
120
121
122
123
      KVComparator comparator, HFileContext fileContext) {
    this.outputStream = outputStream;
    this.path = path;
    this.name = path != null ? path.getName() : outputStream.toString();
    this.hFileContext = fileContext;
    DataBlockEncoding encoding = hFileContext.getDataBlockEncoding();
    if (encoding != DataBlockEncoding.NONE) {
      this.blockEncoder = new HFileDataBlockEncoderImpl(encoding);
    } else {
      this.blockEncoder = NoOpDataBlockEncoder.INSTANCE;
    }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hbase.io.encoding.DataBlockEncoding

  • org.apache.crunch.io.hbase.HFileTargetIT
  • org.apache.hadoop.hbase.IntegrationTestLazyCfLoading
  • org.apache.hadoop.hbase.io.hfile.AbstractHFileWriter
  • org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl
  • org.apache.hadoop.hbase.io.hfile.HFileReaderV2
  • org.apache.hadoop.hbase.io.hfile.HFileReaderV2$EncodedScannerV2
  • org.apache.hadoop.hbase.io.hfile.LruBlockCache
  • org.apache.hadoop.hbase.io.hfile.TestCacheOnWrite
  • org.apache.hadoop.hbase.mapreduce.HFileOutputFormat
  • org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.