Examples of org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$WriterThread

Package org.apache.hadoop.hbase.io.hfile.bucket

Examples of org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$WriterThread

org.apache.hadoop.hbase.io.hfile.bucket.BucketCache
BucketCache uses {@link BucketAllocator} to allocate/free block, and use{@link BucketCache#ramCache} and {@link BucketCache#backingMap} in order todetermine whether a given element hit. It could uses memory {@link ByteBufferIOEngine} or file {@link FileIOEngine}to store/read the block data. Eviction is using similar algorithm as {@link org.apache.hadoop.hbase.io.hfile.LruBlockCache}BucketCache could be used as mainly a block cache(see {@link CombinedBlockCache}), combined with LruBlockCache to decrease CMS and fragment by GC. Also could be used as a secondary cache(e.g. using Fusionio to store block) to enlarge cache space by {@link org.apache.hadoop.hbase.io.hfile.LruBlockCache#setVictimCache}

      long bucketCacheSize = (long) (bucketCachePercentage < 1 ? mu.getMax()
          * bucketCachePercentage : bucketCachePercentage * 1024 * 1024);


      boolean combinedWithLru = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
          DEFAULT_BUCKET_CACHE_COMBINED);
      BucketCache bucketCache = null;
      if (bucketCacheIOEngineName != null && bucketCacheSize > 0) {
        int writerThreads = conf.getInt(BUCKET_CACHE_WRITER_THREADS_KEY,
            DEFAULT_BUCKET_CACHE_WRITER_THREADS);
        int writerQueueLen = conf.getInt(BUCKET_CACHE_WRITER_QUEUE_KEY,
            DEFAULT_BUCKET_CACHE_WRITER_QUEUE);
        String persistentPath = conf.get(BUCKET_CACHE_PERSISTENT_PATH_KEY);
        float combinedPercentage = conf.getFloat(
            BUCKET_CACHE_COMBINED_PERCENTAGE_KEY,
            DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE);
        if (combinedWithLru) {
          lruCacheSize = (long) ((1 - combinedPercentage) * bucketCacheSize);
          bucketCacheSize = (long) (combinedPercentage * bucketCacheSize);
        }
        try {
          int ioErrorsTolerationDuration = conf.getInt(
              "hbase.bucketcache.ioengine.errors.tolerated.duration",
              BucketCache.DEFAULT_ERROR_TOLERATION_DURATION);
          bucketCache = new BucketCache(bucketCacheIOEngineName,
              bucketCacheSize, writerThreads, writerQueueLen, persistentPath,
              ioErrorsTolerationDuration);
        } catch (IOException ioex) {
          LOG.error("Can't instantiate bucket cache", ioex);
          throw new RuntimeException(ioex);

View Full Code Here

      long bucketCacheSize = (long) (bucketCachePercentage < 1 ? mu.getMax()
          * bucketCachePercentage : bucketCachePercentage * 1024 * 1024);


      boolean combinedWithLru = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
          DEFAULT_BUCKET_CACHE_COMBINED);
      BucketCache bucketCache = null;
      if (bucketCacheIOEngineName != null && bucketCacheSize > 0) {
        int writerThreads = conf.getInt(BUCKET_CACHE_WRITER_THREADS_KEY,
            DEFAULT_BUCKET_CACHE_WRITER_THREADS);
        int writerQueueLen = conf.getInt(BUCKET_CACHE_WRITER_QUEUE_KEY,
            DEFAULT_BUCKET_CACHE_WRITER_QUEUE);
        String persistentPath = conf.get(BUCKET_CACHE_PERSISTENT_PATH_KEY);
        float combinedPercentage = conf.getFloat(
            BUCKET_CACHE_COMBINED_PERCENTAGE_KEY,
            DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE);
        if (combinedWithLru) {
          lruCacheSize = (long) ((1 - combinedPercentage) * bucketCacheSize);
          bucketCacheSize = (long) (combinedPercentage * bucketCacheSize);
        }
        try {
          int ioErrorsTolerationDuration = conf.getInt(
              "hbase.bucketcache.ioengine.errors.tolerated.duration",
              BucketCache.DEFAULT_ERROR_TOLERATION_DURATION);
          bucketCache = new BucketCache(bucketCacheIOEngineName,
              bucketCacheSize, blockSize, writerThreads, writerQueueLen, persistentPath,
              ioErrorsTolerationDuration);
        } catch (IOException ioex) {
          LOG.error("Can't instantiate bucket cache", ioex);
          throw new RuntimeException(ioex);

View Full Code Here

      long bucketCacheSize = (long) (bucketCachePercentage < 1 ? mu.getMax()
          * bucketCachePercentage : bucketCachePercentage * 1024 * 1024);


      boolean combinedWithLru = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
          DEFAULT_BUCKET_CACHE_COMBINED);
      BucketCache bucketCache = null;
      if (bucketCacheIOEngineName != null && bucketCacheSize > 0) {
        int writerThreads = conf.getInt(BUCKET_CACHE_WRITER_THREADS_KEY,
            DEFAULT_BUCKET_CACHE_WRITER_THREADS);
        int writerQueueLen = conf.getInt(BUCKET_CACHE_WRITER_QUEUE_KEY,
            DEFAULT_BUCKET_CACHE_WRITER_QUEUE);
        String persistentPath = conf.get(BUCKET_CACHE_PERSISTENT_PATH_KEY);
        float combinedPercentage = conf.getFloat(
            BUCKET_CACHE_COMBINED_PERCENTAGE_KEY,
            DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE);
        if (combinedWithLru) {
          lruCacheSize = (long) ((1 - combinedPercentage) * bucketCacheSize);
          bucketCacheSize = (long) (combinedPercentage * bucketCacheSize);
        }
        try {
          int ioErrorsTolerationDuration = conf.getInt(
              "hbase.bucketcache.ioengine.errors.tolerated.duration",
              BucketCache.DEFAULT_ERROR_TOLERATION_DURATION);
          bucketCache = new BucketCache(bucketCacheIOEngineName,
              bucketCacheSize, writerThreads, writerQueueLen, persistentPath,
              ioErrorsTolerationDuration);
        } catch (IOException ioex) {
          LOG.error("Can't instantiate bucket cache", ioex);
          throw new RuntimeException(ioex);

View Full Code Here

      bucketSizes = new int[configuredBucketSizes.length];
      for (int i = 0; i < configuredBucketSizes.length; i++) {
        bucketSizes[i] = Integer.parseInt(configuredBucketSizes[i].trim());
      }
    }
    BucketCache bucketCache = null;
    try {
      int ioErrorsTolerationDuration = c.getInt(
        "hbase.bucketcache.ioengine.errors.tolerated.duration",
        BucketCache.DEFAULT_ERROR_TOLERATION_DURATION);
      // Bucket cache logs its stats on creation internal to the constructor.
      bucketCache = new BucketCache(bucketCacheIOEngineName,
        bucketCacheSize, blockSize, bucketSizes, writerThreads, writerQueueLen, persistentPath,
        ioErrorsTolerationDuration);
    } catch (IOException ioex) {
      LOG.error("Can't instantiate bucket cache", ioex); throw new RuntimeException(ioex);
    }

View Full Code Here

    if (blockCacheDisabled) return null;
    MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
    LruBlockCache l1 = getL1(conf, mu);
    // blockCacheDisabled is set as a side-effect of getL1(), so check it again after the call.
    if (blockCacheDisabled) return null;
    BucketCache l2 = getL2(conf, mu);
    if (l2 == null) {
      GLOBAL_BLOCK_CACHE_INSTANCE = l1;
    } else {
      boolean combinedWithLru = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
        DEFAULT_BUCKET_CACHE_COMBINED);

View Full Code Here

    assertTrue(bcs[0] instanceof LruBlockCache);
    LruBlockCache lbc = (LruBlockCache)bcs[0];
    assertEquals(CacheConfig.getLruCacheSize(this.conf,
        ManagementFactory.getMemoryMXBean().getHeapMemoryUsage()), lbc.getMaxSize());
    assertTrue(bcs[1] instanceof BucketCache);
    BucketCache bc = (BucketCache)bcs[1];
    // getMaxSize comes back in bytes but we specified size in MB
    assertEquals(bcSize, bc.getMaxSize() / (1024 * 1024));
  }

View Full Code Here

    basicBlockCacheOps(cc, false, false);
    assertTrue(cc.getBlockCache() instanceof LruBlockCache);
    // TODO: Assert sizes allocated are right and proportions.
    LruBlockCache lbc = (LruBlockCache)cc.getBlockCache();
    assertEquals(lruExpectedSize, lbc.getMaxSize());
    BucketCache bc = lbc.getVictimHandler();
    // getMaxSize comes back in bytes but we specified size in MB
    assertEquals(bcExpectedSize, bc.getMaxSize());
    // Test the L1+L2 deploy works as we'd expect with blocks evicted from L1 going to L2.
    long initialL1BlockCount = lbc.getBlockCount();
    long initialL2BlockCount = bc.getBlockCount();
    Cacheable c = new DataCacheEntry();
    BlockCacheKey bck = new BlockCacheKey("bck", 0);
    lbc.cacheBlock(bck, c, false, false);
    assertEquals(initialL1BlockCount + 1, lbc.getBlockCount());
    assertEquals(initialL2BlockCount, bc.getBlockCount());
    // Force evictions by putting in a block too big.
    final long justTooBigSize = lbc.acceptableSize() + 1;
    lbc.cacheBlock(new BlockCacheKey("bck2", 0), new DataCacheEntry() {
      @Override
      public long heapSize() {
        return justTooBigSize;
      }


      @Override
      public int getSerializedLength() {
        return (int)heapSize();
      }
    });
    // The eviction thread in lrublockcache needs to run.
    while (initialL1BlockCount != lbc.getBlockCount()) Threads.sleep(10);
    assertEquals(initialL1BlockCount, lbc.getBlockCount());
    long count = bc.getBlockCount();
    assertTrue(initialL2BlockCount + 1 <= count);
  }

View Full Code Here

      long bucketCacheSize = (long) (bucketCachePercentage < 1 ? mu.getMax()
          * bucketCachePercentage : bucketCachePercentage * 1024 * 1024);


      boolean combinedWithLru = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
          DEFAULT_BUCKET_CACHE_COMBINED);
      BucketCache bucketCache = null;
      if (bucketCacheIOEngineName != null && bucketCacheSize > 0) {
        int writerThreads = conf.getInt(BUCKET_CACHE_WRITER_THREADS_KEY,
            DEFAULT_BUCKET_CACHE_WRITER_THREADS);
        int writerQueueLen = conf.getInt(BUCKET_CACHE_WRITER_QUEUE_KEY,
            DEFAULT_BUCKET_CACHE_WRITER_QUEUE);
        String persistentPath = conf.get(BUCKET_CACHE_PERSISTENT_PATH_KEY);
        float combinedPercentage = conf.getFloat(
            BUCKET_CACHE_COMBINED_PERCENTAGE_KEY,
            DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE);
        String[] configuredBucketSizes = conf.getStrings(BUCKET_CACHE_BUCKETS_KEY);
        int[] bucketSizes = null;
        if (configuredBucketSizes != null) {
          bucketSizes = new int[configuredBucketSizes.length];
          for (int i = 0; i < configuredBucketSizes.length; i++) {
            bucketSizes[i] = Integer.parseInt(configuredBucketSizes[i].trim());
          }
        }
        if (combinedWithLru) {
          lruCacheSize = (long) ((1 - combinedPercentage) * bucketCacheSize);
          bucketCacheSize = (long) (combinedPercentage * bucketCacheSize);
        }
        try {
          int ioErrorsTolerationDuration = conf.getInt(
              "hbase.bucketcache.ioengine.errors.tolerated.duration",
              BucketCache.DEFAULT_ERROR_TOLERATION_DURATION);
          bucketCache = new BucketCache(bucketCacheIOEngineName,
              bucketCacheSize, blockSize, bucketSizes, writerThreads, writerQueueLen, persistentPath,
              ioErrorsTolerationDuration);
        } catch (IOException ioex) {
          LOG.error("Can't instantiate bucket cache", ioex);
          throw new RuntimeException(ioex);

View Full Code Here

      bucketSizes = new int[configuredBucketSizes.length];
      for (int i = 0; i < configuredBucketSizes.length; i++) {
        bucketSizes[i] = Integer.parseInt(configuredBucketSizes[i].trim());
      }
    }
    BucketCache bucketCache = null;
    try {
      int ioErrorsTolerationDuration = c.getInt(
        "hbase.bucketcache.ioengine.errors.tolerated.duration",
        BucketCache.DEFAULT_ERROR_TOLERATION_DURATION);
      // Bucket cache logs its stats on creation internal to the constructor.
      bucketCache = new BucketCache(bucketCacheIOEngineName,
        bucketCacheSize, blockSize, bucketSizes, writerThreads, writerQueueLen, persistentPath,
        ioErrorsTolerationDuration);
    } catch (IOException ioex) {
      LOG.error("Can't instantiate bucket cache", ioex); throw new RuntimeException(ioex);
    }

View Full Code Here

    if (blockCacheDisabled) return null;
    MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
    LruBlockCache l1 = getL1(conf, mu);
    // blockCacheDisabled is set as a side-effect of getL1(), so check it again after the call.
    if (blockCacheDisabled) return null;
    BucketCache l2 = getL2(conf, mu);
    if (l2 == null) {
      GLOBAL_BLOCK_CACHE_INSTANCE = l1;
    } else {
      boolean combinedWithLru = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
        DEFAULT_BUCKET_CACHE_COMBINED);

View Full Code Here

0 1

TOP

Related Classes of org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$WriterThread

com.google.common.base.Stopwatch

com.sleepycat.je.Cursor

com.sleepycat.je.DatabaseEntry

com.sleepycat.je.OperationStatus

com.sleepycat.je.Transaction

org.apache.flume.event.SimpleEvent

org.apache.hadoop.hbase.io.hfile.BlockCacheKey

org.apache.hadoop.hbase.io.hfile.Cacheable

org.apache.hadoop.hbase.io.hfile.CacheConfig

org.apache.hadoop.hbase.io.hfile.CachedBlock

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.