Package org.apache.hadoop.hbase.util

Examples of org.apache.hadoop.hbase.util.BloomFilter


          return true;
      }

      // Cache Bloom filter as a local variable in case it is set to null by
      // another thread on an IO error.
      BloomFilter bloomFilter = this.generalBloomFilter;

      if (bloomFilter == null) {
        return true;
      }

      // Empty file
      if (reader.getTrailer().getEntryCount() == 0)
        return false;

      try {
        boolean shouldCheckBloom;
        ByteBuffer bloom;
        if (bloomFilter.supportsAutoLoading()) {
          bloom = null;
          shouldCheckBloom = true;
        } else {
          bloom = reader.getMetaBlock(HFileWriterV1.BLOOM_FILTER_DATA_KEY,
              true);
          shouldCheckBloom = bloom != null;
        }

        if (shouldCheckBloom) {
          boolean exists;

          // Whether the primary Bloom key is greater than the last Bloom key
          // from the file info. For row-column Bloom filters this is not yet
          // a sufficient condition to return false.
          boolean keyIsAfterLast = lastBloomKey != null
              && bloomFilter.getComparator().compare(key, lastBloomKey) > 0;

          if (bloomFilterType == BloomType.ROWCOL) {
            // Since a Row Delete is essentially a DeleteFamily applied to all
            // columns, a file might be skipped if using row+col Bloom filter.
            // In order to ensure this file is included an additional check is
            // required looking only for a row bloom.
            byte[] rowBloomKey = bloomFilter.createBloomKey(row, 0, row.length,
                null, 0, 0);

            if (keyIsAfterLast
                && bloomFilter.getComparator().compare(rowBloomKey,
                    lastBloomKey) > 0) {
              exists = false;
            } else {
              exists =
                  bloomFilter.contains(key, 0, key.length, bloom) ||
                  bloomFilter.contains(rowBloomKey, 0, rowBloomKey.length,
                      bloom);
            }
          } else {
            exists = !keyIsAfterLast
                && bloomFilter.contains(key, 0, key.length, bloom);
          }

          getSchemaMetrics().updateBloomMetrics(exists);
          return exists;
        }
View Full Code Here


    System.out.println("Mid-key: " + Bytes.toStringBinary(reader.midkey()));

    // Printing general bloom information
    DataInput bloomMeta = reader.getGeneralBloomFilterMetadata();
    BloomFilter bloomFilter = null;
    if (bloomMeta != null)
      bloomFilter = BloomFilterFactory.createFromMeta(bloomMeta, reader);

    System.out.println("Bloom filter:");
    if (bloomFilter != null) {
      System.out.println(FOUR_SPACES + bloomFilter.toString().replaceAll(
          ByteBloomFilter.STATS_RECORD_SEP, "\n" + FOUR_SPACES));
    } else {
      System.out.println(FOUR_SPACES + "Not present");
    }

    // Printing delete bloom information
    bloomMeta = reader.getDeleteBloomFilterMetadata();
    bloomFilter = null;
    if (bloomMeta != null)
      bloomFilter = BloomFilterFactory.createFromMeta(bloomMeta, reader);

    System.out.println("Delete Family Bloom filter:");
    if (bloomFilter != null) {
      System.out.println(FOUR_SPACES
          + bloomFilter.toString().replaceAll(ByteBloomFilter.STATS_RECORD_SEP,
              "\n" + FOUR_SPACES));
    } else {
      System.out.println(FOUR_SPACES + "Not present");
    }
  }
View Full Code Here

    public boolean passesDeleteFamilyBloomFilter(byte[] row, int rowOffset,
        int rowLen) {
      // Cache Bloom filter as a local variable in case it is set to null by
      // another thread on an IO error.
      BloomFilter bloomFilter = this.deleteFamilyBloomFilter;

      // Empty file or there is no delete family at all
      if (reader.getTrailer().getEntryCount() == 0 || deleteFamilyCnt == 0) {
        return false;
      }

      if (bloomFilter == null) {
        return true;
      }

      try {
        if (!bloomFilter.supportsAutoLoading()) {
          return true;
        }
        return bloomFilter.contains(row, rowOffset, rowLen, null);
      } catch (IllegalArgumentException e) {
        LOG.error("Bad Delete Family bloom filter data -- proceeding without",
            e);
        setDeleteFamilyBloomFilterFaulty();
      }
View Full Code Here

          return true;
      }

      // Cache Bloom filter as a local variable in case it is set to null by
      // another thread on an IO error.
      BloomFilter bloomFilter = this.generalBloomFilter;

      if (bloomFilter == null) {
        return true;
      }

      // Empty file
      if (reader.getTrailer().getEntryCount() == 0)
        return false;

      try {
        boolean shouldCheckBloom;
        ByteBuffer bloom;
        if (bloomFilter.supportsAutoLoading()) {
          bloom = null;
          shouldCheckBloom = true;
        } else {
          bloom = reader.getMetaBlock(HFileWriterV1.BLOOM_FILTER_DATA_KEY,
              true);
          shouldCheckBloom = bloom != null;
        }

        if (shouldCheckBloom) {
          boolean exists;

          // Whether the primary Bloom key is greater than the last Bloom key
          // from the file info. For row-column Bloom filters this is not yet
          // a sufficient condition to return false.
          boolean keyIsAfterLast = lastBloomKey != null
              && bloomFilter.getComparator().compare(key, lastBloomKey) > 0;

          if (bloomFilterType == BloomType.ROWCOL) {
            // Since a Row Delete is essentially a DeleteFamily applied to all
            // columns, a file might be skipped if using row+col Bloom filter.
            // In order to ensure this file is included an additional check is
            // required looking only for a row bloom.
            byte[] rowBloomKey = bloomFilter.createBloomKey(row, 0, row.length,
                null, 0, 0);

            if (keyIsAfterLast
                && bloomFilter.getComparator().compare(rowBloomKey,
                    lastBloomKey) > 0) {
              exists = false;
            } else {
              exists =
                  bloomFilter.contains(key, 0, key.length, bloom) ||
                  bloomFilter.contains(rowBloomKey, 0, rowBloomKey.length,
                      bloom);
            }
          } else {
            exists = !keyIsAfterLast
                && bloomFilter.contains(key, 0, key.length, bloom);
          }

          getSchemaMetrics().updateBloomMetrics(exists);
          return exists;
        }
View Full Code Here

        throws IOException {
      writer = new HFile.Writer(fs, path, blocksize, compress, comparator.getRawComparator());

      this.kvComparator = comparator;

      BloomFilter bloom = null;
      BloomType bt = BloomType.NONE;

      if (bloomType != BloomType.NONE && conf != null) {
        float err = conf.getFloat(IO_STOREFILE_BLOOM_ERROR_RATE, (float)0.01);
        // Since in row+col blooms we have 2 calls to shouldSeek() instead of 1
        // and the false positives are adding up, we should keep the error rate
        // twice as low in order to maintain the number of false positives as
        // desired by the user
        if (bloomType == BloomType.ROWCOL) {
          err /= 2;
        }
        int maxFold = conf.getInt(IO_STOREFILE_BLOOM_MAX_FOLD, 7);
        int tooBig = conf.getInt(IO_STOREFILE_BLOOM_MAX_KEYS, 128*1000*1000);
       
        if (maxKeys < tooBig) {
          try {
            bloom = new ByteBloomFilter(maxKeys, err,
                Hash.getHashType(conf), maxFold);
            bloom.allocBloom();
            bt = bloomType;
          } catch (IllegalArgumentException iae) {
            LOG.warn(String.format(
              "Parse error while creating bloom for %s (%d, %f)",
              path, maxKeys, err), iae);
View Full Code Here

        return timeRangeTracker.includesTimeRange(scan.getTimeRange());
      }
    }

    private boolean passesBloomFilter(Scan scan, final SortedSet<byte[]> columns) {
      BloomFilter bm = this.bloomFilter;
      if (bm == null || !scan.isGetScan()) {
        return true;
      }
      byte[] row = scan.getStartRow();
      byte[] key;
      switch (this.bloomFilterType) {
        case ROW:
          key = row;
          break;
        case ROWCOL:
          if (columns != null && columns.size() == 1) {
            byte[] col = columns.first();
            key = Bytes.add(row, col);
            break;
          }
          //$FALL-THROUGH$
        default:
          return true;
      }

      try {
        ByteBuffer bloom = reader.getMetaBlock(BLOOM_FILTER_DATA_KEY, true);
        if (bloom != null) {
          if (this.bloomFilterType == BloomType.ROWCOL) {
            // Since a Row Delete is essentially a DeleteFamily applied to all
            // columns, a file might be skipped if using row+col Bloom filter.
            // In order to ensure this file is included an additional check is
            // required looking only for a row bloom.
            return bm.contains(key, bloom) ||
                bm.contains(row, bloom);
          }
          else {
            return bm.contains(key, bloom);
          }
        }
      } catch (IOException e) {
        LOG.error("Error reading bloom filter data -- proceeding without",
            e);
View Full Code Here

        throws IOException {
      writer = new HFile.Writer(fs, path, blocksize, compress, comparator.getRawComparator());

      this.kvComparator = comparator;

      BloomFilter bloom = null;
      BloomType bt = BloomType.NONE;

      if (bloomType != BloomType.NONE && conf != null) {
        float err = conf.getFloat(IO_STOREFILE_BLOOM_ERROR_RATE, (float)0.01);
        // Since in row+col blooms we have 2 calls to shouldSeek() instead of 1
        // and the false positives are adding up, we should keep the error rate
        // twice as low in order to maintain the number of false positives as
        // desired by the user
        if (bloomType == BloomType.ROWCOL) {
          err /= 2;
        }
        int maxFold = conf.getInt(IO_STOREFILE_BLOOM_MAX_FOLD, 7);
        int tooBig = conf.getInt(IO_STOREFILE_BLOOM_MAX_KEYS, 128*1000*1000);
       
        if (maxKeys < tooBig) {
          try {
            bloom = new ByteBloomFilter(maxKeys, err,
                Hash.getHashType(conf), maxFold);
            bloom.allocBloom();
            bt = bloomType;
          } catch (IllegalArgumentException iae) {
            LOG.warn(String.format(
              "Parse error while creating bloom for %s (%d, %f)",
              path, maxKeys, err), iae);
View Full Code Here

        return timeRangeTracker.includesTimeRange(scan.getTimeRange());
      }
    }

    private boolean passesBloomFilter(Scan scan, final SortedSet<byte[]> columns) {
      BloomFilter bm = this.bloomFilter;
      if (bm == null || !scan.isGetScan()) {
        return true;
      }
      byte[] row = scan.getStartRow();
      byte[] key;
      switch (this.bloomFilterType) {
        case ROW:
          key = row;
          break;
        case ROWCOL:
          if (columns != null && columns.size() == 1) {
            byte[] col = columns.first();
            key = Bytes.add(row, col);
            break;
          }
          //$FALL-THROUGH$
        default:
          return true;
      }

      try {
        ByteBuffer bloom = reader.getMetaBlock(BLOOM_FILTER_DATA_KEY, true);
        if (bloom != null) {
          if (this.bloomFilterType == BloomType.ROWCOL) {
            // Since a Row Delete is essentially a DeleteFamily applied to all
            // columns, a file might be skipped if using row+col Bloom filter.
            // In order to ensure this file is included an additional check is
            // required looking only for a row bloom.
            return bm.contains(key, bloom) ||
                bm.contains(row, bloom);
          }
          else {
            return bm.contains(key, bloom);
          }
        }
      } catch (IOException e) {
        LOG.error("Error reading bloom filter data -- proceeding without",
            e);
View Full Code Here

      System.out.println ("Unable to retrieve the midkey");
    }

    // Printing general bloom information
    DataInput bloomMeta = reader.getGeneralBloomFilterMetadata();
    BloomFilter bloomFilter = null;
    if (bloomMeta != null)
      bloomFilter = BloomFilterFactory.createFromMeta(bloomMeta, reader);

    System.out.println("Bloom filter:");
    if (bloomFilter != null) {
      System.out.println(FOUR_SPACES + bloomFilter.toString().replaceAll(
          ByteBloomFilter.STATS_RECORD_SEP, "\n" + FOUR_SPACES));
    } else {
      System.out.println(FOUR_SPACES + "Not present");
    }

    // Printing delete bloom information
    bloomMeta = reader.getDeleteBloomFilterMetadata();
    bloomFilter = null;
    if (bloomMeta != null)
      bloomFilter = BloomFilterFactory.createFromMeta(bloomMeta, reader);

    System.out.println("Delete Family Bloom filter:");
    if (bloomFilter != null) {
      System.out.println(FOUR_SPACES
          + bloomFilter.toString().replaceAll(ByteBloomFilter.STATS_RECORD_SEP,
              "\n" + FOUR_SPACES));
    } else {
      System.out.println(FOUR_SPACES + "Not present");
    }
  }
View Full Code Here

    public boolean passesDeleteFamilyBloomFilter(byte[] row, int rowOffset,
        int rowLen) {
      // Cache Bloom filter as a local variable in case it is set to null by
      // another thread on an IO error.
      BloomFilter bloomFilter = this.deleteFamilyBloomFilter;

      // Empty file or there is no delete family at all
      if (reader.getTrailer().getEntryCount() == 0 || deleteFamilyCnt == 0) {
        return false;
      }

      if (bloomFilter == null) {
        return true;
      }

      try {
        if (!bloomFilter.supportsAutoLoading()) {
          return true;
        }
        return bloomFilter.contains(row, rowOffset, rowLen, null);
      } catch (IllegalArgumentException e) {
        LOG.error("Bad Delete Family bloom filter data -- proceeding without",
            e);
        setDeleteFamilyBloomFilterFaulty();
      }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hbase.util.BloomFilter

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.