Examples of org.elasticsearch.common.util.BloomFilter$BitArray

org.elasticsearch.common.util.BloomFilter
A factory that can use different fpp based on size.


    public static void main(String[] args) throws Exception {
        SecureRandom random = new SecureRandom();
        final int ELEMENTS = (int) SizeValue.parseSizeValue("1m").singles();
        final double fpp = 0.01;
        BloomFilter gFilter = BloomFilter.create(ELEMENTS, fpp);
        System.out.println("G SIZE: " + new ByteSizeValue(gFilter.getSizeInBytes()));


        FuzzySet lFilter = FuzzySet.createSetBasedOnMaxMemory((int) gFilter.getSizeInBytes());
        //FuzzySet lFilter = FuzzySet.createSetBasedOnQuality(ELEMENTS, 0.97f);


        for (int i = 0; i < ELEMENTS; i++) {
            BytesRef bytesRef = new BytesRef(Strings.randomBase64UUID(random));
            gFilter.put(bytesRef);
            lFilter.addValue(bytesRef);
        }


        int lFalse = 0;
        int gFalse = 0;
        for (int i = 0; i < ELEMENTS; i++) {
            BytesRef bytesRef = new BytesRef(Strings.randomBase64UUID(random));
            if (gFilter.mightContain(bytesRef)) {
                gFalse++;
            }
            if (lFilter.contains(bytesRef) == FuzzySet.ContainsResult.MAYBE) {
                lFalse++;
            }

View Full Code Here

                    continue;
                }
                FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field);
                TermsEnum termsEnum = terms.iterator(null);


                BloomFilter bloomFilter = null;


                DocsEnum docsEnum = null;
                while (true) {
                    BytesRef term = termsEnum.next();
                    if (term == null) {
                        break;
                    }
                    if (bloomFilter == null) {
                        bloomFilter = bloomFilterFactory.createFilter(state.segmentInfo.getDocCount());
                        assert bloomFilters.containsKey(field) == false;
                        bloomFilters.put(fieldInfo, bloomFilter);
                    }
                    // Make sure there's at least one doc for this term:
                    docsEnum = termsEnum.docs(null, docsEnum, 0);
                    if (docsEnum.nextDoc() != DocsEnum.NO_MORE_DOCS) {
                        bloomFilter.put(term);
                    }
                }
            }
        }

View Full Code Here


                // First field in the output file is the number of fields+blooms saved
                bloomOutput.writeInt(nonSaturatedBlooms.size());
                for (Entry<FieldInfo, BloomFilter> entry : nonSaturatedBlooms) {
                    FieldInfo fieldInfo = entry.getKey();
                    BloomFilter bloomFilter = entry.getValue();
                    bloomOutput.writeInt(fieldInfo.number);
                    saveAppropriatelySizedBloomFilter(bloomOutput, bloomFilter, fieldInfo);
                }
                CodecUtil.writeFooter(bloomOutput);
            } finally {

View Full Code Here

TOP

Related Classes of org.elasticsearch.common.util.BloomFilter$BitArray

org.elasticsearch.benchmark.bloom.BloomBench

org.elasticsearch.index.codec.postingsformat.BloomFilterPostingsFormat$BloomFilteredFieldsConsumer

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.