Examples of net.agkn.hll.HLL

net.agkn.hll.HLL
inria.fr/flajolet/Publications/FlFuGaMe07.pdf"> 'HyperLogLog' data structure and algorithm is used, which combines both probabilistic and non-probabilistic techniques to improve the accuracy and storage requirements of the original algorithm.
More specifically, initializing and storing a new {@link HLL} willallocate a sentinel value symbolizing the empty set ( {@link HLLType#EMPTY}). After adding the first few values, a sorted list of unique integers is stored in a {@link HLLType#EXPLICIT} hash set. When configured, accuracy canbe sacrificed for memory footprint: the values in the sorted list are "promoted" to a " {@link HLLType#SPARSE}" map-based HyperLogLog structure. Finally, when enough registers are set, the map-based HLL will be converted to a bit-packed " {@link HLLType#FULL}" HyperLogLog structure.
This data structure is interoperable with the implementations found at:
- postgresql-hll, and
- js-hll
when properly serialized. @author timon

           throws CloneNotSupportedException {
        for(int log2m=MINIMUM_LOG2M_PARAM; log2m<=16; log2m++) {
            for(int regw=MINIMUM_REGWIDTH_PARAM; regw<=MAXIMUM_REGWIDTH_PARAM; regw++) {
                for(int expthr=MINIMUM_EXPTHRESH_PARAM; expthr<=MAXIMUM_EXPTHRESH_PARAM; expthr++ ) {
                    for(final boolean sparse: new boolean[]{true, false}) {
                        HLL hll = new HLL(log2m, regw, expthr, sparse, hllType);
                        for(final Long item: items) {
                            hll.addRaw(item);
                        }
                        HLL copy = HLL.fromBytes(hll.toBytes());
                        assertEquals(copy.cardinality(), hll.cardinality());
                        assertEquals(copy.getType(), hll.getType());
                        assertEquals(copy.toBytes(), hll.toBytes());


                        HLL clone = hll.clone();
                        assertEquals(clone.cardinality(), hll.cardinality());
                        assertEquals(clone.getType(), hll.getType());
                        assertEquals(clone.toBytes(), hll.toBytes());
                    }
                }
            }
        }
    }

View Full Code Here

     */
    private static void fullCardinalityCorrectionTest(final ISchemaVersion schemaVersion) throws IOException {
        final FileWriter output = openOutput(schemaVersion, "cardinality_correction", TestType.ADD);


        // the accumulator, starts empty
        final HLL hll = newHLL(HLLType.FULL);
        initLineAdd(output, hll, schemaVersion);


        // run through some values in the small range correction
        for(int i=0; i<((1 << LOG2M) - 1); i++) {
            final long rawValue = constructHLLValue(LOG2M, i, 1);

View Full Code Here

        final FileWriter output = openOutput(schemaVersion, "comprehensive_promotion", TestType.ADD);


        final Random random = new Random(SEED);


        // the accumulator, starts empty
        final HLL hll = newHLL(HLLType.EMPTY);
        initLineAdd(output, hll, schemaVersion);


        for(int i=0; i<10000/*arbitrary*/; i++) {
            cumulativeAddLine(output, hll, random.nextLong(), schemaVersion);
        }

View Full Code Here

     * - SPARSE U "barely underpopulated" FULL => FULL
     */
    private static void sparseFullRepresentationTest(final ISchemaVersion schemaVersion) throws IOException {
        final FileWriter output = openOutput(schemaVersion, "sparse_full_representation", TestType.UNION);


        final HLL emptyHLL1 = newHLL(HLLType.EMPTY);
        final HLL emptyHLL2 = newHLL(HLLType.EMPTY);


        cumulativeUnionLine(output, emptyHLL1, emptyHLL2, schemaVersion);


        // NOTE:  In this test the sparseReference will be the "expected" value
        //        from the C representation, since it doesn't choose representation
        //        based on original encoding, but rather on the promotion rules
        //        and the declared type of the "receiving" field.
        //        It is the manually-constructed union result.


        // "underpopulated" FULL U EMPTY => SPARSE
        final HLL fullHLL = newHLL(HLLType.FULL);
        fullHLL.addRaw(constructHLLValue(LOG2M, 0/*ix*/, 1/*val*/));


        final HLL sparseHLL = newHLL(HLLType.SPARSE);
        sparseHLL.addRaw(constructHLLValue(LOG2M, 0/*ix*/, 1/*val*/));


        output.write(stringCardinality(fullHLL) + "," + toByteA(fullHLL, schemaVersion) + "," + stringCardinality(sparseHLL) + "," + toByteA(sparseHLL, schemaVersion) + "\n");
        output.flush();


        // "underpopulated" FULL (small) U SPARSE (small) => SPARSE
        final HLL fullHLL2 = newHLL(HLLType.FULL);
        fullHLL2.addRaw(constructHLLValue(LOG2M, 1/*ix*/, 1/*val*/));


        sparseHLL.addRaw(constructHLLValue(LOG2M, 1/*ix*/, 1/*val*/));


        output.write(stringCardinality(fullHLL2) + "," + toByteA(fullHLL2, schemaVersion) + "," + stringCardinality(sparseHLL) + "," + toByteA(sparseHLL, schemaVersion) + "\n");
        output.flush();


        // "underpopulated" FULL (just on edge) U SPARSE (small) => FULL
        final HLL fullHLL3 = newHLL(HLLType.FULL);
        for(int i=2; i<(SPARSE_THRESHOLD + 1); i++) {
            fullHLL3.addRaw(constructHLLValue(LOG2M, i/*ix*/, 1/*val*/));
            sparseHLL.addRaw(constructHLLValue(LOG2M, i/*ix*/, 1/*val*/));
        }


        output.write(stringCardinality(fullHLL3) + "," + toByteA(fullHLL3, schemaVersion) + "," + stringCardinality(sparseHLL) + "," + toByteA(sparseHLL, schemaVersion) + "\n");
        output.flush();

View Full Code Here

     */
    private static void sparseStepTest(final ISchemaVersion schemaVersion) throws IOException {
        final FileWriter output = openOutput(schemaVersion, "sparse_step", TestType.ADD);


        // the accumulator, starts empty sparse probabilistic
        final HLL hll = newHLL(HLLType.SPARSE);
        initLineAdd(output, hll, schemaVersion);


        for(int i=0; i<SPARSE_THRESHOLD; i++) {
            final long rawValue = constructHLLValue(LOG2M, i, ((i % REGISTER_MAX_VALUE) + 1));
            cumulativeAddLine(output, hll, rawValue, schemaVersion);

View Full Code Here

        final FileWriter output = openOutput(schemaVersion, "sparse_random", TestType.ADD);


        final Random random = new Random(SEED);


        // the accumulator, starts empty
        final HLL hll = newHLL(HLLType.SPARSE);
        initLineAdd(output, hll, schemaVersion);


        for(int i=0; i<SPARSE_THRESHOLD; i++) {
            final int registerIndex = Math.abs(random.nextInt()) % REGISTER_COUNT;
            final int registerValue = ((Math.abs(random.nextInt()) % REGISTER_MAX_VALUE) + 1);

View Full Code Here

     */
    private static void sparseEdgeTest(final ISchemaVersion schemaVersion) throws IOException {
        final FileWriter output = openOutput(schemaVersion, "sparse_edge", TestType.ADD);


        // the accumulator, starts empty
        final HLL hll = newHLL(HLLType.SPARSE);
        initLineAdd(output, hll, schemaVersion);


        final long firstValue = constructHLLValue(LOG2M, 0, 2);
        cumulativeAddLine(output, hll, firstValue, schemaVersion);

View Full Code Here

        final FileWriter output = openOutput(schemaVersion, "explicit_promotion", TestType.UNION);


        final Random random = new Random(SEED);


        // the accumulator, starts empty
        final HLL hll = newHLL(HLLType.EMPTY);
        final HLL emptyHLL = newHLL(HLLType.EMPTY);
        cumulativeUnionLine(output, hll, emptyHLL, schemaVersion);


        for(int i=0; i<(EXPLICIT_THRESHOLD+500)/*should be greater than promotion cutoff*/; i++) {
            // make an EXPLICIT set and populate with cardinality 1
            final HLL explicitHLL = newHLL(HLLType.EXPLICIT);
            explicitHLL.addRaw(random.nextLong());


            cumulativeUnionLine(output, hll, explicitHLL, schemaVersion);
        }


        output.flush();

View Full Code Here

        final FileWriter output = openOutput(schemaVersion, "sparse_promotion", TestType.UNION);


        final Random random = new Random(SEED);


        // the accumulator, starts empty
        final HLL hll = newHLL(HLLType.EMPTY);
        final HLL emptyHLL = newHLL(HLLType.EMPTY);
        cumulativeUnionLine(output, hll, emptyHLL, schemaVersion);




        for(int i=0; i<(SPARSE_THRESHOLD + 1000)/*should be greater than promotion cutoff*/; i++) {
            // make a SPARSE set and populate with cardinality 1
            final HLL sparseHLL = newHLL(HLLType.SPARSE);


            final int registerIndex = Math.abs(random.nextInt()) % REGISTER_COUNT;
            final int registerValue = ((Math.abs(random.nextInt()) % REGISTER_MAX_VALUE) + 1);
            final long rawValue = constructHLLValue(LOG2M, registerIndex, registerValue);
            sparseHLL.addRaw(rawValue);


            cumulativeUnionLine(output, hll, sparseHLL, schemaVersion);
        }


        output.flush();

View Full Code Here

        final FileWriter output = openOutput(schemaVersion, "explicit_explicit", TestType.UNION);


        final Random random = new Random(SEED);


        // the accumulator, starts empty
        final HLL hll = newHLL(HLLType.EMPTY);
        final HLL emptyHLL = newHLL(HLLType.EMPTY);


        cumulativeUnionLine(output, hll, emptyHLL, schemaVersion);


        for(int i=0; i<EXPLICIT_THRESHOLD; i++) {
            // make an EXPLICIT set and populate with cardinality 1
            final HLL explicitHLL = newHLL(HLLType.EXPLICIT);
            explicitHLL.addRaw(random.nextLong());


            // union it into the accumulator twice, to test overlap (cardinality should not change)
            cumulativeUnionLine(output, hll, explicitHLL, schemaVersion);
            cumulativeUnionLine(output, hll, explicitHLL, schemaVersion);
        }

View Full Code Here

0 1

TOP

Related Classes of net.agkn.hll.HLL

it.unimi.dsi.fastutil.ints.Int2ByteOpenHashMap

it.unimi.dsi.fastutil.longs.LongOpenHashSet

net.agkn.hll.IntegrationTestGenerator

net.agkn.hll.serialization.HLLMetadata

net.agkn.hll.serialization.HLLSerializationTest

net.agkn.hll.serialization.IHLLMetadata

net.agkn.hll.serialization.ISchemaVersion

net.agkn.hll.serialization.IWordDeserializer

net.agkn.hll.serialization.IWordSerializer

net.agkn.hll.util.BitVector

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.