Examples of HyperLogLog

com.clearspring.analytics.stream.cardinality.HyperLogLog
inria.fr/flajolet/Publications/FlFuGaMe07.pdf
HLL is an improved version of LogLog that is capable of estimating the cardinality of a set with accuracy = 1.04/sqrt(m) where m = 2^b. So we can control accuracy vs space usage by increasing or decreasing b.
The main benefit of using HLL over LL is that it only requires 64% of the space that LL does to get the same accuracy.
This implementation implements a single counter. If a large (millions) number of counters are required you may want to refer to:
http://dsiutils.dsi.unimi.it/
It has a more complex implmentation of HLL that supports multiple counters in a single object, drastically reducing the java overhead from creating a large number of objects.
This implementation leveraged a javascript implementation that Yammer has been working on:
https://github.com/yammer/probablyjs
io.airlift.stats.cardinality.HyperLogLog

Examples of io.airlift.stats.cardinality.HyperLogLog

    }


    @InputFunction
    public static void input(HyperLogLogState state, @SqlType(StandardTypes.VARCHAR) Slice value)
    {
        HyperLogLog hll = getOrCreateHyperLogLog(state);
        state.addMemoryUsage(-hll.estimatedInMemorySize());
        hll.add(value);
        state.addMemoryUsage(hll.estimatedInMemorySize());
    }

View Full Code Here

Examples of io.airlift.stats.cardinality.HyperLogLog

    }


    @InputFunction
    public static void input(HyperLogLogState state, @SqlType(StandardTypes.BIGINT) long value)
    {
        HyperLogLog hll = getOrCreateHyperLogLog(state);
        state.addMemoryUsage(-hll.estimatedInMemorySize());
        hll.add(value);
        state.addMemoryUsage(hll.estimatedInMemorySize());
    }

View Full Code Here

Examples of io.airlift.stats.cardinality.HyperLogLog

        state.addMemoryUsage(hll.estimatedInMemorySize());
    }


    private static HyperLogLog getOrCreateHyperLogLog(HyperLogLogState state)
    {
        HyperLogLog hll = state.getHyperLogLog();
        if (hll == null) {
            hll = HyperLogLog.newInstance(NUMBER_OF_BUCKETS);
            state.setHyperLogLog(hll);
            state.addMemoryUsage(hll.estimatedInMemorySize());
        }
        return hll;
    }

View Full Code Here

Examples of io.airlift.stats.cardinality.HyperLogLog

    }


    @CombineFunction
    public static void combineState(HyperLogLogState state, HyperLogLogState otherState)
    {
        HyperLogLog input = otherState.getHyperLogLog();


        HyperLogLog previous = state.getHyperLogLog();
        if (previous == null) {
            state.setHyperLogLog(input);
            state.addMemoryUsage(input.estimatedInMemorySize());
        }
        else {
            state.addMemoryUsage(-previous.estimatedInMemorySize());
            previous.mergeWith(input);
            state.addMemoryUsage(previous.estimatedInMemorySize());
        }
    }

View Full Code Here

Examples of io.airlift.stats.cardinality.HyperLogLog


    @InputFunction
    @IntermediateInputFunction
    public static void merge(HyperLogLogState state, @SqlType(StandardTypes.HYPER_LOG_LOG) Slice value)
    {
        HyperLogLog input = HyperLogLog.newInstance(value);


        HyperLogLog previous = state.getHyperLogLog();
        if (previous == null) {
            state.setHyperLogLog(input);
            state.addMemoryUsage(input.estimatedInMemorySize());
        }
        else {
            state.addMemoryUsage(-previous.estimatedInMemorySize());
            previous.mergeWith(input);
            state.addMemoryUsage(previous.estimatedInMemorySize());
        }
    }

View Full Code Here

Examples of io.airlift.stats.cardinality.HyperLogLog


    @ScalarFunction
    @SqlType(HyperLogLogType.class)
    public static Slice createHll(@SqlType(BigintType.class) long value)
    {
        HyperLogLog hll = HyperLogLog.newInstance(4096);
        hll.add(value);
        return hll.serialize();
    }

View Full Code Here

Examples of io.airlift.stats.cardinality.HyperLogLog


    @ScalarFunction
    @SqlType(HyperLogLogType.class)
    public static Slice createHll(@SqlType(BigintType.class) long value)
    {
        HyperLogLog hll = HyperLogLog.newInstance(4096);
        hll.add(value);
        return hll.serialize();
    }

View Full Code Here

0 1 2

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.