Package io.airlift.stats.cardinality

Examples of io.airlift.stats.cardinality.HyperLogLog

25
26
27
28
29
30
31
32
33
    @ScalarFunction
    @SqlType(StandardTypes.HYPER_LOG_LOG)
    public static Slice createHll(@SqlType(StandardTypes.BIGINT) long value)
    {
        HyperLogLog hll = HyperLogLog.newInstance(4096);
        hll.add(value);
        return hll.serialize();
    }
View Full Code Here
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
    }

    @Override
    protected void processInput(HyperLogLogState state, Block block, int index)
    {
        HyperLogLog input = HyperLogLog.newInstance(block.getSlice(index));

        HyperLogLog previous = state.getHyperLogLog();
        if (previous == null) {
            state.setHyperLogLog(input);
            state.addMemoryUsage(input.estimatedInMemorySize());
        }
        else {
            state.addMemoryUsage(-previous.estimatedInMemorySize());
            previous.mergeWith(input);
            state.addMemoryUsage(previous.estimatedInMemorySize());
        }
    }
View Full Code Here
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
    }

    @Override
    protected void processInput(HyperLogLogState state, Block block, int index)
    {
        HyperLogLog hll = state.getHyperLogLog();
        if (hll == null) {
            hll = HyperLogLog.newInstance(NUMBER_OF_BUCKETS);
            state.setHyperLogLog(hll);
            state.addMemoryUsage(hll.estimatedInMemorySize());
        }

        state.addMemoryUsage(-hll.estimatedInMemorySize());
        add(block, index, parameterType, hll);
        state.addMemoryUsage(hll.estimatedInMemorySize());
    }
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
    }

    @Override
    protected void combineState(HyperLogLogState state, HyperLogLogState otherState)
    {
        HyperLogLog input = otherState.getHyperLogLog();

        HyperLogLog previous = state.getHyperLogLog();
        if (previous == null) {
            state.setHyperLogLog(input);
            state.addMemoryUsage(input.estimatedInMemorySize());
        }
        else {
            state.addMemoryUsage(-previous.estimatedInMemorySize());
            previous.mergeWith(input);
            state.addMemoryUsage(previous.estimatedInMemorySize());
        }
    }
View Full Code Here
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
                // skip null values
                if (!values.isNull() && (masks == null || masks.getBoolean())) {
                    long groupId = groupIdsBlock.getGroupId(position);

                    HyperLogLog hll = estimators.get(groupId);
                    if (hll == null) {
                        hll = HyperLogLog.newInstance(NUMBER_OF_BUCKETS);
                        estimators.set(groupId, hll);
                        sizeOfValues += hll.estimatedInMemorySize();
                    }

                    sizeOfValues -= hll.estimatedInMemorySize();
                    add(values, parameterType, hll);
                    sizeOfValues += hll.estimatedInMemorySize();
                }
            }
            checkState(!values.advanceNextPosition(), "group id and value blocks have different number of entries");
        }
View Full Code Here
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
                // skip null values
                if (!intermediates.isNull()) {
                    long groupId = groupIdsBlock.getGroupId(position);

                    HyperLogLog input = HyperLogLog.newInstance(intermediates.getSlice());

                    HyperLogLog previous = estimators.get(groupId);
                    if (previous == null) {
                        estimators.set(groupId, input);
                        sizeOfValues += input.estimatedInMemorySize();
                    }
                    else {
                        sizeOfValues -= previous.estimatedInMemorySize();
                        previous.mergeWith(input);
                        sizeOfValues += previous.estimatedInMemorySize();
                    }
                }
            }
            checkState(!intermediates.advanceNextPosition());
        }
View Full Code Here
241
242
243
244
245
246
247
248
249
250
251
252
253
        }

        @Override
        public void evaluateFinal(int groupId, BlockBuilder output)
        {
            HyperLogLog estimator = estimators.get(groupId);
            if (estimator == null) {
                output.appendNull();
            }
            else {
                output.appendSlice(estimator.serialize());
            }
        }
View Full Code Here
102
103
104
105
106
107
108
109
110
111
112
            BlockCursor intermediates = block.cursor();

            for (int position = 0; position < block.getPositionCount(); position++) {
                checkState(intermediates.advanceNextPosition());
                if (!intermediates.isNull()) {
                    HyperLogLog instance = HyperLogLog.newInstance(intermediates.getSlice());

                    if (estimator == null) {
                        estimator = instance;
                    }
                    else {
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
            }
        }

        private void add(BlockCursor cursor)
        {
            HyperLogLog instance = HyperLogLog.newInstance(cursor.getSlice());

            if (estimator == null) {
                estimator = instance;
            }
            else {
View Full Code Here
141
142
143
144
145
146
147
148
149
150
151
152
153
        }

        @Override
        public void evaluateFinal(int groupId, BlockBuilder output)
        {
            HyperLogLog estimator = estimators.get(groupId);
            if (estimator == null) {
                output.appendNull();
            }
            else {
                output.appendSlice(estimator.serialize());
            }
        }
View Full Code Here
TOP

Related Classes of io.airlift.stats.cardinality.HyperLogLog

  • com.facebook.presto.operator.aggregation.ApproximateSetAggregation
  • com.facebook.presto.operator.aggregation.ApproximateSetAggregation$ApproximateSetAccumulator
  • com.facebook.presto.operator.aggregation.ApproximateSetAggregation$ApproximateSetGroupedAccumulator
  • com.facebook.presto.operator.aggregation.MergeHyperLogLogAggregation
  • com.facebook.presto.operator.aggregation.MergeHyperLogLogAggregation$MergeHyperLogLogAccumulator
  • com.facebook.presto.operator.aggregation.MergeHyperLogLogAggregation$MergeHyperLogLogGroupedAccumulator
  • com.facebook.presto.operator.scalar.CreateHll
  • com.facebook.presto.tests.CreateHll
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.