Package com.clearspring.analytics.stream.cardinality

Examples of com.clearspring.analytics.stream.cardinality.ICardinality

58
59
60
61
62
63
64
65
66
67
68
69
   
      if(bref1 != null && bref2 != null) {
        HyperLogLogPlus hll1 = HyperLogLogPlus.Builder.build( bref1 );
        HyperLogLogPlus hll2 = HyperLogLogPlus.Builder.build( bref2 );
     
        ICardinality merged = hll1.merge(hll2);
      return merged.getBytes();
      } else {
        return null;
      }
     
    } catch(Exception e) {
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
      CardinalityMergeException {
    if (buffer == null) {
      return;
    }
   
    ICardinality other = HyperLogLogPlus.Builder.build(buffer);
   
    // if hll estimator hasn't been allocated yet, just set it equal to the partial
    if (hll == null) {
      LOG.debug("hll is null; other.sizeof = " + other.sizeof());
      hll = other;
      precision = (int) Math.ceil(Math.log(other.sizeof())/Math.log(2.0));
      LOG.debug("precision set to: " + precision);
    } else {
      hll.merge(other);
    }
  }
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
                System.err.print("Bad update rate: '" + args[0] + "'  Update rate must be an integer.");
                usage();
            }
        }

        ICardinality card = AdaptiveCounting.Builder.obyCount(Integer.MAX_VALUE).build();

        BufferedReader in = new BufferedReader(new InputStreamReader(System.in));

        String line = null;
        while ((line = in.readLine()) != null)
        {
            card.offer(line);
            count++;

            if (updateRate > 0 && count % updateRate == 0)
            {
                System.out.println(formatSummary(count, card.cardinality()));
            }
        }

        System.out.println(formatSummary(count, card.cardinality()));
    }
View Full Code Here
74
75
76
77
78
79
80
81
82
83
84
  }

  @Override
  public ICardinality finalAggregate(ICardinality aggregate, TupleEntry partialAggregate) {
    try {
      ICardinality hll = HyperLogLogPlus.Builder.build(Bytes.getBytes((BytesWritable) partialAggregate.getObject(0)));
      aggregate.merge(hll);
      return aggregate;
    } catch (CardinalityMergeException cme) {
      throw new RuntimeException(cme);
    } catch (IOException e) {
View Full Code Here
TOP

Related Classes of com.clearspring.analytics.stream.cardinality.ICardinality

  • brickhouse.udf.hll.CombineHyperLogLogUDF
  • brickhouse.udf.hll.HLLBuffer
  • com.clearspring.analytics.util.ObyCount
  • com.liveramp.cascading_ext.combiner.lib.HyperLogLogExactAggregator
  • org.apache.cassandra.io.sstable.format.SSTableReader
  • org.apache.cassandra.io.sstable.metadata.CompactionMetadata$CompactionMetadataSerializer
  • org.apache.cassandra.io.sstable.SSTableReader
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.