Package brickhouse.analytics.uniques

Examples of brickhouse.analytics.uniques.SketchSet

61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
    Object obj = arg0[0].get();
    if(obj == null) {
      return null;
    }
    List oldList = listInspector.getList(obj);
    SketchSet sketchSet  = new SketchSet( sketchSetSize);
    for( Object oldObj : oldList) {
      if( oldObj == null) {
        LOG.warn(" Object in uninspected List is null");
      } else {
        String newStr = listElemInspector.getPrimitiveJavaObject(oldObj);
        sketchSet.addItem(newStr);
      }
    }
    return sketchSet.getMinHashItems();
  }
View Full Code Here
12
13
14
15
16
17
18
19
20
21
 

  public void init(int size) {
    if( sketchSet == null ||
        ((sketchSet.getMaxItems() != size) && (size != -1))) {
      sketchSet = new SketchSet( size);
    } else {
      sketchSet.clear();
    }
  }
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
  @Override
  public Object evaluate(DeferredObject[] args) throws HiveException {
    String grouping = this.groupInspector.getPrimitiveJavaObject( args[0].get());
    if( lastGrouping == null || !lastGrouping.equals( grouping)) {
      lastGrouping = grouping;
      prevValue = new SketchSet();
    }
    List<String> prevHashItems = prevValue.getMinHashItems();
   
   
    List newList = listInspector.getList( args[1].get());
View Full Code Here
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
    ///  as an argument
    int sketchSize = Math.max( a.size() , b.size() );
    if( sketchSize < SketchSetUDAF.DEFAULT_SKETCH_SET_SIZE)
        sketchSize = SketchSetUDAF.DEFAULT_SKETCH_SET_SIZE;
   
    SketchSet sketchA = new SketchSet(sketchSize);
    SketchSet sketchB = new SketchSet(sketchSize);
    SketchSet sketchAUB = new SketchSet(sketchSize);
   
   
    for(String aStr : a) {
      sketchA.addItem( aStr);
      sketchAUB.addItem( aStr);
    }
    for(String bStr : b) {
      sketchB.addItem( bStr);
      sketchAUB.addItem( bStr);
    }
   
    double aEst = sketchA.estimateReach();
    double bEst = sketchB.estimateReach();
    double aubEst = sketchAUB.estimateReach();
   
    /// Intersection is
    double ainterb =  aEst + bEst - aubEst;
    double sim = ainterb/aubEst;
   
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
    if( obj == null) {
      return null;
    }
    List oldList = listInspector.getList(obj);
    int sketchSize = listInspector.getListLength( obj);
    SketchSet sketchSet = new SketchSet(sketchSize );
    for( Object oldObj : oldList) {
      if( oldObj == null) {
        LOG.warn(" Object in uninspected List is null");
      } else {
        String newStr = listElemInspector.getPrimitiveJavaObject(oldObj);
        if(newStr == null)
          LOG.warn(" inspected object is null !!! ");
        else
            sketchSet.addItem( newStr);
       
      }
    }
    return sketchSet.getMinHashes();
  }
View Full Code Here
286
287
288
289
290
291
292
293
      MultiDaySketchBuffer countBuff = (MultiDaySketchBuffer) buff;
      if (daysArr != null) {
        countBuff.counts = new long[daysArr.length];
          countBuff.sketches = new SketchSet[daysArr.length];
          for (int i = 0; i < countBuff.sketches.length; ++i)
            countBuff.sketches[i] = new SketchSet();
      }
    }
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
  private PrimitiveCategory elemCategory;
  private int sketchSetSize = SketchSetUDAF.DEFAULT_SKETCH_SET_SIZE;
 
  @Override
  public Object evaluate(DeferredObject[] arg0) throws HiveException {
    SketchSet ss = new SketchSet(sketchSetSize);
    for( int i=0; i< arg0.length; ++i) {
      Object listObj = arg0[i].get();
      int listLen = listInspectors[i].getListLength(listObj);
      for(int j=0; j< listLen; ++j ) {
           Object uninspObj = listInspectors[i].getListElement(listObj, j);
           switch( elemCategory) {
           case STRING:
             StringObjectInspector strInspector = (StringObjectInspector) listInspectors[i].getListElementObjectInspector();
             String item = strInspector.getPrimitiveJavaObject(uninspObj);
             ss.addItem(item);
             break;
           case LONG:
             LongObjectInspector bigintInspector = (LongObjectInspector) listInspectors[i].getListElementObjectInspector();
             long itemHash = bigintInspector.get(uninspObj);
             ss.addHash( itemHash);
             break;
           }
      }
    }
      switch( elemCategory) {
        case STRING:
          return ss.getMinHashItems();
        case LONG:
          return ss.getMinHashes();
        default:
          /// will never happen
          throw new HiveException("Unexpected Element Category " + elemCategory);
      }
  }
View Full Code Here
TOP

Related Classes of brickhouse.analytics.uniques.SketchSet

  • brickhouse.udf.sketch.CombinePreviousSketchUDF
  • brickhouse.udf.sketch.CombineSketchUDF
  • brickhouse.udf.sketch.ConvertToSketchUDF
  • brickhouse.udf.sketch.MultiDaySketcherUDAF$MultiDayAggUDAFEvaluator
  • brickhouse.udf.sketch.SetSimilarityUDF
  • brickhouse.udf.sketch.SketchHashesUDF
  • brickhouse.udf.sketch.SketchSetBuffer
  • com.google.common.hash.HashCode
  • java.math.BigDecimal
  • java.util.ArrayList
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.