Package com.taobao.top.analysis.statistics.data

Examples of com.taobao.top.analysis.statistics.data.DistinctCountEntryValue


          for (Integer k : groupby)
          {
            key.append(contents[k]).append("--");
          }
         
          DistinctCountEntryValue distinctEntry = result.get(key.toString());
         
          if (distinctEntry == null)
          {
            distinctEntry = new DistinctCountEntryValue();
            ByteBloomFilter bloomFilter;
           
            bloomFilter = new ByteBloomFilter(maxKeys,errorRate,1);
            distinctEntry.setBloomFilter(bloomFilter);
            result.put(key.toString(), distinctEntry);
          }
         
          distinctEntry.add(contents[distinctColumn]);
        }
        catch(Exception ex)
        {
          System.out.print(ex.getCause());
        }
View Full Code Here


   * @throws InterruptedException
   * @throws UnsupportedEncodingException
   */
  public static void main(String[] args) throws InterruptedException, UnsupportedEncodingException {

    DistinctCountEntryValue distinctEntry = new DistinctCountEntryValue();
    ByteBloomFilter bloomFilter = new ByteBloomFilter(10000000,0.0001F,1);
    distinctEntry.setBloomFilter(bloomFilter);
   
    String[][] content = new String[1000][100];
    byte[][][] content1 = new byte[1000][100][];
   
    for(int i =0 ; i < 1000; i++)
      for(int j =0 ; j < 100; j++)
      {
        if ( i > 900 || j > 90)
        {
          content[i][j] = "hello00";
          content1[i][j] = "hello00".getBytes("UTF-8");
        }
        else
        {
          content[i][j] = new StringBuilder("hello").append(i).append(j).toString();
          content1[i][j] = new StringBuilder("hello").append(i).append(j).toString().getBytes("UTF-8");
        }
         
      }
   
    Map <String,String> map = new HashMap<String,String>();
   
   
   
    long beg = System.currentTimeMillis();
   
    for(int i =0 ; i < 1000 ; i++)
    {
      for(int j = 0 ; j < 100; j++)
      {
        //map.put(content[i][j], content[i][j]);
        distinctEntry.add(content1[i][j]);
      }
    }
   
//    System.out.println("count: " + String.valueOf(map.size()));
    System.out.println("count: " + String.valueOf(distinctEntry.getCount()));
    System.out.println("bytesize : " + String.valueOf(distinctEntry.getBloomFilter().getByteSize()));
    System.out.println("已使用内存:" + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / (1024*1024)+"M/");
    System.out.println("消耗时间:" + String.valueOf(System.currentTimeMillis() - beg));
   
    Thread.sleep(2000);
    map.clear();
View Full Code Here

    }
    else
    {
      //一种情况是result里面还是原生态的数据,不是bloom过滤器,则需要构建bloom过滤器   
      String nkey = key.substring(AnalysisConstants.MAGIC_NUM.length(), key.length() - value.toString().length());
      DistinctCountEntryValue distinctEntry = (DistinctCountEntryValue)result.get(nkey);
     
      if (distinctEntry == null)
      {
        distinctEntry = createDCEntryValue(entry);
        result.put(nkey, distinctEntry);
      }
     
      try
      {
        //存在一定危险性,key与nkey冲突
        if (result.get(key) != null && !(result.get(key) instanceof DistinctCountEntryValue))
        {
          distinctEntry.add(result.get(key).toString());
          result.remove(key);
        }
       
        distinctEntry.add(value.toString());
      }
      catch(Exception ex)
      {
        logger.error(ex);
      }
View Full Code Here

   
  }
 
  DistinctCountEntryValue createDCEntryValue(ReportEntry entry)
  {
    DistinctCountEntryValue distinctEntryValue = new DistinctCountEntryValue();
    ByteBloomFilter bloomFilter;
   
    //240k
    int maxKeys = 100000;
    float errorRate = 0.0001F;
   
    if (entry.getAdditions().get(AnalysisConstants.ANALYSIS_BLOOM_MAXKEYS) != null)
      maxKeys = (Integer)entry.getAdditions().get(AnalysisConstants.ANALYSIS_BLOOM_MAXKEYS);
   
    if (entry.getAdditions().get(AnalysisConstants.ANALYSIS_BLOOM_ERRORRATE) != null)
      errorRate = (Float)entry.getAdditions().get(AnalysisConstants.ANALYSIS_BLOOM_ERRORRATE);
     
    bloomFilter = new ByteBloomFilter(maxKeys,errorRate,1);
   
    distinctEntryValue.setBloomFilter(bloomFilter);
   
    return distinctEntryValue;
  }
View Full Code Here

TOP

Related Classes of com.taobao.top.analysis.statistics.data.DistinctCountEntryValue

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.