Examples of com.taobao.top.analysis.statistics.data.ReportEntry

Package com.taobao.top.analysis.statistics.data

Examples of com.taobao.top.analysis.statistics.data.ReportEntry

com.taobao.top.analysis.statistics.data.ReportEntry
SimpleMapReduce Entry @author fangweng

    int emptyLine=0;//拉取空行的次数
    int exceptionLine=0;//一行中，只要entry有异常，该行就是存在异常的行。
    int size = 0;
    String record;
    
    ReportEntry entry = null;
    
    long beg = System.currentTimeMillis();
    
    try 
    {
      //逐行处理
      while ((record = reader.readLine()) != null) 
      {
        boolean failure=false;
        
        try
        {
          if (record == null || "".equals(record)) 
          {
            emptyLine++;
            continue;
          }
          
          size += record.getBytes().length;
          
          String[] contents = StringUtils.splitByWholeSeparatorPreserveAllTokens(record, splitRegex);
          Iterator<String> keys = entryPool.keySet().iterator();
          while (keys.hasNext()) 
          {
            try 
            {
              String key = keys.next();
              entry = entryPool.get(key);
              if(!entry.isLazy()){
                  processSingleLine(entry, contents,jobtask,jobTaskResult, taskExecuteInfo);
//                if(!processSingleLine(entry, contents,jobtask,jobTaskResult, taskExecuteInfo)) {
//                    if(entry.getReports().contains("appAuthReport"))
//                        logger.error("key null, record:" + record);
//                }
              }
              
            } 
            catch (Throwable e) 
            {
                            if (!failure)
                                exceptionLine++;


                            failure = true;


                             if (!threshold.sholdBlock())
                            logger.error(
                                new StringBuilder().append("Entry :").append(entry.getId()).append(", job : ")
                                    .append(jobtask.getJobName()).append(", entry:").append(entry.getName())
                                    .append("\r\n record: ").append(record).toString(), e);
                        }
          }


          if(!failure)

View Full Code Here

        
        //输出title
        if(needTitle)
          for(int i =0 ; i < rs.size(); i++)
          {
            ReportEntry entry = report.getReportEntrys().get(i);
            bout.write(entry.getName());
            
            if (i == rs.size() -1)
              bout.write("\r\n");
            else
              bout.write(",");
            
          }
        
        //按行开始输出内容
        for(int i = 0 ; i < rs.size(); i++)
        {
          ReportEntry entry = report.getReportEntrys().get(i);
          
          Map<String, Object> m = entryResultPool.get(entry.getId());
          
          if (m == null || (m != null && m.size() == 0))
            continue;
          
          Iterator<String> iter = m.keySet().iterator();
          
          while(iter.hasNext())
          {
            String key = iter.next();
            
            // 作average的中间临时变量不处理
            if (key.startsWith(AnalysisConstants.PREF_SUM)
                || key.startsWith(AnalysisConstants.PREF_COUNT)) {
              continue;
            }
            
            boolean needProcess = true;
            
            //判断是否前面已经有输出
            for(int j = 0; j < i; j++)
            {
              if (entryResultPool.get(report.getReportEntrys().get(j).getId())  != null
                  && entryResultPool.get(report.getReportEntrys().get(j).getId()).containsKey(key))
              {
                needProcess = false;
                break;
              }
            }
            
            if (needProcess)
            {
              for(int j = 0 ; j < i ; j++)
              {
                bout.write("0,");
              }
              
              for(int j = i ; j < rs.size(); j++)
              {
                
                ReportEntry tmpEntry = report.getReportEntrys().get(j);
                
                Object value = null;
                
                if (entryResultPool.get(tmpEntry.getId()) != null)
                  value = entryResultPool.get(tmpEntry.getId()).get(key);
                List<String> formatStack =((SimpleFilter) tmpEntry.getValueFilter()).getFormatStack();
                if (value != null && formatStack != null
                    && formatStack.size() > 0) {
                  value = ReportUtil.formatValue(
                      formatStack, value);
                }

View Full Code Here

            }
            if (multiMap.size() == 0) {
                return AnalysisConstants.IGNORE_PROCESS;
            }
            MapAndParam generateKeyMap = multiMap.get(0);
            ReportEntry rEntry = null;
            for (int i = 1, n = multiMap.size(); i < n; i++) {
                try {
                    rEntry = entry.clone();
                }
                catch (CloneNotSupportedException e) {
                    logger.error(e);
                }
                rEntry.setMapClass(multiMap.get(i).getMapInstance());
                rEntry.setMapParams(multiMap.get(i).getMapParam());
                if (AnalysisConstants.IGNORE_PROCESS.equals(multiMap.get(i).getMapInstance()
                    .generateKey(rEntry, contents, jobtask))) {
                    return AnalysisConstants.IGNORE_PROCESS;
                }
            }
            try {
                rEntry = entry.clone();
            }
            catch (CloneNotSupportedException e) {
                logger.error(e);
            }
            rEntry.setMapClass(generateKeyMap.getMapInstance());
            rEntry.setMapParams(generateKeyMap.getMapParam());
            String key =
                    generateKeyMap.getMapInstance().generateKey(rEntry, contents, jobtask);
            return key;
        }
    }

View Full Code Here

            }
            if (multiMap.size() == 0) {
                return super.generateValue(entry, contents, jobtask);
            }
            MapAndParam generateKeyMap = multiMap.get(0);
            ReportEntry rEntry = null;
            try {
                rEntry = entry.clone();
            }
            catch (CloneNotSupportedException e) {
                logger.error(e);
            }
            rEntry.setMapClass(generateKeyMap.getMapInstance());
            rEntry.setMapParams(generateKeyMap.getMapParam());
            return generateKeyMap.getMapInstance().generateValue(rEntry, contents, jobtask);
        }
    }

View Full Code Here

            continue;
          }


          if (tag.equalsIgnoreCase("ReportEntry")
              || tag.equalsIgnoreCase("entry")) {
            ReportEntry entry = new ReportEntry();
            if (tag.equalsIgnoreCase("ReportEntry"))
              setReportEntry(true, start, entry, report,
                  rule.getEntryPool(), rule.getAliasPool(),
                  globalConditions, globalValuefilter,
                  globalMapClass, parents);
            else {
              setReportEntry(false, start, entry, report,
                  rule.getEntryPool(), rule.getAliasPool(),
                  globalConditions, globalValuefilter,
                  globalMapClass, parents);
            }


            if (entry.getId() != null) {
              if (rule.getEntryPool().get(entry.getId()) != null)
                throw new java.lang.RuntimeException(
                    "ID confict:" + entry.getId());
              rule.getEntryPool().put(entry.getId(), entry);
            }


            // 增加引用标识
            if (tag.equalsIgnoreCase("entry")) {
              if (entry.getId() != null)
                rule.getReferEntrys().put(entry.getId(), entry);
              else if (report.getReportEntrys() != null
                  && report.getReportEntrys().size() > 0)
                rule.getReferEntrys().put(
                    report.getReportEntrys()
                        .get(report.getReportEntrys()
                            .size() - 1).getId(),
                    report.getReportEntrys()
                        .get(report.getReportEntrys()
                            .size() - 1));
            }


            ReportEntry _tmpEntry = entry;


            if (_tmpEntry.getId() == null
                && report.getReportEntrys() != null
                && report.getReportEntrys().size() > 0)
              _tmpEntry = report.getReportEntrys().get(
                  report.getReportEntrys().size() - 1);
            
            List<Object> bindingStack = ((SimpleCalculator)_tmpEntry.getCalculator()).getBindingStack();
            String valueExpression = ((SimpleCalculator)_tmpEntry.getCalculator()).getValue();
            if (bindingStack != null) {
              if (valueExpression != null
                  && valueExpression.indexOf(
                      "entry(") >= 0)
                for (Object k : bindingStack) {
                  rule.getReferEntrys().put((String)k, null);
                }
            }


            continue;
          }


          if (tag.equalsIgnoreCase("report")) {
            if (report != null) {
                if(report.getId() != null) {
                    if(rule.getReportPool().get(report.getId()) != null)
                        throw new java.lang.RuntimeException(
                                        "ID confict:" + report.getId());
                }
              rule.getReportPool().put(report.getId(), report);
            }


            report = new Report();
            setReport(start, report, rule.getReportPool());
            continue;
          }


          if (tag.equalsIgnoreCase("entryList")) {
            report.setReportEntrys(new ArrayList<ReportEntry>());
            continue;
          }


        }


        if (event.isEndElement()) {
          EndElement end = event.asEndElement();


          String tag = end.getName().getLocalPart();


          if (tag.equalsIgnoreCase("reports") && report != null) {
            rule.getReportPool().put(report.getId(), report);
            continue;
          }


        }


      }
      // 给刚刚记录的parent Entry对象打上标识
      for (Iterator<String> iterator = parents.iterator(); iterator
          .hasNext();) {
        String parent = iterator.next();
        ReportEntry parentEntry = rule.getEntryPool().get(parent);
        rule.getParentEntryPool().put(parent, parentEntry);
      }


      // 删除没有被引用的公用的定义
            // 考虑到每一个job（也就是过去的instance的概念）对应于同一个rule，并非是一个配置文件对应一个rule，因此

View Full Code Here

    if ((!isPublic && start.getAttributeByName(new QName("", "id")) != null
        && start.getAttributeByName(new QName("", "name")) == null) 
        ||(start.getAttributeByName(new QName("", "refId")) != null))
    {


      ReportEntry node = entryPool.get(start.getAttributeByName(
          new QName("", "id")).getValue());


      if (node != null) {
        report.getReportEntrys().add(node);
        
        //给node增加report的属性
        node.addReport(report);
      } else {
        String errorMsg = new StringBuilder()
            .append("reportEntry not exist :")
            .append(start.getAttributeByName(new QName("", "id"))
                .getValue()).toString();

View Full Code Here

      
      Set<String> masters = jobTask.getStatisticsRule().getMasters();
      
      for(String entryId : _entryResults.keySet())
      {
        ReportEntry reportEntry = rule.getEntryPool().get(entryId);
        Set<String> reports = reportEntry.getReports();
        if(reports == null || reports.size() == 0) {
            String master = config.getMasterAddress()+":"+config.getMasterPort();
            logger.error("reportEntry has no report " + reportEntry.getId());
            if (_masterEntryResults.get(master) == null)
                    {
                        _masterEntryResults.put(master, new HashMap<String,Map<String,Object>>());
                    }

View Full Code Here

      Iterator<String> entryKeys = entryPool.keySet().iterator();


      while (entryKeys.hasNext()) {
        String entryId = entryKeys.next();


        ReportEntry entry = entryPool.get(entryId);
        if (entry.isLazy()) {
          Map<String, Object> t = result.remove(entryId);


          if (t != null)
            t.clear();
        }

View Full Code Here

            Iterator<String> entryKeys = entryPool.keySet().iterator();


            while (entryKeys.hasNext()) {
                String entryId = entryKeys.next();


                ReportEntry entry = entryPool.get(entryId);


                if (entry.isPeriod()) {
                    Map<String, Object> t = result.remove(entryId);


                    if (t != null)
                        t.clear();
                }
                //遍历删除平均值计算
                if(entry.getGroupFunction() != null && entry.getGroupFunction() instanceof AvgFunction) {
                    if(result.get(entryId) == null)
                        continue;
                    Iterator<Map.Entry<String, Object>> entries = result.get(entryId).entrySet().iterator();
                    while(entries.hasNext()) {
                        Entry<String, Object> en = entries.next();

View Full Code Here

    // 二级lazy的优先顺序保证
    Collections.sort(entryKeys);


    for (String entryId : entryKeys) {


      ReportEntry entry = entryPool
          .get(entryId);
      if (entry.isLazy()) 
      {
        if (result.get(entryId) == null)
          result.put(entryId, new HashMap<String, Object>());
        SimpleCalculator c = (SimpleCalculator)entry.getCalculator();
        if (c.getBindingStack() != null
            && c.getBindingStack().size() > 0) {
          List<Object> _bindingStack = c.getBindingStack();
          int size = _bindingStack.size();


          String leftEntryId = (String) _bindingStack.get(0);


          Map<String, Object> leftMap = result.get(leftEntryId);


          if (leftMap == null
              || (leftMap != null && leftMap.size() <= 0)) {
            continue;
          }
          Iterator<String> iter = leftMap.keySet().iterator();
          java.util.Map<String, Double> cacheMap = new java.util.HashMap<String, Double>();
          while (iter.hasNext()) {
            try {
              String nodekey = iter.next();
              Object nodevalue = result.get(leftEntryId).get(
                  nodekey);
              Object rightvalue = null;


              for (int i = 0; i < size - 1; i++) {
                String rightkey = (String) _bindingStack
                    .get(i + 1);


                if (rightkey.startsWith("sum:")) {
                  rightkey = rightkey.substring(rightkey
                      .indexOf("sum:") + "sum:".length());
                  double sumValue = 0;
                  if (cacheMap.get(rightkey) != null) {
                    sumValue = cacheMap.get(rightkey);


                  } else {
                    Iterator<Object> rValues = result
                        .get(rightkey).values()
                        .iterator();


                    while (rValues.hasNext()) {
                      Object rv = rValues.next();


                      if (rv != null) {
                        if (rv instanceof String)
                          sumValue += Double
                              .valueOf((String) rv);
                        else {
                          sumValue += (Double) rv;
                        }
                      }


                    }
                    cacheMap.put(rightkey, sumValue);
                  }


                  rightvalue = sumValue;


                } else {
                  // 简单实现只支持两位计算，同时是+或者-
                  if (rightkey
                      .indexOf(AnalysisConstants.OPERATE_PLUS_CHAR) > 0
                      || rightkey
                          .indexOf(AnalysisConstants.OPERATE_MINUS_CHAR) > 0) {


                    String l;
                    String r;


                    if (rightkey
                        .indexOf(AnalysisConstants.OPERATE_PLUS_CHAR) > 0) {
                      l = rightkey
                          .substring(
                              0,
                              rightkey.indexOf(AnalysisConstants.OPERATE_PLUS_CHAR))
                          .trim();


                      r = rightkey
                          .substring(
                              rightkey.indexOf(AnalysisConstants.OPERATE_PLUS_CHAR) + 1)
                          .trim();


                      if (result.get(l) == null
                          || result.get(r) == null
                          || (result.get(l) != null && result
                              .get(l)
                              .get(nodekey) == null)
                          || (result.get(r) != null && result
                              .get(r)
                              .get(nodekey) == null))
                        continue;


                      rightvalue = Double.valueOf(result
                          .get(l).get(nodekey)
                          .toString())
                          + Double.valueOf(result
                              .get(r)
                              .get(nodekey)
                              .toString());
                    } else {
                      l = rightkey
                          .substring(
                              0,
                              rightkey.indexOf(AnalysisConstants.OPERATE_MINUS_CHAR))
                          .trim();


                      r = rightkey
                          .substring(
                              rightkey.indexOf(AnalysisConstants.OPERATE_MINUS_CHAR) + 1)
                          .trim();


                      if (result.get(l) == null
                          || result.get(r) == null
                          || (result.get(l) != null && result
                              .get(l)
                              .get(nodekey) == null)
                          || (result.get(r) != null && result
                              .get(r)
                              .get(nodekey) == null))
                        continue;


                      rightvalue = Double.valueOf(result
                          .get(l).get(nodekey)
                          .toString())
                          - Double.valueOf(result
                              .get(r)
                              .get(nodekey)
                              .toString());
                    }


                                    }
                                    else {
                                        try {
                                            rightvalue = result.get(rightkey).get(nodekey);
                                        }
                                        catch (Throwable e) {
                                            if(!threshold.sholdBlock())
                                                logger.error("resultkey is null" + rightkey, e);
                                        }
                                    }
                }


                if (rightvalue != null) {
                  if (nodevalue != null) {
                    if (c.getOperatorStack().get(i) == AnalysisConstants.OPERATE_PLUS) {
                      if (nodevalue instanceof Double
                          || rightvalue instanceof Double)
                        nodevalue = Double
                            .valueOf(nodevalue
                                .toString())
                            + Double.valueOf(rightvalue
                                .toString());
                      else
                        nodevalue = (Long) nodevalue
                            + (Long) rightvalue;


                      continue;
                    }


                    if (c.getOperatorStack().get(i) == AnalysisConstants.OPERATE_MINUS) {
                      if (nodevalue instanceof Double
                          || rightvalue instanceof Double)
                        nodevalue = Double
                            .valueOf(nodevalue
                                .toString())
                            - Double.valueOf(rightvalue
                                .toString());
                      else
                        nodevalue = (Long) nodevalue
                            - (Long) rightvalue;


                      continue;
                    }


                    if (c.getOperatorStack().get(i) == AnalysisConstants.OPERATE_RIDE) {
                      if (nodevalue instanceof Double
                          || rightvalue instanceof Double)
                        nodevalue = Double
                            .valueOf(nodevalue
                                .toString())
                            * Double.valueOf(rightvalue
                                .toString());
                      else
                        nodevalue = (Long) nodevalue
                            * (Long) rightvalue;


                      continue;
                    }


                    if (c.getOperatorStack().get(i) == AnalysisConstants.OPERATE_DIVIDE) {
                      nodevalue = Double
                          .valueOf(nodevalue
                              .toString())
                          / Double.valueOf(rightvalue
                              .toString());


                      continue;
                    }
                  } else {
                    // nodevalue=rightvalue;
                  }


                }


              }


              result.get(entryId).put(nodekey, nodevalue);
            } catch (Exception ex) {


              logger.error(
                  new StringBuilder("entry : ").append(
                      entry.getName()).append(
                      " lazy process error!"), ex);
              continue;
            }


          }
          cacheMap.clear();
        }


      }
      else
        //处理average的情况
        if (entry.getGroupFunction() instanceof AvgFunction)
        {
          Map<String,Object> av = result.get(entryId);
          Map<String,Object> tv = new HashMap<String,Object>();
          
          if (av != null)

View Full Code Here

0 1

TOP

Related Classes of com.taobao.top.analysis.statistics.data.ReportEntry

com.taobao.top.analysis.node.component.FileJobBuilder

com.taobao.top.analysis.node.component.SlaveNode

com.taobao.top.analysis.node.operation.CreateReportOperation

com.taobao.top.analysis.statistics.map.MultiConditionMapper

com.taobao.top.analysis.statistics.StatisticsEngine

com.taobao.top.analysis.util.ReportUtil

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.