Package org.apache.hadoop.mapreduce

Examples of org.apache.hadoop.mapreduce.CounterGroup


    @Override
    protected void handleFailure(Counters counters) throws IOException {
      Configuration conf = job.getConfiguration();
      HConnection conn = HConnectionManager.getConnection(conf);
      TableName tableName = TableName.valueOf(COMMON_TABLE_NAME);
      CounterGroup g = counters.getGroup("undef");
      Iterator<Counter> it = g.iterator();
      while (it.hasNext()) {
        String keyString = it.next().getName();
        byte[] key = Bytes.toBytes(keyString);
        HRegionLocation loc = conn.relocateRegion(tableName, key);
        LOG.error("undefined row " + keyString + ", " + loc);
      }
      g = counters.getGroup("unref");
      it = g.iterator();
      while (it.hasNext()) {
        String keyString = it.next().getName();
        byte[] key = Bytes.toBytes(keyString);
        HRegionLocation loc = conn.relocateRegion(tableName, key);
        LOG.error("unreferred row " + keyString + ", " + loc);
View Full Code Here


        "Reduce Value",
        "Total Value"));
    buff.append("\n------------------------------------------"+
        "---------------------------------------------");
    for (String groupName : totalCounters.getGroupNames()) {
         CounterGroup totalGroup = totalCounters.getGroup(groupName);
         CounterGroup mapGroup = mapCounters.getGroup(groupName);
         CounterGroup reduceGroup = reduceCounters.getGroup(groupName);
     
         Format decimal = new DecimalFormat();
         Iterator<org.apache.hadoop.mapreduce.Counter> ctrItr =
           totalGroup.iterator();
         while(ctrItr.hasNext()) {
           org.apache.hadoop.mapreduce.Counter counter = ctrItr.next();
           String name = counter.getName();
           String mapValue =
             decimal.format(mapGroup.findCounter(name).getValue());
           String reduceValue =
             decimal.format(reduceGroup.findCounter(name).getValue());
           String totalValue =
             decimal.format(counter.getValue());

           buff.append(
               String.format("\n|%1$-30s|%2$-30s|%3$-10s|%4$-10s|%5$-10s",
View Full Code Here

          tr().
            th(".group.ui-state-default", "Counter Group").
            th(".ui-state-default", "Counters")._()._().
        tbody();
    for (CounterGroup g : total) {
      CounterGroup mg = map == null ? null : map.getGroup(g.getName());
      CounterGroup rg = reduce == null ? null : reduce.getGroup(g.getName());
      ++numGroups;
      // This is mostly for demonstration :) Typically we'd introduced
      // a CounterGroup block to reduce the verbosity. OTOH, this
      // serves as an indicator of where we're in the tag hierarchy.
      TR<THEAD<TABLE<TD<TR<TBODY<TABLE<DIV<Hamlet>>>>>>>> groupHeadRow = tbody.
        tr().
          th().$title(g.getName()).$class("ui-state-default").
            _(fixGroupDisplayName(g.getDisplayName()))._().
          td().$class(C_TABLE).
            table(".dt-counters").$id(job.getID()+"."+g.getName()).
              thead().
                tr().th(".name", "Name");

      if (map != null) {
        groupHeadRow.th("Map").th("Reduce");
      }
      // Ditto
      TBODY<TABLE<TD<TR<TBODY<TABLE<DIV<Hamlet>>>>>>> group = groupHeadRow.
            th(map == null ? "Value" : "Total")._()._().
        tbody();
      for (Counter counter : g) {
        // Ditto
        TR<TBODY<TABLE<TD<TR<TBODY<TABLE<DIV<Hamlet>>>>>>>> groupRow = group.
          tr();
          if (task == null && mg == null && rg == null) {
            groupRow.td().$title(counter.getName())._(counter.getDisplayName()).
            _();
          } else {
            groupRow.td().$title(counter.getName()).
              a(url(urlBase,urlId,g.getName(),
                  counter.getName()), counter.getDisplayName()).
            _();
          }
        if (map != null) {
          Counter mc = mg == null ? null : mg.findCounter(counter.getName());
          Counter rc = rg == null ? null : rg.findCounter(counter.getName());
          groupRow.
            td(mc == null ? "0" : String.valueOf(mc.getValue())).
            td(rc == null ? "0" : String.valueOf(rc.getValue()));
        }
        groupRow.td(String.valueOf(counter.getValue()))._();
View Full Code Here

      }
      for(Map.Entry<TaskAttemptId, TaskAttempt> entry :
        task.getAttempts().entrySet()) {
        long value = 0;
        Counters counters = entry.getValue().getCounters();
        CounterGroup group = (counters != null) ? counters
          .getGroup($(COUNTER_GROUP)) : null;
        if(group != null)  {
          Counter c = group.findCounter($(COUNTER_NAME));
          if(c != null) {
            value = c.getValue();
          }
        }
        values.put(MRApps.toString(entry.getKey()), value);
      }
     
      return;
    }
    // Get all types of counters
    Map<TaskId, Task> tasks = job.getTasks();
    for(Map.Entry<TaskId, Task> entry : tasks.entrySet()) {
      long value = 0;
      Counters counters = entry.getValue().getCounters();
      CounterGroup group = (counters != null) ? counters
        .getGroup($(COUNTER_GROUP)) : null;
      if(group != null)  {
        Counter c = group.findCounter($(COUNTER_NAME));
        if(c != null) {
          value = c.getValue();
        }
      }
      values.put(MRApps.toString(entry.getKey()), value);
View Full Code Here

    this.id = MRApps.toString(job.getID());

    if (total != null) {
      for (CounterGroup g : total) {
        if (g != null) {
          CounterGroup mg = map == null ? null : map.getGroup(g.getName());
          CounterGroup rg = reduce == null ? null : reduce
            .getGroup(g.getName());

          CounterGroupInfo cginfo = new CounterGroupInfo(g.getName(), g,
            mg, rg);
          counterGroup.add(cginfo);
View Full Code Here


  private long getCounterValue(Counters counters, String key)
     throws ParseException {
    for (String groupName : counters.getGroupNames()) {
       CounterGroup totalGroup = counters.getGroup(groupName);
       Iterator<Counter> itrCounter = totalGroup.iterator();
       while (itrCounter.hasNext()) {
         Counter counter = itrCounter.next();
         if (counter.getName().equals(key)) {
           return counter.getValue();
         }
View Full Code Here

      // example is intentionally verbose for clarity. First get all the
      // Counters,
      // then we get the CounterGroup, then we get the Counter, then we
      // get the count.
      Counters counters = mrState.getCounters();
      CounterGroup counterGroup = counters.getGroup("CommentWords");
      Counter counter = counterGroup.findCounter("count");
      long wordCount = counter.getValue(); // Finally!

      // Let's create a special datastore Entity for this value so
      // we can reference it on the ViewComments page
      Entity totalCountEntity = new Entity("TotalWordCount",
View Full Code Here

    SequenceFileOutputFormat.setCompressOutput(job, true);
    job.submit();
   
    int retVal = job.waitForCompletion(true)?0:1;
   
    CounterGroup counters          = job.getCounters().getGroup(CollectionMapper.RESOLVER_GROUP);   
    Counter constructMessageMS     = counters.findCounter(CollectionMapper.CONSTRUCT_MESSAGE_MS);
    Counter parseResponseMS        = counters.findCounter(CollectionMapper.PARSE_RESPONSE_MS);
    Counter performRequestMS       = counters.findCounter(CollectionMapper.PERFORM_REQUEST_MS);
    Counter totalRequestHandlingMS = counters.findCounter(CollectionMapper.TOTAL_REQUEST_HANDLING_MS);
   
    Log.info("Total ConstructMessage percent: "+ (double)(constructMessageMS.getValue()*100L)/((double)totalRequestHandlingMS.getValue()));
    Log.info("Total ParseResponse percent:    "+ (double)(parseResponseMS.getValue()*100L)/((double)totalRequestHandlingMS.getValue()));
    Log.info("Total PerformRequest percent:   "+ (double)(performRequestMS.getValue()*100L)/((double)totalRequestHandlingMS.getValue()));
   
 
View Full Code Here

  }

  static Counters fromAvro(JhCounters counters) {
    Counters result = new Counters();
    for (JhCounterGroup g : counters.groups) {
      CounterGroup group =
          result.addGroup(StringInterner.weakIntern(g.name.toString()),
              StringInterner.weakIntern(g.displayName.toString()));
      for (JhCounter c : g.counts) {
        group.addCounter(StringInterner.weakIntern(c.name.toString()),
            StringInterner.weakIntern(c.displayName.toString()), c.value);
      }
    }
    return result;
  }
View Full Code Here

      }
      for(Map.Entry<TaskAttemptId, TaskAttempt> entry :
        task.getAttempts().entrySet()) {
        long value = 0;
        Counters counters = entry.getValue().getCounters();
        CounterGroup group = (counters != null) ? counters
          .getGroup($(COUNTER_GROUP)) : null;
        if(group != null)  {
          Counter c = group.findCounter($(COUNTER_NAME));
          if(c != null) {
            value = c.getValue();
          }
        }
        values.put(MRApps.toString(entry.getKey()), value);
      }
     
      return;
    }
    // Get all types of counters
    Map<TaskId, Task> tasks = job.getTasks();
    for(Map.Entry<TaskId, Task> entry : tasks.entrySet()) {
      long value = 0;
      CounterGroup group = entry.getValue().getCounters()
        .getGroup($(COUNTER_GROUP));
      if(group != null)  {
        Counter c = group.findCounter($(COUNTER_NAME));
        if(c != null) {
          value = c.getValue();
        }
      }
      values.put(MRApps.toString(entry.getKey()), value);
View Full Code Here

TOP

Related Classes of org.apache.hadoop.mapreduce.CounterGroup

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.