Examples of org.apache.hadoop.mapreduce.Counter

org.apache.hadoop.mapreduce.Counter
A named counter that tracks the progress of a map/reduce job.
Counters represent global counters, defined either by the Map-Reduce framework or applications. Each Counter is named by an {@link Enum} and has a long for the value.
Counters are bunched into Groups, each comprising of counters from a particular Enum class.

    Map<String, Long> countersMap = Maps.newHashMap();


    for (StageResult sr : stages) {
      Iterator<Counter> iterator = sr.getCounters().getGroup(counterGroupName).iterator();
      while (iterator.hasNext()) {
        Counter counter = (Counter) iterator.next();
        countersMap.put(counter.getDisplayName(), counter.getValue());
      }
    }


    return countersMap;
  }

View Full Code Here


    @Override
    protected void map(ImmutableBytesWritable key, Result value, Context context)
        throws IOException, InterruptedException {
      byte[] row = value.getRow();
      Counter c = getCounter(row);
      c.increment(1);
    }

View Full Code Here

      Counter c = getCounter(row);
      c.increment(1);
    }


    private Counter getCounter(byte[] row) {
      Counter c = null;
      if (Bytes.indexOf(row, Bytes.toBytes(VISIBILITY_EXPS[0])) != -1) {
        c = rowsExp1;
      } else if (Bytes.indexOf(row, Bytes.toBytes(VISIBILITY_EXPS[1])) != -1) {
        c = rowsExp2;
      } else if (Bytes.indexOf(row, Bytes.toBytes(VISIBILITY_EXPS[2])) != -1) {

View Full Code Here

        throw new IllegalStateException("You should call run() first");
      }


      Counters counters = job.getCounters();


      Counter referenced = counters.findCounter(Counts.REFERENCED);
      Counter unreferenced = counters.findCounter(Counts.UNREFERENCED);
      Counter undefined = counters.findCounter(Counts.UNDEFINED);
      Counter multiref = counters.findCounter(Counts.EXTRAREFERENCES);


      boolean success = true;
      //assert
      if (expectedReferenced != referenced.getValue()) {
        LOG.error("Expected referenced count does not match with actual referenced count. " +
            "expected referenced=" + expectedReferenced + " ,actual=" + referenced.getValue());
        success = false;
      }


      if (unreferenced.getValue() > 0) {
        boolean couldBeMultiRef = (multiref.getValue() == unreferenced.getValue());
        LOG.error("Unreferenced nodes were not expected. Unreferenced count=" + unreferenced.getValue()
            + (couldBeMultiRef ? "; could be due to duplicate random numbers" : ""));
        success = false;
      }

View Full Code Here

      throw new IllegalStateException("You should call run() first");
    }
    
    Counters counters = job.getCounters();
    
    Counter referenced = counters.findCounter(Counts.REFERENCED);
    Counter unreferenced = counters.findCounter(Counts.UNREFERENCED);
    Counter undefined = counters.findCounter(Counts.UNDEFINED);
    
    boolean success = true;
    //assert
    if (expectedReferenced != referenced.getValue()) {
      LOG.error("Expected referenced count does not match with actual referenced count. " +
          "expected referenced=" + expectedReferenced + " ,actual=" + referenced.getValue());
      success = false;
    }


    if (unreferenced.getValue() > 0) { 
      LOG.error("Unreferenced nodes were not expected. Unreferenced count=" + unreferenced.getValue());
      success = false;
    }
    
    if (undefined.getValue() > 0) { 
      LOG.error("Found an undefined node. Undefined count=" + undefined.getValue());
      success = false;
    }
    
    return success;
  }

View Full Code Here

    new MapDriver<LongWritable, Text, Text, IntWritable>()
      .withMapper(new MaxTemperatureMapper())
      .withInputValue(value)
      .withCounters(counters)
      .runTest();
    Counter c = counters.findCounter(MaxTemperatureMapper.Temperature.MALFORMED);
    assertThat(c.getValue(), is(1L));
  }

View Full Code Here

    }
  }


  private void setSummarySlotSeconds(JobSummary summary, Counters allCounters) {


    Counter slotMillisMapCounter = allCounters
      .findCounter(JobCounter.SLOTS_MILLIS_MAPS);
    if (slotMillisMapCounter != null) {
      summary.setMapSlotSeconds(slotMillisMapCounter.getValue() / 1000);
    }


    Counter slotMillisReduceCounter = allCounters
      .findCounter(JobCounter.SLOTS_MILLIS_REDUCES);
    if (slotMillisReduceCounter != null) {
      summary.setReduceSlotSeconds(slotMillisReduceCounter.getValue() / 1000);
    }
  }

View Full Code Here

    }
  }


  private void setSummarySlotSeconds(JobSummary summary, Counters allCounters) {


    Counter slotMillisMapCounter = allCounters
      .findCounter(JobCounter.SLOTS_MILLIS_MAPS);
    if (slotMillisMapCounter != null) {
      summary.setMapSlotSeconds(slotMillisMapCounter.getValue() / 1000);
    }


    Counter slotMillisReduceCounter = allCounters
      .findCounter(JobCounter.SLOTS_MILLIS_REDUCES);
    if (slotMillisReduceCounter != null) {
      summary.setReduceSlotSeconds(slotMillisReduceCounter.getValue() / 1000);
    }
  }

View Full Code Here

        long value =
          Long.parseLong(getBlock(counterString, UNIT_OPEN, UNIT_CLOSE,
                                  counterIndex));


        // Add the counter
        Counter counter = group.findCounter(counterName);
        counter.setDisplayName(counterDisplayName);
        counter.increment(value);


        // Get the next counter
        counterString =
          getBlock(groupString, COUNTER_OPEN, COUNTER_CLOSE, groupIndex);
      }

View Full Code Here

    }
    Counters counters = new Counters();
    for (CounterGroup xGrp : tezCounters) {
      counters.addGroup(xGrp.getName(), xGrp.getDisplayName());
      for (TezCounter xCounter : xGrp) {
        Counter counter =
            counters.findCounter(xGrp.getName(), xCounter.getName());
        counter.setValue(xCounter.getValue());


      }
    }
    return counters;
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapreduce.Counter

co.nubetech.hiho.dedup.TestDedupKeyMapper

co.nubetech.hiho.dedup.TestDedupKeyReducer

co.nubetech.hiho.dedup.TestDedupValueMapper

co.nubetech.hiho.dedup.TestDedupValueReducer

co.nubetech.hiho.merge.TestMergeKeyMapper

co.nubetech.hiho.merge.TestMergeKeyReducer

co.nubetech.hiho.merge.TestMergeValueMapper

co.nubetech.hiho.merge.TestMergeValueReducer

com.asakusafw.runtime.compatibility.JobCompatibility$MockStatusReporter

com.ebay.erl.mobius.core.mapred.CounterUpdateThread

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.