Examples of org.apache.hadoop.mapreduce.Counter

org.apache.hadoop.mapreduce.Counter
A named counter that tracks the progress of a map/reduce job.
Counters represent global counters, defined either by the Map-Reduce framework or applications. Each Counter is named by an {@link Enum} and has a long for the value.
Counters are bunched into Groups, each comprising of counters from a particular Enum class.


  @Override
  public void incrAllCounters(CounterGroupBase<T> rightGroup) {
    try {
      for (Counter right : rightGroup) {
        Counter left = findCounter(right.getName(), right.getDisplayName());
        left.increment(right.getValue());
      }
    } catch (LimitExceededException e) {
      counters.clear();
      throw e;
    }

View Full Code Here

      if (start != 0 && now - start <= Integer.MAX_VALUE) {
        splitsBlock.getProgressWallclockTime().extend(newProgress,
            (int) (now - start));
      }


      Counter cpuCounter = counters.findCounter(TaskCounter.CPU_MILLISECONDS);
      if (cpuCounter != null && cpuCounter.getValue() <= Integer.MAX_VALUE) {
        splitsBlock.getProgressCPUTime().extend(newProgress,
            (int) cpuCounter.getValue()); // long to int? TODO: FIX. Same below
      }


      Counter virtualBytes = counters
        .findCounter(TaskCounter.VIRTUAL_MEMORY_BYTES);
      if (virtualBytes != null) {
        splitsBlock.getProgressVirtualMemoryKbytes().extend(newProgress,
            (int) (virtualBytes.getValue() / (MEMORY_SPLITS_RESOLUTION)));
      }


      Counter physicalBytes = counters
        .findCounter(TaskCounter.PHYSICAL_MEMORY_BYTES);
      if (physicalBytes != null) {
        splitsBlock.getProgressPhysicalMemoryKbytes().extend(newProgress,
            (int) (physicalBytes.getValue() / (MEMORY_SPLITS_RESOLUTION)));
      }
    }
  }

View Full Code Here

    Map<String, Counter> c = lookupCache.get(groupName);
    if (c == null) {
      c = Maps.newHashMap();
      lookupCache.put(groupName, c);
    }
    Counter counter = c.get(counterName);
    if (counter == null) {
      try {
        counter = active.findCounter(groupName, counterName);
      } catch (Exception e) {
        // Recover from this by creating a new active instance

View Full Code Here

    @Override
    public boolean incrCounter(Enum<?> name, long delta) {
        if (context == null) {
            return false;
        }
        Counter counter = context.getCounter(name);
        counter.increment(delta);
        return true;
    }

View Full Code Here

    @Override
    public boolean incrCounter(String group, String name, long delta) {
        if (context == null) {
            return false;
        }
        Counter counter = context.getCounter(group, name);
        counter.increment(delta);
        return true;
    }

View Full Code Here

    @Override
    public boolean incrCounter(Enum<?> name, long delta) {
        if (context == null) {
            return false;
        }
        Counter counter = context.getCounter(name);
        counter.increment(delta);
        return true;
    }

View Full Code Here

    @Override
    public boolean incrCounter(String group, String name, long delta) {
        if (context == null) {
            return false;
        }
        Counter counter = context.getCounter(group, name);
        counter.increment(delta);
        return true;
    }

View Full Code Here

  }


  @Test
  public void skip() throws Exception {


    Counter skippedInstances = EasyMock.createMock(Counter.class);


    EasyMock.expect(ctx.getCounter(IndexInstancesMapper.Counter.SKIPPED_INSTANCES)).andReturn(skippedInstances);
    skippedInstances.increment(1);


    EasyMock.replay(ctx, skippedInstances);


    IndexInstancesMapper indexInstances = new IndexInstancesMapper();
    setField(indexInstances, "labelIndex", labelIndex);

View Full Code Here


  @Test
  public void testToUsersReducerMinPreferencesUserPasses() throws Exception {
    Reducer<VarLongWritable,VarLongWritable,VarLongWritable,VectorWritable>.Context context =
        EasyMock.createMock(Reducer.Context.class);
    Counter userCounters = EasyMock.createMock(Counter.class);


    ToUserVectorsReducer reducer = new ToUserVectorsReducer();
    setField(reducer, "minPreferences", 2);


    EasyMock.expect(context.getCounter(ToUserVectorsReducer.Counters.USERS)).andReturn(userCounters);
    userCounters.increment(1);
    context.write(EasyMock.eq(new VarLongWritable(123)), MathHelper.vectorMatches(
        MathHelper.elem(TasteHadoopUtils.idToIndex(456L), 1.0), MathHelper.elem(TasteHadoopUtils.idToIndex(789L), 1.0)));


    EasyMock.replay(context, userCounters);

View Full Code Here

   */
  @Test
  public void testToUserVectorReducer() throws Exception {
    Reducer<VarLongWritable,VarLongWritable,VarLongWritable,VectorWritable>.Context context =
      EasyMock.createMock(Reducer.Context.class);
    Counter userCounters = EasyMock.createMock(Counter.class);


    EasyMock.expect(context.getCounter(ToUserVectorsReducer.Counters.USERS)).andReturn(userCounters);
    userCounters.increment(1);
    context.write(EasyMock.eq(new VarLongWritable(12L)), MathHelper.vectorMatches(
        MathHelper.elem(TasteHadoopUtils.idToIndex(34L), 1.0), MathHelper.elem(TasteHadoopUtils.idToIndex(56L), 2.0)));


    EasyMock.replay(context, userCounters);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapreduce.Counter

co.nubetech.hiho.dedup.TestDedupKeyMapper

co.nubetech.hiho.dedup.TestDedupKeyReducer

co.nubetech.hiho.dedup.TestDedupValueMapper

co.nubetech.hiho.dedup.TestDedupValueReducer

co.nubetech.hiho.merge.TestMergeKeyMapper

co.nubetech.hiho.merge.TestMergeKeyReducer

co.nubetech.hiho.merge.TestMergeValueMapper

co.nubetech.hiho.merge.TestMergeValueReducer

com.asakusafw.runtime.compatibility.JobCompatibility$MockStatusReporter

com.ebay.erl.mobius.core.mapred.CounterUpdateThread

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.