Examples of org.apache.tez.common.counters.TezCounter

org.apache.tez.common.counters.TezCounter
A named counter that tracks the progress of a job.
Counters represent global counters, defined either by the framework or applications. Each Counter is named by an {@link Enum} and has a long for the value.
Counters are bunched into Groups, each comprising of counters from a particular Enum class.

    public void incrementGcCounter() {
      if (null == counters) {
        return; // nothing to do.
      }


      TezCounter gcCounter =
        counters.findCounter(TaskCounter.GC_TIME_MILLIS);
      if (null != gcCounter) {
        gcCounter.increment(getElapsedGc());
      }
    }

View Full Code Here

    FileSystem localFS = FileSystem.getLocal(this.conf);
    LocalDirAllocator localDirAllocator = 
        new LocalDirAllocator(TezJobConfig.LOCAL_DIRS);


    // TODO TEZ Get rid of Map / Reduce references.
    TezCounter shuffledMapsCounter = 
        inputContext.getCounters().findCounter(TaskCounter.SHUFFLED_MAPS);
    TezCounter reduceShuffleBytes =
        inputContext.getCounters().findCounter(TaskCounter.REDUCE_SHUFFLE_BYTES);
    TezCounter failedShuffleCounter =
        inputContext.getCounters().findCounter(TaskCounter.FAILED_SHUFFLE);
    TezCounter spilledRecordsCounter = 
        inputContext.getCounters().findCounter(TaskCounter.SPILLED_RECORDS);
    TezCounter reduceCombineInputCounter =
        inputContext.getCounters().findCounter(TaskCounter.COMBINE_INPUT_RECORDS);
    TezCounter mergedMapOutputsCounter =
        inputContext.getCounters().findCounter(TaskCounter.MERGED_MAP_OUTPUTS);
    
    LOG.info("Shuffle assigned with " + numInputs + " inputs" + ", codec: "
        + (codec == null ? "None" : codec.getClass().getName()) + 
        "ifileReadAhead: " + ifileReadAhead);

View Full Code Here

    for (TezCounterGroupProto counterGroupProto : proto.getCounterGroupsList()) {
      CounterGroup group = counters.addGroup(counterGroupProto.getName(),
        counterGroupProto.getDisplayName());
      for (TezCounterProto counterProto :
        counterGroupProto.getCountersList()) {
        TezCounter counter = group.findCounter(
          counterProto.getName(),
          counterProto.getDisplayName());
        counter.setValue(counterProto.getValue());
      }
    }
    return counters;
  }

View Full Code Here

      groupBuilder.setName(counterGroup.getName());
      groupBuilder.setDisplayName(counterGroup.getDisplayName());
      Iterator<TezCounter> counterIterator = counterGroup.iterator();
      int counterIndex = 0;
      while (counterIterator.hasNext()) {
        TezCounter counter = counterIterator.next();
        TezCounterProto tezCounterProto = TezCounterProto.newBuilder()
          .setName(counter.getName())
          .setDisplayName(counter.getDisplayName())
          .setValue(counter.getValue())
          .build();
        groupBuilder.addCounters(counterIndex, tezCounterProto);
        ++counterIndex;
      }
      builder.addCounterGroups(groupIndex, groupBuilder.build());

View Full Code Here

  void incrementGcCounter() {
    if (null == counters) {
      return; // nothing to do.
    }


    TezCounter gcCounter = counters.findCounter(TaskCounter.GC_TIME_MILLIS);
    if (null != gcCounter) {
      gcCounter.increment(getElapsedGc());
    }
  }

View Full Code Here

  public void progress() {
    reporter.progress();
  }


  public Counters.Counter getCounter(String group, String name) {
    TezCounter counter = context.getCounters().findCounter(group, name);
    MRCounters.MRCounter mrCounter = null;
    if (counter != null) {
      mrCounter = new MRCounters.MRCounter(counter);
    }
    return mrCounter;

View Full Code Here

    }
    return mrCounter;
  }


  public Counters.Counter getCounter(Enum<?> name) {
    TezCounter counter = context.getCounters().findCounter(name);
    MRCounters.MRCounter mrCounter = null;
    if (counter != null) {
      mrCounter = new MRCounters.MRCounter(counter);
    }
    return mrCounter;

View Full Code Here

    FileSystem localFS = FileSystem.getLocal(this.conf);
    LocalDirAllocator localDirAllocator = 
        new LocalDirAllocator(TezJobConfig.LOCAL_DIRS);


    // TODO TEZ Get rid of Map / Reduce references.
    TezCounter shuffledInputsCounter = 
        inputContext.getCounters().findCounter(TaskCounter.NUM_SHUFFLED_INPUTS);
    TezCounter reduceShuffleBytes =
        inputContext.getCounters().findCounter(TaskCounter.SHUFFLE_BYTES);
    TezCounter reduceDataSizeDecompressed = inputContext.getCounters().findCounter(
        TaskCounter.SHUFFLE_BYTES_DECOMPRESSED);
    TezCounter failedShuffleCounter =
        inputContext.getCounters().findCounter(TaskCounter.NUM_FAILED_SHUFFLE_INPUTS);
    TezCounter spilledRecordsCounter = 
        inputContext.getCounters().findCounter(TaskCounter.SPILLED_RECORDS);
    TezCounter reduceCombineInputCounter =
        inputContext.getCounters().findCounter(TaskCounter.COMBINE_INPUT_RECORDS);
    TezCounter mergedMapOutputsCounter =
        inputContext.getCounters().findCounter(TaskCounter.MERGED_MAP_OUTPUTS);
    TezCounter bytesShuffedToDisk = inputContext.getCounters().findCounter(
        TaskCounter.SHUFFLE_BYTES_TO_DISK);
    TezCounter bytesShuffedToMem = inputContext.getCounters().findCounter(
        TaskCounter.SHUFFLE_BYTES_TO_MEM);
    
    LOG.info("Shuffle assigned with " + numInputs + " inputs" + ", codec: "
        + (codec == null ? "None" : codec.getClass().getName()) + 
        "ifileReadAhead: " + ifileReadAhead);

View Full Code Here

    for (TezCounterGroupProto counterGroupProto : proto.getCounterGroupsList()) {
      CounterGroup group = counters.addGroup(counterGroupProto.getName(),
        counterGroupProto.getDisplayName());
      for (TezCounterProto counterProto :
        counterGroupProto.getCountersList()) {
        TezCounter counter = group.findCounter(
          counterProto.getName(),
          counterProto.getDisplayName());
        counter.setValue(counterProto.getValue());
      }
    }
    return counters;
  }

View Full Code Here

      groupBuilder.setName(counterGroup.getName());
      groupBuilder.setDisplayName(counterGroup.getDisplayName());
      Iterator<TezCounter> counterIterator = counterGroup.iterator();
      int counterIndex = 0;
      while (counterIterator.hasNext()) {
        TezCounter counter = counterIterator.next();
        TezCounterProto tezCounterProto = TezCounterProto.newBuilder()
          .setName(counter.getName())
          .setDisplayName(counter.getDisplayName())
          .setValue(counter.getValue())
          .build();
        groupBuilder.addCounters(counterIndex, tezCounterProto);
        ++counterIndex;
      }
      builder.addCounterGroups(groupIndex, groupBuilder.build());

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.tez.common.counters.TezCounter

cascading.stats.tez.TezCounterCache

org.apache.hadoop.hive.ql.exec.tez.TezJobMonitor

org.apache.pig.backend.hadoop.executionengine.tez.runtime.TezTaskContext

org.apache.pig.tools.pigstats.tez.TezVertexStats

org.apache.tez.dag.api.DagTypeConverters

org.apache.tez.examples.JoinValidate

org.apache.tez.examples.JoinValidate$JoinValidateProcessor

org.apache.tez.mapreduce.processor.GcTimeUpdater

org.apache.tez.mapreduce.processor.MRTaskReporter

org.apache.tez.runtime.library.common.shuffle.impl.MergeManager

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.