Examples of org.apache.tez.common.counters.TezCounters

org.apache.tez.common.counters.TezCounters

Counters holds per job/task counters, defined either by the framework or applications. Each Counter can be of any {@link Enum} type.
Counters are bunched into {@link CounterGroup}s, each comprising of counters from a particular Enum class.

            if (this.isMapOpts) {
                numberMaps += parallelism;
            } else {
                numberReduces += parallelism;
            }
            TezCounters tezCounters = status.getVertexCounters();
            counters = Maps.newHashMap();
            Iterator<CounterGroup> grpIt = tezCounters.iterator();
            while (grpIt.hasNext()) {
                CounterGroup grp = grpIt.next();
                Iterator<TezCounter> cntIt = grp.iterator();
                Map<String, Long> cntMap = Maps.newHashMap();
                while (cntIt.hasNext()) {

View Full Code Here

        appId = tezJob.getApplicationId().toString().replace("application", "job");
        setConf(tezJob.getConfiguration());
        DAG dag = tezJob.getDAG();
        hdfsBytesRead = -1;
        hdfsBytesWritten = -1;
        TezCounters tezCounters = tezJob.getDAGCounters();
        if (tezCounters != null) {
            counters = covertToHadoopCounters(tezCounters);
        }


        CounterGroup dagGrp = tezCounters.getGroup(DAG_COUNTER_GROUP);
        totalTasks = (int) dagGrp.findCounter("TOTAL_LAUNCHED_TASKS").getValue();


        CounterGroup fsGrp = tezCounters.getGroup(FS_COUNTER_GROUP);
        fileBytesRead = fsGrp.findCounter("FILE_BYTES_READ").getValue();
        fileBytesWritten = fsGrp.findCounter("FILE_BYTES_WRITTEN").getValue();
        hdfsBytesRead = fsGrp.findCounter("HDFS_BYTES_READ").getValue();
        hdfsBytesWritten = fsGrp.findCounter("HDFS_BYTES_WRITTEN").getValue();

View Full Code Here

    state = WritableUtils.readEnum(in, State.class);
    progress = in.readFloat();
    diagnostics = WritableUtils.readString(in);
    userStatusInfo = WritableUtils.readString(in);
    phase = WritableUtils.readEnum(in, Phase.class);
    counters = new TezCounters();
    
    counters.readFields(in);
    
    localOutputSize = in.readLong();
    startTime = in.readLong();

View Full Code Here

      plr.getSize(), plr.getTimeStamp(),
      plr.hasPattern() ? plr.getPattern() : null);
  }


  public static TezCounters convertTezCountersFromProto(TezCountersProto proto) {
    TezCounters counters = new TezCounters();
    for (TezCounterGroupProto counterGroupProto : proto.getCounterGroupsList()) {
      CounterGroup group = counters.addGroup(counterGroupProto.getName(),
        counterGroupProto.getDisplayName());
      for (TezCounterProto counterProto :
        counterGroupProto.getCountersList()) {
        TezCounter counter = group.findCounter(
          counterProto.getName(),

View Full Code Here

    }
  }


  private InputContext createTezInputContext(byte[] payload) {
    ApplicationId applicationId = ApplicationId.newInstance(10000, 1);
    TezCounters counters = new TezCounters();


    InputContext inputContext = mock(InputContext.class);
    doReturn(applicationId).when(inputContext).getApplicationId();
    doReturn(counters).when(inputContext).getCounters();
    doReturn(1).when(inputContext).getDAGAttemptNumber();

View Full Code Here

    double vProgressFloat = 0.0f;
    log("DAG: State: " + dagStatus.getState() + " Progress: "
        + formatter.format(getProgress(dagProgress)) + " " + dagProgress);
    boolean displayCounter = opts != null ? opts.contains(StatusGetOpts.GET_COUNTERS) : false;
    if (displayCounter) {
      TezCounters counters = dagStatus.getDAGCounters();
      if (counters != null) {
        log("DAG Counters:\n" + counters);
      }
    }
    for (String vertex : vertexNames) {
      VertexStatus vStatus = getVertexStatus(vertex, opts);
      if (vStatus == null) {
        log("Could not retrieve status for vertex: " + vertex);
        continue;
      }
      Progress vProgress = vStatus.getProgress();
      if (vProgress != null) {
        vProgressFloat = 0.0f;
        if (vProgress.getTotalTaskCount() == 0) {
          vProgressFloat = 1.0f;
        } else if (vProgress.getTotalTaskCount() > 0) {
          vProgressFloat = getProgress(vProgress);
        }
        log("VertexStatus:" + " VertexName: " + vertex + " Progress: "
            + formatter.format(vProgressFloat) + " " + vProgress);
      }
      if (displayCounter) {
        TezCounters counters = vStatus.getVertexCounters();
        if (counters != null) {
          log("Vertex Counters for " + vertex + ":\n" + counters);
        }
      }
    } // end of for loop

View Full Code Here

          || state == DAGState.KILLED || state == DAGState.SUCCEEDED) {
        this.mayBeConstructFinalFullCounters();
        return fullCounters;
      }


      TezCounters counters = new TezCounters();
      counters.incrAllCounters(dagCounters);
      return incrTaskCounters(counters, vertices.values());


    } finally {
      readLock.unlock();
    }

View Full Code Here

    }
  }


  @Private
  public void constructFinalFullcounters() {
    this.fullCounters = new TezCounters();
    this.fullCounters.incrAllCounters(dagCounters);
    for (Vertex v : this.vertices.values()) {
      this.fullCounters.incrAllCounters(v.getAllCounters());
    }
  }

View Full Code Here


    public void setLocalityCounter(DAGCounter localityCounter) {
      if (!localitySet.get()) {
        localitySet.set(true);
        if (counters == null) {
          counters = new TezCounters();
        }
        if (localityCounter != null) {
          counters.findCounter(localityCounter).increment(1);
          // TODO Maybe validate that the correct value is being set.
        }

View Full Code Here

              + (vertexName.equals("ivertex1") ? "intermediate-reducer"
                  : vertexName)
              + " Progress: " + formatter.format(vProgressFloat));
        }
        if (displayVertexCounters) {
          TezCounters counters = vStatus.getVertexCounters();
          if (counters != null) {
            System.out.println("Vertex Counters for " + vertexName + ": "
              + counters);
          }
        }
      }
    }
    if (displayDAGCounters) {
      TezCounters counters = dagStatus.getDAGCounters();
      if (counters != null) {
        System.out.println("DAG Counters: " + counters);
      }
    }
  }

View Full Code Here

0 1 2 3 4 5 6 7

TOP

Related Classes of org.apache.tez.common.counters.TezCounters

cascading.stats.tez.TezNodeStats

org.apache.hadoop.hive.ql.exec.tez.TezJobMonitor

org.apache.pig.tools.pigstats.tez.TezDAGStats

org.apache.pig.tools.pigstats.tez.TezVertexStats

org.apache.tez.dag.api.client.rpc.DAGClientRPCImpl

org.apache.tez.dag.api.DagTypeConverters

org.apache.tez.dag.app.dag.event.TaskAttemptEventStatusUpdate$TaskAttemptStatusOld

org.apache.tez.dag.app.dag.impl.DAGImpl

org.apache.tez.dag.app.dag.impl.TaskAttemptImpl

org.apache.tez.dag.app.dag.impl.TaskImpl

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.