DAGClient dagClient, HiveConf conf, DAG dag) {
/* Strings for headers and counters */
String hiveCountersGroup = conf.getVar(conf, HiveConf.ConfVars.HIVECOUNTERGROUP);
Set<StatusGetOpts> statusGetOpts = EnumSet.of(StatusGetOpts.GET_COUNTERS);
TezCounters hiveCounters = null;
try {
hiveCounters = dagClient.getDAGStatus(statusGetOpts).getDAGCounters();
} catch (IOException e) {
// best attempt, shouldn't really kill DAG for this
} catch (TezException e) {
// best attempt, shouldn't really kill DAG for this
}
/* If the counters are missing there is no point trying to print progress */
if (hiveCounters == null) {
return;
}
/* Print the per Vertex summary */
console.printInfo(SUMMARY_HEADER);
SortedSet<String> keys = new TreeSet<String>(progressMap.keySet());
Set<StatusGetOpts> statusOptions = new HashSet<StatusGetOpts>(1);
statusOptions.add(StatusGetOpts.GET_COUNTERS);
for (String vertexName : keys) {
Progress progress = progressMap.get(vertexName);
if (progress != null) {
final int totalTasks = progress.getTotalTaskCount();
final int failedTaskAttempts = progress.getFailedTaskAttemptCount();
final int killedTasks = progress.getKilledTaskCount();
final double duration =
perfLogger.getDuration(PerfLogger.TEZ_RUN_VERTEX + vertexName) / 1000.0;
VertexStatus vertexStatus = null;
try {
vertexStatus = dagClient.getVertexStatus(vertexName, statusOptions);
} catch (IOException e) {
// best attempt, shouldn't really kill DAG for this
} catch (TezException e) {
// best attempt, shouldn't really kill DAG for this
}
if (vertexStatus == null) {
continue;
}
Vertex currentVertex = dag.getVertex(vertexName);
List<Vertex> inputVerticesList = currentVertex.getInputVertices();
long hiveInputRecordsFromOtherVertices = 0;
if (inputVerticesList.size() > 0) {
for (Vertex inputVertex : inputVerticesList) {
String inputVertexName = inputVertex.getName();
hiveInputRecordsFromOtherVertices += getCounterValueByGroupName(hiveCounters,
hiveCountersGroup, String.format("%s_",
ReduceSinkOperator.Counter.RECORDS_OUT_INTERMEDIATE.toString()) +
inputVertexName.replace(" ", "_"));
hiveInputRecordsFromOtherVertices += getCounterValueByGroupName(hiveCounters,
hiveCountersGroup, String.format("%s_",
FileSinkOperator.Counter.RECORDS_OUT.toString()) +
inputVertexName.replace(" ", "_"));
}
}
/*
* Get the CPU & GC
*
* counters org.apache.tez.common.counters.TaskCounter
* GC_TIME_MILLIS=37712
* CPU_MILLISECONDS=2774230
*/
final TezCounters vertexCounters = vertexStatus.getVertexCounters();
final double cpuTimeMillis = getCounterValueByGroupName(vertexCounters,
TaskCounter.class.getName(),
TaskCounter.CPU_MILLISECONDS.name());
final double gcTimeMillis = getCounterValueByGroupName(vertexCounters,