Examples of VertexStatus

Examples of org.apache.tez.dag.api.client.VertexStatus

90
91
92
93
94
95
96
97
98
99
100
  public GetVertexStatusResponseProto getVertexStatus(RpcController controller,
      GetVertexStatusRequestProto request) throws ServiceException {
    try {
      String dagId = request.getDagId();
      String vertexName = request.getVertexName();
      VertexStatus status = real.getVertexStatus(dagId, vertexName,
        DagTypeConverters.convertStatusGetOptsFromProto(
          request.getStatusOptionsList()));
      assert status instanceof VertexStatusBuilder;
      VertexStatusBuilder builder = (VertexStatusBuilder) status;
      return GetVertexStatusResponseProto.newBuilder().
View Full Code Here

Examples of org.apache.tez.dag.api.client.VertexStatus

115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
          + " Progress: "
          + (progress.getTotalTaskCount() < 0 ? formatter.format(0.0f) :
            formatter.format((double)(progress.getSucceededTaskCount())
              /progress.getTotalTaskCount())));
      for (String vertexName : vertexNames) {
        VertexStatus vStatus = dagClient.getVertexStatus(vertexName,
          (displayVertexCounters ? opts : null));
        if (vStatus == null) {
          System.out.println("Could not retrieve status for vertex: "
            + vertexName);
          continue;
        }
        Progress vProgress = vStatus.getProgress();
        if (vProgress != null) {
          vProgressFloat = 0.0f;
          if (vProgress.getTotalTaskCount() == 0) {
            vProgressFloat = 1.0f;
          } else if (vProgress.getTotalTaskCount() > 0) {
            vProgressFloat = (double)vProgress.getSucceededTaskCount()
              /vProgress.getTotalTaskCount();
          }
          System.out.println("VertexStatus:"
              + " VertexName: "
              + (vertexName.equals("ivertex1") ? "intermediate-reducer"
                  : vertexName)
              + " Progress: " + formatter.format(vProgressFloat));
        }
        if (displayVertexCounters) {
          TezCounters counters = vStatus.getVertexCounters();
          if (counters != null) {
            System.out.println("Vertex Counters for " + vertexName + ": "
              + counters);
          }
        }
View Full Code Here

Examples of org.apache.tez.dag.api.client.VertexStatus

216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
    System.out.println("DAGStatusWithCounter:" + resultDagStatus);
  }
 
  @Test
  public void testVertexStatus() throws Exception{
    VertexStatus resultVertexStatus = dagClient.getVertexStatus("v1", null);
    verify(mockProxy).getVertexStatus(null, GetVertexStatusRequestProto.newBuilder()
        .setDagId(dagIdStr).setVertexName("v1").build());
    assertEquals(new VertexStatus(vertexStatusProtoWithoutCounters), resultVertexStatus);
    System.out.println("VertexWithoutCounter:" + resultVertexStatus);
   
    resultVertexStatus = dagClient.getVertexStatus("v1", Sets.newSet(StatusGetOpts.GET_COUNTERS));
    verify(mockProxy).getVertexStatus(null, GetVertexStatusRequestProto.newBuilder()
        .setDagId(dagIdStr).setVertexName("v1").addStatusOptions(StatusGetOptsProto.GET_COUNTERS).build());
    assertEquals(new VertexStatus(vertexStatusProtoWithCounters), resultVertexStatus);
    System.out.println("VertexWithCounter:" + resultVertexStatus);
  }
View Full Code Here

Examples of org.apache.tez.dag.api.client.VertexStatus

114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
          + " Progress: "
          + (progress.getTotalTaskCount() < 0 ? formatter.format(0.0f) :
            formatter.format((double)(progress.getSucceededTaskCount())
              /progress.getTotalTaskCount())));
      for (String vertexName : vertexNames) {
        VertexStatus vStatus = dagClient.getVertexStatus(vertexName,
          (displayVertexCounters ? opts : null));
        if (vStatus == null) {
          System.out.println("Could not retrieve status for vertex: "
            + vertexName);
          continue;
        }
        Progress vProgress = vStatus.getProgress();
        if (vProgress != null) {
          vProgressFloat = 0.0f;
          if (vProgress.getTotalTaskCount() == 0) {
            vProgressFloat = 1.0f;
          } else if (vProgress.getTotalTaskCount() > 0) {
            vProgressFloat = (double)vProgress.getSucceededTaskCount()
              /vProgress.getTotalTaskCount();
          }
          System.out.println("VertexStatus:"
              + " VertexName: "
              + (vertexName.equals("ivertex1") ? "intermediate-reducer"
                  : vertexName)
              + " Progress: " + formatter.format(vProgressFloat));
        }
        if (displayVertexCounters) {
          TezCounters counters = vStatus.getVertexCounters();
          if (counters != null) {
            System.out.println("Vertex Counters for " + vertexName + ": "
              + counters);
          }
        }
View Full Code Here

Examples of org.apache.tez.dag.api.client.VertexStatus

253
254
255
256
257
258
259
260
261
262
263
      requestProtoBuilder.addAllStatusOptions(
        DagTypeConverters.convertStatusGetOptsToProto(statusOptions));
    }

    try {
      return new VertexStatus(
        proxy.getVertexStatus(null,
          requestProtoBuilder.build()).getVertexStatus());
    } catch (ServiceException e) {
      // TEZ-151 retrieve wrapped TezException
      throw new TezException(e);
View Full Code Here

Examples of org.apache.tez.dag.api.client.VertexStatus

487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
        final int totalTasks = progress.getTotalTaskCount();
        final int failedTaskAttempts = progress.getFailedTaskAttemptCount();
        final int killedTasks = progress.getKilledTaskCount();
        final double duration =
            perfLogger.getDuration(PerfLogger.TEZ_RUN_VERTEX + vertexName) / 1000.0;
        VertexStatus vertexStatus = null;
        try {
          vertexStatus = dagClient.getVertexStatus(vertexName, statusOptions);
        } catch (IOException e) {
          // best attempt, shouldn't really kill DAG for this
        } catch (TezException e) {
          // best attempt, shouldn't really kill DAG for this
        }

        if (vertexStatus == null) {
          continue;
        }

        Vertex currentVertex = dag.getVertex(vertexName);
        List<Vertex> inputVerticesList = currentVertex.getInputVertices();
        long hiveInputRecordsFromOtherVertices = 0;
        if (inputVerticesList.size() > 0) {

          for (Vertex inputVertex : inputVerticesList) {
            String inputVertexName = inputVertex.getName();
            hiveInputRecordsFromOtherVertices += getCounterValueByGroupName(hiveCounters,
                hiveCountersGroup, String.format("%s_",
                    ReduceSinkOperator.Counter.RECORDS_OUT_INTERMEDIATE.toString()) +
                    inputVertexName.replace(" ", "_"));

            hiveInputRecordsFromOtherVertices += getCounterValueByGroupName(hiveCounters,
                hiveCountersGroup, String.format("%s_",
                    FileSinkOperator.Counter.RECORDS_OUT.toString()) +
                    inputVertexName.replace(" ", "_"));
          }
        }

      /*
       * Get the CPU & GC
       *
       * counters org.apache.tez.common.counters.TaskCounter
       *  GC_TIME_MILLIS=37712
       *  CPU_MILLISECONDS=2774230
       */
        final TezCounters vertexCounters = vertexStatus.getVertexCounters();
        final double cpuTimeMillis = getCounterValueByGroupName(vertexCounters,
            TaskCounter.class.getName(),
            TaskCounter.CPU_MILLISECONDS.name());

        final double gcTimeMillis = getCounterValueByGroupName(vertexCounters,
View Full Code Here

Examples of org.apache.tez.dag.api.client.VertexStatus

653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
      }

      // DAG might have been killed, lets try to get vertex state from AM before dying
      // KILLED or FAILED state
      if (vextexStatusFromAM) {
        VertexStatus vertexStatus = null;
        try {
          vertexStatus = dagClient.getVertexStatus(s, null);
        } catch (IOException e) {
          // best attempt, shouldn't really kill DAG for this
        } catch (TezException e) {
          // best attempt, shouldn't really kill DAG for this
        }
        if (vertexStatus != null) {
          vertexState = vertexStatus.getState();
        }
      }

      // Map 1 ..........  SUCCEEDED      7          7        0        0       0       0
      String nameWithProgress = getNameWithProgress(s, complete, total);
View Full Code Here

Examples of org.apache.tez.dag.api.client.VertexStatus

1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
    }

    public VertexStatus getVertexStatus(String dagIdStr, String vertexName,
        Set<StatusGetOpts> statusOptions)
        throws TezException{
      VertexStatus status = getDAG(dagIdStr)
          .getVertexStatus(vertexName, statusOptions);
      if(status == null) {
        throw new TezException("Unknown vertexName: " + vertexName);
      }
View Full Code Here

Examples of org.apache.tez.dag.api.client.VertexStatus

1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
    }

    public VertexStatus getVertexStatus(String dagIdStr, String vertexName,
        Set<StatusGetOpts> statusOptions)
        throws TezException{
      VertexStatus status = getDAG(dagIdStr)
          .getVertexStatus(vertexName, statusOptions);
      if(status == null) {
        throw new TezException("Unknown vertexName: " + vertexName);
      }
View Full Code Here

Examples of org.apache.tez.dag.api.client.VertexStatus

92
93
94
95
96
97
98
99
100
101
102
  public GetVertexStatusResponseProto getVertexStatus(RpcController controller,
      GetVertexStatusRequestProto request) throws ServiceException {
    try {
      String dagId = request.getDagId();
      String vertexName = request.getVertexName();
      VertexStatus status = real.getVertexStatus(dagId, vertexName,
        DagTypeConverters.convertStatusGetOptsFromProto(
          request.getStatusOptionsList()));
      assert status instanceof VertexStatusBuilder;
      VertexStatusBuilder builder = (VertexStatusBuilder) status;
      return GetVertexStatusResponseProto.newBuilder().
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.