Examples of org.apache.tez.dag.records.TezVertexID

org.apache.tez.dag.records.TezVertexID
TezVertexID represents the immutable and unique identifier for a Vertex in a Tez DAG. Each TezVertexID encompasses multiple Tez Tasks. TezVertezID consists of 2 parts. The first part is the {@link TezDAGID}, that is the Tez DAG that this vertex belongs to. The second part is the vertex number. @see TezDAGID @see TezTaskID

    AMContainerEventAssignTA assignEvent = null;
    
    Map<String, LocalResource> dag1LRs = Maps.newHashMap();
    dag1LRs.put(rsrc1, lr1);


    TezVertexID vertexID11 = TezVertexID.getInstance(dagID1, 1);
    
    //Vertex 1, Task 1, Attempt 1, host1, lr1
    TezTaskAttemptID taID111 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID11, 1), 1);
    TaskAttempt ta111 = mock(TaskAttempt.class);
    AMSchedulerEventTALaunchRequest lrEvent11 = createLaunchRequestEvent(taID111, ta111, resource1, host1, racks, priority1, dag1LRs);


    //Vertex 1, Task 2, Attempt 1, host1, lr1
    TezTaskAttemptID taID112 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID11, 2), 1);
    TaskAttempt ta112 = mock(TaskAttempt.class);
    AMSchedulerEventTALaunchRequest lrEvent12 = createLaunchRequestEvent(taID112, ta112, resource1, host1, racks, priority1, dag1LRs);


    taskSchedulerEventHandler.handleEvent(lrEvent11);
    taskSchedulerEventHandler.handleEvent(lrEvent12);


    Container container1 = createContainer(1, "host1", resource1, priority1);


    // One container allocated.
    drainNotifier.set(false);
    taskScheduler.onContainersAllocated(Collections.singletonList(container1));
    TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
    drainableAppCallback.drain();
    verify(taskSchedulerEventHandler).taskAllocated(eq(ta111), any(Object.class), eq(container1));
    assignEvent = (AMContainerEventAssignTA) eventHandler.verifyInvocation(AMContainerEventAssignTA.class);
    assertEquals(1, assignEvent.getRemoteTaskLocalResources().size());
    
    // Task assigned to container completed successfully. Container should be re-used.
    taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta111, container1.getId(), TaskAttemptState.SUCCEEDED));
    drainableAppCallback.drain();
    verify(taskScheduler).deallocateTask(eq(ta111), eq(true));
    verify(taskSchedulerEventHandler).taskAllocated(eq(ta112), any(Object.class), eq(container1));
    verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
    eventHandler.verifyNoInvocations(AMContainerEventStopRequest.class);
    assignEvent = (AMContainerEventAssignTA) eventHandler.verifyInvocation(AMContainerEventAssignTA.class);
    assertEquals(1, assignEvent.getRemoteTaskLocalResources().size());
    eventHandler.reset();


    // Task assigned to container completed successfully.
    // Verify reuse across hosts.
    taskSchedulerEventHandler.handleEvent(new AMSchedulerEventTAEnded(ta112, container1.getId(), TaskAttemptState.SUCCEEDED));
    drainableAppCallback.drain();
    verify(taskScheduler).deallocateTask(eq(ta112), eq(true));
    verify(rmClient, times(0)).releaseAssignedContainer(eq(container1.getId()));
    eventHandler.verifyNoInvocations(AMContainerEventStopRequest.class);
    eventHandler.reset();


    // Setup DAG2 with additional resources. Make sure the container, even without all resources, is reused.
    TezDAGID dagID2 = TezDAGID.getInstance("0", 2, 0);
    dagIDAnswer.setDAGID(dagID2);
    
    Map<String, LocalResource> dag2LRs = Maps.newHashMap();
    dag2LRs.put(rsrc2, lr2);
    dag2LRs.put(rsrc3, lr3);
    
    TezVertexID vertexID21 = TezVertexID.getInstance(dagID2, 1);
    
    //Vertex 2, Task 1, Attempt 1, host1, lr2
    TezTaskAttemptID taID211 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID21, 1), 1);
    TaskAttempt ta211 = mock(TaskAttempt.class);
    AMSchedulerEventTALaunchRequest lrEvent21 = createLaunchRequestEvent(taID211, ta211, resource1, host1, racks, priority1, dag2LRs);

View Full Code Here

    int appAttemptNumber = 1;
    TezUmbilical tezUmbilical = null;
    String taskVertexName = "currentVertex";
    String destinationVertexName = "destinationVertex";
    TezDAGID dagID = TezDAGID.getInstance("2000", 1, 1);
    TezVertexID vertexID = TezVertexID.getInstance(dagID, 1);
    TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);
    TezTaskAttemptID taskAttemptID = TezTaskAttemptID.getInstance(taskID, 1);
    TezCounters counters = new TezCounters();
    byte[] userPayload = TezUtils.createUserPayloadFromConf(conf);
    RuntimeTask runtimeTask = null;

View Full Code Here

      LOG.info("Using Natural order dag scheduler");
      dag.dagScheduler = new DAGSchedulerNaturalOrder(dag, dag.eventHandler);
    }


    private VertexImpl createVertex(DAGImpl dag, String vertexName, int vId) {
      TezVertexID vertexId = TezBuilderUtils.newVertexID(dag.getID(), vId);


      VertexPlan vertexPlan = dag.getJobPlan().getVertex(vId);
      VertexLocationHint vertexLocationHint = DagTypeConverters
          .convertFromDAGPlan(vertexPlan.getTaskLocationHintList());

View Full Code Here

      doReturn(eventHandler).when(appContext).getEventHandler();
      when(appContext.getApplicationAttemptId()).thenReturn(appAttemptID);
      when(appContext.getApplicationID()).thenReturn(applicationID);


      dagID = new TezDAGID(applicationID, 1);
      vertexID = new TezVertexID(dagID, 1);
      taskID = new TezTaskID(vertexID, 1);
      taskAttemptID = new TezTaskAttemptID(taskID, 1);


      taskSpec = mock(TaskSpec.class);
      doReturn(taskAttemptID).when(taskSpec).getTaskAttemptID();

View Full Code Here


  public static TezTaskAttemptID getMockTaskAttemptId(
      int jobId, int vertexId, int taskId, int taskAttemptId) {
    return new TezTaskAttemptID(
        new TezTaskID(
            new TezVertexID(
                new TezDAGID(
                    ApplicationId.newInstance(0, jobId), jobId),
                    vertexId),
                    taskId)
        , taskAttemptId);

View Full Code Here

  }
  
  public static TezTaskID getMockTaskId(int jobId,
      int vertexId, int taskId) {
    return new TezTaskID(
        new TezVertexID(new TezDAGID(
            ApplicationId.newInstance(0, jobId),
            jobId), vertexId),
            taskId);
  }

View Full Code Here

    return new TezDAGID(
        ApplicationId.newInstance(0, jobId), jobId);
  }
  
  public static TezVertexID getMockVertexId(int jobId, int vId) {
    return new TezVertexID(
        new TezDAGID(
            ApplicationId.newInstance(0, jobId), jobId),
            vId);
  }

View Full Code Here


  // FIXME hack alert converting objects with hard coded id
  public static TezTaskID
      fromMRTaskId(org.apache.hadoop.mapreduce.TaskID taskid) {
    return new TezTaskID(
        new TezVertexID(fromMRJobId(taskid.getJobID()),
                (taskid.getTaskType() == TaskType.MAP ? 0 : 1)
            ),
        taskid.getId());
  }

View Full Code Here

    initDAG(dag);
    startDAG(dag);
    dispatcher.await();


    for (int i = 0 ; i < 6; ++i ) {
      TezVertexID vId = new TezVertexID(dagId, i);
      Vertex v = dag.getVertex(vId);
      Assert.assertEquals(VertexState.RUNNING, v.getState());
      if (i < 2) {
        Assert.assertEquals(0, v.getDistanceFromRoot());
      } else if (i == 2) {
        Assert.assertEquals(1, v.getDistanceFromRoot());
      } else if ( i > 2 && i < 5) {
        Assert.assertEquals(2, v.getDistanceFromRoot());
      } else if (i == 5) {
        Assert.assertEquals(3, v.getDistanceFromRoot());
      }
    }


    for (int i = 0 ; i < 6; ++i ) {
      TezVertexID vId = new TezVertexID(dagId, i);
      LOG.info("Distance from root: v" + i + ":"
          + dag.getVertex(vId).getDistanceFromRoot());
    }
  }

View Full Code Here

  public void testVertexCompletion() {
    initDAG(dag);
    startDAG(dag);
    dispatcher.await();


    TezVertexID vId = new TezVertexID(dagId, 1);
    Vertex v = dag.getVertex(vId);
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(
        new TezTaskID(vId, 0), TaskState.SUCCEEDED));
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(
        new TezTaskID(vId, 1), TaskState.SUCCEEDED));

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.tez.dag.records.TezVertexID

org.apache.hadoop.mapred.TezMRTypeConverter

org.apache.hadoop.mapred.YarnTezDagChild

org.apache.tez.common.IDUtils

org.apache.tez.dag.app.dag.impl.DAGImpl

org.apache.tez.dag.app.dag.impl.DAGImpl$InitTransition

org.apache.tez.dag.app.dag.impl.ShuffleVertexManager

org.apache.tez.dag.app.dag.impl.TestDAGImpl

org.apache.tez.dag.app.dag.impl.TestDAGScheduler

org.apache.tez.dag.app.dag.impl.TestEdge

org.apache.tez.dag.app.dag.impl.TestTaskAttempt

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.