Examples of org.apache.tez.dag.records.TezTaskAttemptID

org.apache.tez.dag.records.TezTaskAttemptID
TezTaskAttemptID represents the immutable and unique identifier for a task attempt. Each task attempt is one particular instance of a Tez Task identified by its TezTaskID. TezTaskAttemptID consists of 2 parts. First part is the {@link TezTaskID}, that this TaskAttemptID belongs to. Second part is the task attempt number.

Applications should never construct or parse TaskAttemptID strings , but rather use appropriate constructors or {@link Class#forName(String)}method. @see TezTaskID

    String [] host1 = {"host1"};
    String [] host2 = {"host2"};


    String [] defaultRack = {"/default-rack"};


    TezTaskAttemptID taID11 = new TezTaskAttemptID(new TezTaskID(vertexID, 1), 1);
    TezTaskAttemptID taID21 = new TezTaskAttemptID(new TezTaskID(vertexID, 2), 1);
    TezTaskAttemptID taID31 = new TezTaskAttemptID(new TezTaskID(vertexID, 3), 1);
    TaskAttempt ta11 = mock(TaskAttempt.class);
    TaskAttempt ta21 = mock(TaskAttempt.class);
    TaskAttempt ta31 = mock(TaskAttempt.class);


    AMSchedulerEventTALaunchRequest lrTa11 = createLaunchRequestEvent(

View Full Code Here

    Priority priority1 = Priority.newInstance(1);


    TezVertexID vertexID1 = new TezVertexID(dagID, 1);


    //Vertex 1, Task 1, Attempt 1, host1
    TezTaskAttemptID taID11 = new TezTaskAttemptID(new TezTaskID(vertexID1, 1), 1);
    TaskAttempt ta11 = mock(TaskAttempt.class);
    AMSchedulerEventTALaunchRequest lrEvent1 = createLaunchRequestEvent(taID11, ta11, resource1, host1, racks, priority1);


    //Vertex 1, Task 2, Attempt 1, host1
    TezTaskAttemptID taID12 = new TezTaskAttemptID(new TezTaskID(vertexID1, 2), 1);
    TaskAttempt ta12 = mock(TaskAttempt.class);
    AMSchedulerEventTALaunchRequest lrEvent2 = createLaunchRequestEvent(taID12, ta12, resource1, host1, racks, priority1);


    //Vertex 1, Task 3, Attempt 1, host2
    TezTaskAttemptID taID13 = new TezTaskAttemptID(new TezTaskID(vertexID1, 3), 1);
    TaskAttempt ta13 = mock(TaskAttempt.class);
    AMSchedulerEventTALaunchRequest lrEvent3 = createLaunchRequestEvent(taID13, ta13, resource1, host2, racks, priority1);


    //Vertex 1, Task 4, Attempt 1, host2
    TezTaskAttemptID taID14 = new TezTaskAttemptID(new TezTaskID(vertexID1, 4), 1);
    TaskAttempt ta14 = mock(TaskAttempt.class);
    AMSchedulerEventTALaunchRequest lrEvent4 = createLaunchRequestEvent(taID14, ta14, resource1, host2, racks, priority1);


    taskSchedulerEventHandler.handleEvent(lrEvent1);
    taskSchedulerEventHandler.handleEvent(lrEvent2);

View Full Code Here

    Priority priority = Priority.newInstance(3);


    TezVertexID vertexID = new TezVertexID(dagID, 1);


    //Vertex 1, Task 1, Attempt 1, no locality information.
    TezTaskAttemptID taID11 = new TezTaskAttemptID(new TezTaskID(vertexID, 1), 1);
    TaskAttempt ta11 = mock(TaskAttempt.class);
    doReturn(vertexID).when(ta11).getVertexID();
    AMSchedulerEventTALaunchRequest lrEvent11 = createLaunchRequestEvent(
      taID11, ta11, resource1, emptyHosts, racks, priority);


    //Vertex1, Task2, Attempt 1,  no locality information.
    TezTaskAttemptID taID12 = new TezTaskAttemptID(new TezTaskID(vertexID, 2), 1);
    TaskAttempt ta12 = mock(TaskAttempt.class);
    doReturn(vertexID).when(ta12).getVertexID();
    AMSchedulerEventTALaunchRequest lrEvent12 = createLaunchRequestEvent(
      taID12, ta12, resource1, emptyHosts, racks, priority);

View Full Code Here


    TezVertexID vertexID1 = new TezVertexID(dagID, 1);
    TezVertexID vertexID2 = new TezVertexID(dagID, 2);


    //Vertex 1, Task 1, Attempt 1, host1
    TezTaskAttemptID taID11 = new TezTaskAttemptID(
      new TezTaskID(vertexID1, 1), 1);
    TaskAttempt ta11 = mock(TaskAttempt.class);
    doReturn(vertexID1).when(ta11).getVertexID();
    AMSchedulerEventTALaunchRequest lrEvent11 = createLaunchRequestEvent(
      taID11, ta11, resource1, host1, racks, priority1);


    //Vertex2, Task1, Attempt 1, host1
    TezTaskAttemptID taID21 = new TezTaskAttemptID(
      new TezTaskID(vertexID2, 1), 1);
    TaskAttempt ta21 = mock(TaskAttempt.class);
    doReturn(vertexID2).when(ta21).getVertexID();
    AMSchedulerEventTALaunchRequest lrEvent21 = createLaunchRequestEvent(
      taID21, ta21, resource1, host1, racks, priority2);

View Full Code Here

    // result of past getTaskAttemptTezEvents calls.
    List<TezEvent> fetchedList;
    TezTaskID taskId = getNewTaskID();
    scheduleTaskAttempt(taskId);
    sendTezEventsToTask(taskId, 2);
    TezTaskAttemptID attemptID = mockTask.getAttemptList().iterator().next()
        .getID();
    fetchedList = mockTask.getTaskAttemptTezEvents(attemptID, 0, 100);
    assertEquals(2, fetchedList.size());


    // Add events, make sure underlying list is the same, and no exceptions are

View Full Code Here

    scheduledTasks.clear();
    
    when(mockSrcVertex1.getTotalTasks()).thenReturn(2);
    when(mockSrcVertex2.getTotalTasks()).thenReturn(2);


    TezTaskAttemptID mockSrcAttemptId11 = 
        new TezTaskAttemptID(new TezTaskID(mockSrcVertexId1, 0), 0);
    TezTaskAttemptID mockSrcAttemptId12 = 
        new TezTaskAttemptID(new TezTaskID(mockSrcVertexId1, 1), 0);
    TezTaskAttemptID mockSrcAttemptId21 = 
        new TezTaskAttemptID(new TezTaskID(mockSrcVertexId2, 0), 0);
    TezTaskAttemptID mockSrcAttemptId31 = 
        new TezTaskAttemptID(new TezTaskID(mockSrcVertexId3, 0), 0);


    byte[] payload =
        VertexManagerEventPayloadProto.newBuilder().setOutputSize(5000L).build().toByteArray();
    VertexManagerEvent vmEvent = new VertexManagerEvent("Vertex", payload);
    // parallelism not change due to large data size

View Full Code Here

      }


      TezHeartbeatResponse response = new TezHeartbeatResponse();
      response.setLastRequestId(requestId);


      TezTaskAttemptID taskAttemptID = request.getCurrentTaskAttemptID();
      if (taskAttemptID != null) {
        ContainerId containerIdFromMap = attemptToInfoMap.get(taskAttemptID);
        if(containerIdFromMap == null || !containerIdFromMap.equals(containerId)) {
          throw new TezException("Attempt " + taskAttemptID
            + " is not recognized for heartbeat");
        }


        if(containerInfo.lastRequestId+1 != requestId) {
          throw new TezException("Container " + containerId
              + " has invalid request id. Expected: "
              + containerInfo.lastRequestId+1
              + " and actual: " + requestId);
        }


        List<TezEvent> inEvents = request.getEvents();
        if (LOG.isDebugEnabled()) {
          LOG.debug("Ping from " + taskAttemptID.toString() +
              " events: " + (inEvents != null? inEvents.size() : -1));
        }
        if(inEvents!=null && !inEvents.isEmpty()) {
          TezVertexID vertexId = taskAttemptID.getTaskID().getVertexID();
          context.getEventHandler().handle(
              new VertexEventRouteEvent(vertexId, inEvents));
        }
        taskHeartbeatHandler.pinged(taskAttemptID);
        List<TezEvent> outEvents = context
            .getCurrentDAG()
            .getVertex(taskAttemptID.getTaskID().getVertexID())
            .getTask(taskAttemptID.getTaskID())
            .getTaskAttemptTezEvents(taskAttemptID, request.getStartIndex(),
                request.getMaxEvents());
        response.setEvents(outEvents);
      }
      containerInfo.lastRequestId = requestId;

View Full Code Here

    Assert.assertTrue(scheduler.totalTasksToSchedule == 3);
    Assert.assertTrue(scheduler.numSourceTasksCompleted == 0);
    Assert.assertTrue(scheduler.pendingTasks.isEmpty());
    Assert.assertTrue(scheduledTasks.size() == 3); // all tasks scheduled


    TezTaskAttemptID mockSrcAttemptId11 = 
        new TezTaskAttemptID(new TezTaskID(mockSrcVertexId1, 0), 0);
    TezTaskAttemptID mockSrcAttemptId12 = 
        new TezTaskAttemptID(new TezTaskID(mockSrcVertexId1, 1), 0);
    TezTaskAttemptID mockSrcAttemptId21 = 
        new TezTaskAttemptID(new TezTaskID(mockSrcVertexId2, 0), 0);
    TezTaskAttemptID mockSrcAttemptId22 = 
        new TezTaskAttemptID(new TezTaskID(mockSrcVertexId2, 1), 0);
    TezTaskAttemptID mockSrcAttemptId31 = 
        new TezTaskAttemptID(new TezTaskID(mockSrcVertexId3, 0), 0);
    
    // min, max > 0 and min == max
    scheduler = createScheduler(conf, mockManagedVertex, 0.25f, 0.25f);
    scheduler.onVertexStarted(null);
    Assert.assertTrue(scheduler.pendingTasks.size() == 3); // no tasks scheduled

View Full Code Here

          ApplicationId appId = ApplicationId.newInstance(
              Long.valueOf(parts[1]), Integer.parseInt(parts[2]));
          TezDAGID dagId = new TezDAGID(appId, Integer.parseInt(parts[3]));
          TezVertexID vId = new TezVertexID(dagId, Integer.parseInt(parts[4]));
          TezTaskID tId = new TezTaskID(vId, Integer.parseInt(parts[5]));
          return new TezTaskAttemptID(tId, Integer.parseInt(parts[6]));
        } else
          exceptionMsg = "Bad TaskType identifier. TaskAttemptId string : "
              + str + " is not properly formed.";
      }
    } catch (Exception ex) {

View Full Code Here

    String taskVertexName = "currentVertex";
    String destinationVertexName = "destinationVertex";
    TezDAGID dagID = new TezDAGID("2000", 1, 1);
    TezVertexID vertexID = new TezVertexID(dagID, 1);
    TezTaskID taskID = new TezTaskID(vertexID, 1);
    TezTaskAttemptID taskAttemptID = new TezTaskAttemptID(taskID, 1);
    TezCounters counters = new TezCounters();
    byte[] userPayload = TezUtils.createUserPayloadFromConf(conf);
    RuntimeTask runtimeTask = null;
    
    int shufflePort = 2112;

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.tez.dag.records.TezTaskAttemptID

org.apache.hadoop.mapred.MRVertexOutputCommitter

org.apache.hadoop.mapred.YarnTezDagChild

org.apache.tez.common.IDUtils

org.apache.tez.dag.app.dag.impl.Edge

org.apache.tez.dag.app.dag.impl.TaskAttemptImpl$OutputReportedFailedTransition

org.apache.tez.dag.app.dag.impl.TaskImpl$AttemptProcessingCompleteTransition

org.apache.tez.dag.app.dag.impl.TaskImpl$AttemptSucceededTransition

org.apache.tez.dag.app.dag.impl.TaskImpl$TaskRetroactiveFailureTransition

org.apache.tez.dag.app.dag.impl.TaskImpl$TaskRetroactiveKilledTransition

org.apache.tez.dag.app.dag.impl.TestEdge

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.