Examples of org.apache.hadoop.mapreduce.TaskID

org.apache.hadoop.mapreduce.TaskID
TaskID represents the immutable and unique identifier for a Map or Reduce Task. Each TaskID encompasses multiple attempts made to execute the Map or Reduce Task, each of which are uniquely indentified by their TaskAttemptID. TaskID consists of 3 parts. First part is the {@link JobID}, that this TaskInProgress belongs to. Second part of the TaskID is either 'm' or 'r' representing whether the task is a map task or a reduce task. And the third part is the task number.
An example TaskID is : task_200707121733_0003_m_000005 , which represents the fifth map task in the third job running at the jobtracker started at 200707121733.
Applications should never construct or parse TaskID strings , but rather use appropriate constructors or {@link #forName(String)} method. @see JobID @see TaskAttemptID

    Map<TaskID, TaskInfo> tasks = job.getAllTasks();


    // validate info of each task
    for (TaskInfo task : tasks.values()) {


      TaskID tid = task.getTaskId();
      long startTime = task.getStartTime();
      assertTrue("Invalid Start time", startTime > 0);
      
      long finishTime = task.getFinishTime();
      assertTrue("Task FINISH_TIME is < START_TIME in history file",

View Full Code Here

    JobTracker jt = mr.getJobTrackerRunner().getJobTracker();
    JobInProgress jip = jt.getJob(job.getID());


    // Get the 1st map, 1st reduce, cleanup & setup taskIDs and
    // validate their history info
    TaskID mapTaskId = new TaskID(job.getID(), TaskType.MAP, 0);
    TaskID reduceTaskId = new TaskID(job.getID(), TaskType.REDUCE, 0);


    TaskInProgress cleanups[] = jip.cleanup;
    TaskID cleanupTaskId;
    if (cleanups[0].isComplete()) {
      cleanupTaskId = cleanups[0].getTIPId();
    }
    else {
      cleanupTaskId = cleanups[1].getTIPId();
    }


    TaskInProgress setups[] = jip.setup;
    TaskID setupTaskId;
    if (setups[0].isComplete()) {
      setupTaskId = setups[0].getTIPId();
    }
    else {
      setupTaskId = setups[1].getTIPId();
    }


    Map<TaskID, TaskInfo> tasks = jobInfo.getAllTasks();


    // validate info of the 4 tasks(cleanup, setup, 1st map, 1st reduce)    


    for (TaskInfo task : tasks.values()) {
      TaskID tid = task.getTaskId();


      if (tid.equals(mapTaskId) ||
          tid.equals(reduceTaskId) ||
          tid.equals(cleanupTaskId) ||
          tid.equals(setupTaskId)) {


        TaskInProgress tip = jip.getTaskInProgress
        (org.apache.hadoop.mapred.TaskID.downgrade(tid));
        assertTrue("START_TIME of Task " + tid + " obtained from history " +
            "file did not match the expected value",

View Full Code Here

    for (TaskInfo task : tasks.values()) {
      // validate info of each attempt
      for (TaskAttemptInfo attempt : task.getAllTaskAttempts().values()) {


        TaskAttemptID attemptId = attempt.getAttemptId();
        TaskID tid = attemptId.getTaskID();


        TaskInProgress tip = jip.getTaskInProgress
        (org.apache.hadoop.mapred.TaskID.downgrade(tid));
        
        TaskStatus ts = tip.getTaskStatus(

View Full Code Here

    // Get splits for the TaskAttempt
    TaskSplitMetaInfo split = 
     taskSplitMetaInfo[taskAttemptID.getTaskID().getId()];
    int locality = getClosestLocality(taskTracker, split);


    TaskID taskId = taskAttemptID.getTaskID();
    TaskType taskType = taskAttemptID.getTaskType();
    if (taskId.getTaskType() != TaskType.MAP) {
      assert false : "Task " + taskId + " is not MAP :" + taskId.getTaskType();
    }
    
    TaskAttemptInfo taskAttemptInfo = jobStory.getMapTaskAttemptInfoAdjusted(
        taskId.getId(), taskAttemptID.getId(), locality);


    if (LOG.isDebugEnabled()) {
      LOG.debug("get an attempt: "
          + taskAttemptID.toString()
          + ", state="

View Full Code Here

   * @return TaskAttemptInfo for the reduce task-attempt
   */
  private TaskAttemptInfo getReduceTaskAttemptInfo(TaskTracker taskTracker,
      TaskAttemptID taskAttemptID) {
    assert (taskAttemptID.getTaskType() == TaskType.REDUCE);
    TaskID taskId = taskAttemptID.getTaskID();
    TaskType taskType = taskAttemptID.getTaskType();


    TaskAttemptInfo taskAttemptInfo = jobStory.getTaskAttemptInfo(taskType,
        taskId.getId(), taskAttemptID.getId());
    if (LOG.isDebugEnabled()) {
      LOG.debug("get an attempt: "
          + taskAttemptID.toString()
          + ", state="
          + taskAttemptInfo.getRunState()

View Full Code Here

   * @return a string like $name-[mrsct]-$id$extension
   */
  public synchronized static String getUniqueFile(TaskAttemptContext context,
                                                  String name,
                                                  String extension) {
    TaskID taskId = context.getTaskAttemptID().getTaskID();
    int partition = taskId.getId();
    StringBuilder result = new StringBuilder();
    result.append(name);
    result.append('-');
    result.append(
        TaskID.getRepresentingCharacter(taskId.getTaskType()));
    result.append('-');
    result.append(NUMBER_FORMAT.format(partition));
    result.append(extension);
    return result.toString();
  }

View Full Code Here

  // FIXME we need to use DAG Id but we are using App Id
  public TaskAttemptContextImpl(Configuration conf,
      TezTaskContext taskContext, boolean isMap) {
    // TODO NEWTEZ Can the jt Identifier string be taskContext.getUniqueId ?
    this(conf, new TaskAttemptID(
        new TaskID(String.valueOf(taskContext.getApplicationId()
            .getClusterTimestamp()), taskContext.getApplicationId().getId(),
            isMap ? TaskType.MAP : TaskType.REDUCE,
            taskContext.getTaskIndex()),
            taskContext.getTaskAttemptNumber()), taskContext);
  }

View Full Code Here

      return;
    }


    Job job = runFailingMapperJob();


    TaskID taskID = new TaskID(job.getJobID(), TaskType.MAP, 0);
    TaskAttemptID aId = new TaskAttemptID(taskID, 0);
    System.out.println("Diagnostics for " + aId + " :");
    for (String diag : job.getTaskDiagnostics(aId)) {
      System.out.println(diag);
    }

View Full Code Here

   */
  @Test(timeout = 10000)
  public void testTaskAttemptFinishedEvent() throws Exception {


    JobID jid = new JobID("001", 1);
    TaskID tid = new TaskID(jid, TaskType.REDUCE, 2);
    TaskAttemptID taskAttemptId = new TaskAttemptID(tid, 3);
    Counters counters = new Counters();
    TaskAttemptFinishedEvent test = new TaskAttemptFinishedEvent(taskAttemptId,
        TaskType.REDUCE, "TEST", 123L, "RAKNAME", "HOSTNAME", "STATUS",
        counters);

View Full Code Here

   * @throws Exception
   */
  @Test(timeout = 10000)
  public void testTaskUpdated() throws Exception {
    JobID jid = new JobID("001", 1);
    TaskID tid = new TaskID(jid, TaskType.REDUCE, 2);
    TaskUpdatedEvent test = new TaskUpdatedEvent(tid, 1234L);
    assertEquals(test.getTaskId().toString(), tid.toString());
    assertEquals(test.getFinishTime(), 1234L);


  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapreduce.TaskID

co.nubetech.hiho.mapreduce.lib.output.AppendSequenceFileOutputFormat

co.nubetech.hiho.mapreduce.lib.output.AppendTextOutputFormat

com.alimama.mdrill.index.IndexMapper

com.alimama.mdrill.index.IndexReducer

com.alimama.mdrill.index.utils.JobIndexPublic

com.alimama.quanjingmonitor.kmeans.KMeansGroupReducer

com.asakusafw.runtime.compatibility.JobCompatibility

com.datasalt.pangool.solr.SolrRecordWriter

com.datasalt.pangool.tuplemr.mapred.lib.output.TestTupleInputOutputFormat

com.mozilla.pig.storage.SeqFileMultiStorage$MultiStorageSequenceOutputFormat

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.