Package org.apache.hadoop.mapreduce

Examples of org.apache.hadoop.mapreduce.TaskType

2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
    // Update jobhistory
    TaskTrackerStatus ttStatus =
      this.jobtracker.getTaskTrackerStatus(status.getTaskTracker());
    String trackerHostname = jobtracker.getNode(ttStatus.getHost()).toString();
    TaskType taskType = getTaskType(tip);

    TaskAttemptStartedEvent tse = new TaskAttemptStartedEvent(
        status.getTaskID(), taskType, status.getStartTime(),
        status.getTaskTracker(),  ttStatus.getHttpPort());
   
View Full Code Here
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
    long startTime = taskStatus.getStartTime();
    long finishTime = taskStatus.getFinishTime();
    List<String> taskDiagnosticInfo = tip.getDiagnosticInfo(taskid);
    String diagInfo = taskDiagnosticInfo == null ? "" :
      StringUtils.arrayToString(taskDiagnosticInfo.toArray(new String[0]));
    TaskType taskType = getTaskType(tip);
    TaskAttemptStartedEvent tse = new TaskAttemptStartedEvent(
        taskid, taskType, startTime, taskTrackerName, taskTrackerPort);
   
    jobHistory.logEvent(tse, taskid.getJobID());
  
View Full Code Here
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
    status.setFinishTime(JobTracker.getClock().getTime());
    boolean wasComplete = tip.isComplete();
    updateTaskStatus(tip, status);
    boolean isComplete = tip.isComplete();
    if (wasComplete && !isComplete) { // mark a successful tip as failed
      TaskType taskType = getTaskType(tip);
      TaskFailedEvent tfe =
        new TaskFailedEvent(tip.getTIPId(), tip.getExecFinishTime(), taskType,
            reason, TaskStatus.State.FAILED.toString(),
            taskid);
     
View Full Code Here
441
442
443
444
445
446
447
448
449
450
451
    }

    private TaskAttemptID getTaskAttemptID(
      boolean isMap, boolean isSpeculative) {
      JobID jobId = getJobID();
      TaskType t = TaskType.REDUCE;
      if (isMap) {
        t = TaskType.MAP;
      }
      if (!isSpeculative) {
        return new TaskAttemptID(
View Full Code Here
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
      long finishTime = task.getFinishTime();
      assertTrue("Task FINISH_TIME is < START_TIME in history file",
                 startTime < finishTime);

      // Make sure that the Task type exists and it is valid
      TaskType type = task.getTaskType();
      assertTrue("Unknown Task type \"" + type + "\" is seen in " +
                 "history file for task " + tid,
                 (type.equals(TaskType.MAP) ||
                  type.equals(TaskType.REDUCE) ||
                  type.equals(TaskType.JOB_CLEANUP) ||
                  type.equals(TaskType.JOB_SETUP)));

      if (type.equals(TaskType.MAP)) {
        String splits = task.getSplitLocations();
        //order in the condition OR check is important here
        if (!splitsCanBeEmpty || splits.length() != 0) {
          Matcher m = splitsPattern.matcher(splits);
          assertTrue("Unexpected format of SPLITS \"" + splits + "\" is seen" +
View Full Code Here
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
        long finishTime = attempt.getFinishTime();
        assertTrue("Task FINISH_TIME is < START_TIME in history file",
            startTime < finishTime);

        // Make sure that the Task type exists and it is valid
        TaskType type = attempt.getTaskType();
        assertTrue("Unknown Task type \"" + type + "\" is seen in " +
                   "history file for task attempt " + id,
                   (type.equals(TaskType.MAP) || type.equals(TaskType.REDUCE) ||
                    type.equals(TaskType.JOB_CLEANUP) ||
                    type.equals(TaskType.JOB_SETUP)));

        // Validate task status
        String status = attempt.getTaskStatus();
        assertTrue("Unexpected TASK_STATUS \"" + status + "\" is seen in" +
                   " history file for task attempt " + id,
                   (status.equals(TaskStatus.State.SUCCEEDED.toString()) ||
                    status.equals(TaskStatus.State.FAILED.toString()) ||
                    status.equals(TaskStatus.State.KILLED.toString())));

        // Successful Reduce Task Attempts should have valid SHUFFLE_FINISHED
        // time and SORT_FINISHED time
        if (type.equals(TaskType.REDUCE) &&
            status.equals(TaskStatus.State.SUCCEEDED.toString())) {
          long shuffleFinishTime = attempt.getShuffleFinishTime();
          assertTrue(startTime < shuffleFinishTime);
         
          long sortFinishTime = attempt.getSortFinishTime();
          assertTrue(shuffleFinishTime < sortFinishTime);
        }
        else if (type.equals(TaskType.MAP) &&
            status.equals(TaskStatus.State.SUCCEEDED.toString())) {
          // Successful MAP Task Attempts should have valid MAP_FINISHED time
         long mapFinishTime = attempt.getMapFinishTime();
         assertTrue(startTime < mapFinishTime);
        }
View Full Code Here
206
207
208
209
210
211
212
213
214
215
216
    TaskSplitMetaInfo split =
     taskSplitMetaInfo[taskAttemptID.getTaskID().getId()];
    int locality = getClosestLocality(taskTracker, split);

    TaskID taskId = taskAttemptID.getTaskID();
    TaskType taskType = taskAttemptID.getTaskType();
    if (taskId.getTaskType() != TaskType.MAP) {
      assert false : "Task " + taskId + " is not MAP :" + taskId.getTaskType();
    }
   
    TaskAttemptInfo taskAttemptInfo = jobStory.getMapTaskAttemptInfoAdjusted(
View Full Code Here
275
276
277
278
279
280
281
282
283
284
285
   */
  private TaskAttemptInfo getReduceTaskAttemptInfo(TaskTracker taskTracker,
      TaskAttemptID taskAttemptID) {
    assert (taskAttemptID.getTaskType() == TaskType.REDUCE);
    TaskID taskId = taskAttemptID.getTaskID();
    TaskType taskType = taskAttemptID.getTaskType();

    TaskAttemptInfo taskAttemptInfo = jobStory.getTaskAttemptInfo(taskType,
        taskId.getId(), taskAttemptID.getId());
    if (LOG.isDebugEnabled()) {
      LOG.debug("get an attempt: "
View Full Code Here
367
368
369
370
371
372
373
374
375
376
377
        break; // This is the only exit of the while (true) loop
      }

      // Determine which task type to assign this time
      // First try choosing a task type which is not rejected
      TaskType taskType;
      if (mapRejected) {
        taskType = TaskType.REDUCE;
      } else if (reduceRejected) {
        taskType = TaskType.MAP;
      } else {
View Full Code Here
767
768
769
770
771
772
773
774
775
776
777
   */
  private void preemptTasks(List<PoolSchedulable> scheds, int tasksToPreempt) {
    if (scheds.isEmpty() || tasksToPreempt == 0)
      return;
   
    TaskType taskType = scheds.get(0).getTaskType();
   
    // Collect running tasks of our type from over-scheduled pools
    List<TaskStatus> runningTasks = new ArrayList<TaskStatus>();
    for (PoolSchedulable sched: scheds) {
      if (sched.getRunningTasks() > sched.getFairShare())
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapreduce.TaskType

  • org.apache.blur.mapreduce.lib.BlurOutputCommitter
  • org.apache.hadoop.mapred.CapacityTaskScheduler$TaskSchedulingMgr$QueueComparator
  • org.apache.hadoop.mapred.CapacityTestUtils$FakeJobInProgress
  • org.apache.hadoop.mapred.CoronaJobTracker
  • org.apache.hadoop.mapred.FairScheduler
  • org.apache.hadoop.mapred.JobInProgress
  • org.apache.hadoop.mapred.SimulatorJobInProgress
  • org.apache.hadoop.mapred.taskdetailshistory_jsp
  • org.apache.hadoop.mapred.TestFairScheduler
  • org.apache.hadoop.mapred.TestFairScheduler$FakeJobInProgress
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.