Package org.apache.hadoop.mapreduce

Examples of org.apache.hadoop.mapreduce.TaskType

156
157
158
159
160
161
162
163
164
165
166
  }

  @Override
  public boolean needsTaskCommit(TaskAttemptContext context) throws IOException {
    TaskAttemptID taskAttemptID = context.getTaskAttemptID();
    TaskType taskType = taskAttemptID.getTaskType();
    if (taskType == TaskType.MAP && context.getNumReduceTasks() != 0) {
      return false;
    } else {
      return true;
    }
View Full Code Here
208
209
210
211
212
213
214
215
216
217
218
      abstract TaskType getTaskType();
     
      public int compare(CapacitySchedulerQueue q1, CapacitySchedulerQueue q2) {
        // look at how much capacity they've filled. Treat a queue with
        // capacity=0 equivalent to a queue running at capacity
        TaskType taskType = getTaskType();
        double r1 = (0 == q1.getCapacity(taskType))? 1.0f:
          (double)q1.getNumSlotsOccupied(taskType)/(double) q1.getCapacity(taskType);
        double r2 = (0 == q2.getCapacity(taskType))? 1.0f:
          (double)q2.getNumSlotsOccupied(taskType)/(double) q2.getCapacity(taskType);
        if (r1<r2) return -1;
View Full Code Here
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
      }
    }
  }

  void setFirstTaskLaunchTime(TaskInProgress tip) {
    TaskType key = tip.getFirstTaskType();

    synchronized(firstTaskLaunchTimes) {
      // Could be optimized to do only one lookup with a little more code
      if (!firstTaskLaunchTimes.containsKey(key)) {
        firstTaskLaunchTimes.put(key, tip.getExecStartTime());
View Full Code Here
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
      tip.addRunningTask(id, tts.getTrackerName());
    }
    final JobTrackerInstrumentation metrics = jobtracker.getInstrumentation();

    // keeping the earlier ordering intact
    TaskType name;
    String splits = "";
    Enum counter = null;
    if (tip.isJobSetupTask()) {
      launchedSetup = true;
      name = TaskType.JOB_SETUP;
View Full Code Here
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
      }
    }
  }

  void setFirstTaskLaunchTime(TaskInProgress tip) {
    TaskType key = getTaskType(tip);

    synchronized(firstTaskLaunchTimes) {
      // Could be optimized to do only one lookup with a little more code
      if (!firstTaskLaunchTimes.containsKey(key)) {
        firstTaskLaunchTimes.put(key, tip.getExecStartTime());
View Full Code Here
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
    // Update jobhistory
    TaskTrackerStatus ttStatus =
      this.jobtracker.getTaskTrackerStatus(status.getTaskTracker());
    String trackerHostname = jobtracker.getNode(ttStatus.getHost()).toString();
    TaskType taskType = getTaskType(tip);

    TaskAttemptStartedEvent tse = new TaskAttemptStartedEvent(
        status.getTaskID(), taskType, status.getStartTime(),
        status.getTaskTracker(),  ttStatus.getHttpPort(), -1);
   
View Full Code Here
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
    long startTime = taskStatus.getStartTime();
    long finishTime = taskStatus.getFinishTime();
    List<String> taskDiagnosticInfo = tip.getDiagnosticInfo(taskid);
    String diagInfo = taskDiagnosticInfo == null ? "" :
      StringUtils.arrayToString(taskDiagnosticInfo.toArray(new String[0]));
    TaskType taskType = getTaskType(tip);
    TaskAttemptStartedEvent tse = new TaskAttemptStartedEvent(
        taskid, taskType, startTime, taskTrackerName, taskTrackerPort, -1);
   
    jobHistory.logEvent(tse, taskid.getJobID());
View Full Code Here
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
    status.setFinishTime(JobTracker.getClock().getTime());
    boolean wasComplete = tip.isComplete();
    updateTaskStatus(tip, status);
    boolean isComplete = tip.isComplete();
    if (wasComplete && !isComplete) { // mark a successful tip as failed
      TaskType taskType = getTaskType(tip);
      TaskFailedEvent tfe =
        new TaskFailedEvent(tip.getTIPId(), tip.getExecFinishTime(), taskType,
            reason, TaskStatus.State.FAILED.toString(),
            taskid);
     
View Full Code Here
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
      long finishTime = task.getFinishTime();
      assertTrue("Task FINISH_TIME is < START_TIME in history file",
                 startTime < finishTime);

      // Make sure that the Task type exists and it is valid
      TaskType type = task.getTaskType();
      assertTrue("Unknown Task type \"" + type + "\" is seen in " +
                 "history file for task " + tid,
                 (type.equals(TaskType.MAP) ||
                  type.equals(TaskType.REDUCE) ||
                  type.equals(TaskType.JOB_CLEANUP) ||
                  type.equals(TaskType.JOB_SETUP)));

      if (type.equals(TaskType.MAP)) {
        String splits = task.getSplitLocations();
        //order in the condition OR check is important here
        if (!splitsCanBeEmpty || splits.length() != 0) {
          Matcher m = splitsPattern.matcher(splits);
          assertTrue("Unexpected format of SPLITS \"" + splits + "\" is seen" +
View Full Code Here
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
        long finishTime = attempt.getFinishTime();
        assertTrue("Task FINISH_TIME is < START_TIME in history file",
            startTime < finishTime);

        // Make sure that the Task type exists and it is valid
        TaskType type = attempt.getTaskType();
        assertTrue("Unknown Task type \"" + type + "\" is seen in " +
                   "history file for task attempt " + id,
                   (type.equals(TaskType.MAP) || type.equals(TaskType.REDUCE) ||
                    type.equals(TaskType.JOB_CLEANUP) ||
                    type.equals(TaskType.JOB_SETUP)));

        // Validate task status
        String status = attempt.getTaskStatus();
        assertTrue("Unexpected TASK_STATUS \"" + status + "\" is seen in" +
                   " history file for task attempt " + id,
                   (status.equals(TaskStatus.State.SUCCEEDED.toString()) ||
                    status.equals(TaskStatus.State.FAILED.toString()) ||
                    status.equals(TaskStatus.State.KILLED.toString())));

        // Successful Reduce Task Attempts should have valid SHUFFLE_FINISHED
        // time and SORT_FINISHED time
        if (type.equals(TaskType.REDUCE) &&
            status.equals(TaskStatus.State.SUCCEEDED.toString())) {
          long shuffleFinishTime = attempt.getShuffleFinishTime();
          assertTrue(startTime < shuffleFinishTime);
         
          long sortFinishTime = attempt.getSortFinishTime();
          assertTrue(shuffleFinishTime < sortFinishTime);
        }
        else if (type.equals(TaskType.MAP) &&
            status.equals(TaskStatus.State.SUCCEEDED.toString())) {
          // Successful MAP Task Attempts should have valid MAP_FINISHED time
         long mapFinishTime = attempt.getMapFinishTime();
         assertTrue(startTime < mapFinishTime);
        }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapreduce.TaskType

  • org.apache.blur.mapreduce.lib.BlurOutputCommitter
  • org.apache.hadoop.mapred.CapacityTaskScheduler$TaskSchedulingMgr$QueueComparator
  • org.apache.hadoop.mapred.CapacityTestUtils$FakeJobInProgress
  • org.apache.hadoop.mapred.CoronaJobTracker
  • org.apache.hadoop.mapred.FairScheduler
  • org.apache.hadoop.mapred.JobInProgress
  • org.apache.hadoop.mapred.SimulatorJobInProgress
  • org.apache.hadoop.mapred.taskdetailshistory_jsp
  • org.apache.hadoop.mapred.TestFairScheduler
  • org.apache.hadoop.mapred.TestFairScheduler$FakeJobInProgress
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.