Examples of org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobInProgress

Package org.apache.hadoop.mapred.FakeObjectUtilities

Examples of org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobInProgress

org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobInProgress

   */
  public void testNumSlotsUsedForTaskCleanup() throws IOException {
    // Create a high RAM job with a map task's cleanup task and a reduce task's
    // cleanup task. Make this Fake job a high RAM job by setting the slots
    // required for map/reduce task to 2.
    FakeJobInProgress job = createJob(TaskType.TASK_CLEANUP);
    jobTracker.jobs.put(job.getJobID(), job);
   
    // create TT status for testing getSetupAndCleanupTasks
    List<TaskStatus> taskStatuses = new ArrayList<TaskStatus>();
    TaskTrackerStatus ttStatus =
      createTaskTrackerStatus(trackers[0], taskStatuses);//create dummy status

View Full Code Here

    conf.setSpeculativeExecution(false);
    conf.setNumMapTasks(2);
    conf.setNumReduceTasks(2);
    conf.set(JobContext.REDUCE_FAILURES_MAXPERCENT, ".70");
    conf.set(JobContext.MAP_FAILURES_MAX_PERCENT, ".70");
    FakeJobInProgress job = null;
    if (taskType == null) {
      conf.setBoolean(JobContext.SETUP_CLEANUP_NEEDED, false);
      job = new FakeJobInProgress(conf, jobTracker);
    } else if (taskType == TaskType.JOB_SETUP) {
      job = new FakeJobWithSetupTask(conf, jobTracker);
    } else if (taskType == TaskType.TASK_CLEANUP) {
      job = new FakeJobWithTaskCleanupTask(conf, jobTracker);
    }
    job.setClusterSize(trackers.length);
    job.initTasks();
    return job;
  }

View Full Code Here

   * if it is free.
   * @throws IOException
   */
  public void testSetupTaskReturnedForFreeMapSlots() throws IOException {
    // create a job with a setup task.
    FakeJobInProgress job = createJob(TaskType.JOB_SETUP);
    jobTracker.jobs.put(job.getJobID(), job);
    
    // create a status simulating a free tasktracker
    List<TaskStatus> reports = new ArrayList<TaskStatus>();
    TaskTrackerStatus ttStatus 
      = createTaskTrackerStatus(trackers[2], reports);

View Full Code Here

   * a setup task.
   * @throws IOException
   */
  public void testMapSlotsCountedForSetup() throws IOException {
    // create a job with a setup task.
    FakeJobInProgress job = createJob(TaskType.JOB_SETUP);
    jobTracker.jobs.put(job.getJobID(), job);
    
    // create another job for reservation
    FakeJobInProgress job1 = createJob(null);
    jobTracker.jobs.put(job1.getJobID(), job1);
   
    // create TT status for testing getSetupAndCleanupTasks
    List<TaskStatus> taskStatuses = new ArrayList<TaskStatus>();
    addNewTaskStatus(job, TaskType.MAP, true, trackers[0], taskStatuses);
    TaskTrackerStatus ttStatus

View Full Code Here

   * a setup task.
   * @throws IOException
   */
  public void testReduceSlotsCountedForSetup() throws IOException {
    // create a job with a setup task.
    FakeJobInProgress job = createJob(TaskType.JOB_SETUP);
    jobTracker.jobs.put(job.getJobID(), job);
    
    // create another job for reservation
    FakeJobInProgress job1 = createJob(null);
    jobTracker.jobs.put(job1.getJobID(), job1);
    
    // create TT status for testing getSetupAndCleanupTasks
    List<TaskStatus> reports = new ArrayList<TaskStatus>();
    // because free map slots are checked first in code,
    // we fill up map slots also.

View Full Code Here

   */
  public void testNumSlotsUsedForTaskCleanup() throws IOException {
    // Create a high RAM job with a map task's cleanup task and a reduce task's
    // cleanup task. Make this Fake job a high RAM job by setting the slots
    // required for map/reduce task to 2.
    FakeJobInProgress job = createJob(TaskType.TASK_CLEANUP);
    jobTracker.jobs.put(job.getJobID(), job);
   
    // create TT status for testing getSetupAndCleanupTasks
    List<TaskStatus> taskStatuses = new ArrayList<TaskStatus>();
    TaskTrackerStatus ttStatus =
      createTaskTrackerStatus(trackers[0], taskStatuses);//create dummy status

View Full Code Here


    TaskAttemptID[] tid = new TaskAttemptID[2];
    JobConf conf = new JobConf();
    conf.setNumMapTasks(1);
    conf.setNumReduceTasks(1);
    FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker);
    job.initTasks();
    
    // Tracker 0 gets the map task
    tid[0] = job.findMapTask(trackers[0]);


    job.finishTask(tid[0]);


    // Advance clock. Tracker 0 would have got lost
    clock.advance(8 * 1000);


    jobTracker.checkExpiredTrackers();
    
    // Tracker 1 establishes contact with JT 
    FakeObjectUtilities.establishFirstContact(jobTracker, trackers[1]);
    
    // Tracker1 should get assigned the lost map task
    tid[1] =  job.findMapTask(trackers[1]);


    assertNotNull("Map Task from Lost Tracker did not get reassigned", tid[1]);
    
    assertEquals("Task ID of reassigned map task does not match",
        tid[0].getTaskID().toString(), tid[1].getTaskID().toString());
    
    job.finishTask(tid[1]);
    
  }

View Full Code Here

    JobConf conf = new JobConf();
    conf.setNumMapTasks(1);
    conf.setNumReduceTasks(1);
    conf.set(MRJobConfig.MAX_TASK_FAILURES_PER_TRACKER, "1");
    conf.set(MRJobConfig.SETUP_CLEANUP_NEEDED, "false");
    FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker);
    job.initTasks();
    job.setClusterSize(4);
    
    // Tracker 0 gets the map task
    tid[0] = job.findMapTask(trackers[0]);


    job.finishTask(tid[0]);


    // validate the total tracker count
    assertEquals("Active tracker count mismatch", 
                 1, jobTracker.getClusterStatus(false).getTaskTrackers());
    
    // lose the tracker
    clock.advance(1100);
    jobTracker.checkExpiredTrackers();
    assertFalse("Tracker 0 not lost", 
        jobTracker.getClusterStatus(false).getActiveTrackerNames()
                  .contains(trackers[0]));
    
    // validate the total tracker count
    assertEquals("Active tracker count mismatch", 
                 0, jobTracker.getClusterStatus(false).getTaskTrackers());
    
    // Tracker 1 establishes contact with JT 
    FakeObjectUtilities.establishFirstContact(jobTracker, trackers[1]);
    
    // Tracker1 should get assigned the lost map task
    tid[1] =  job.findMapTask(trackers[1]);


    assertNotNull("Map Task from Lost Tracker did not get reassigned", tid[1]);
    
    assertEquals("Task ID of reassigned map task does not match",
        tid[0].getTaskID().toString(), tid[1].getTaskID().toString());
    
    // finish the map task
    job.finishTask(tid[1]);


    // finish the reduce task
    tid[2] =  job.findReduceTask(trackers[1]);
    job.finishTask(tid[2]);
    
    // check if job is successful
    assertEquals("Job not successful", 
                 JobStatus.SUCCEEDED, job.getStatus().getRunState());
    
    // check if the tracker is lost
    // validate the total tracker count
    assertEquals("Active tracker count mismatch", 
                 1, jobTracker.getClusterStatus(false).getTaskTrackers());

View Full Code Here

    JobConf conf = new JobConf();
    conf.setNumMapTasks(1);
    conf.setNumReduceTasks(0);
    conf.set(MRJobConfig.MAX_TASK_FAILURES_PER_TRACKER, "1");
    conf.set(MRJobConfig.SETUP_CLEANUP_NEEDED, "false");
    FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker);
    job.initTasks();
    job.setClusterSize(4);
    
    // check if the tracker count is correct
    assertEquals("Active tracker count mismatch", 
                 1, jobTracker.taskTrackers().size());
    
    // Tracker 0 gets the map task
    tid[0] = job.findMapTask(trackers[0]);
    // Fail the task
    job.failTask(tid[0]);
    
    // Tracker 1 establishes contact with JT
    FakeObjectUtilities.establishFirstContact(jobTracker, trackers[1]);
    // check if the tracker count is correct
    assertEquals("Active tracker count mismatch", 
                 2, jobTracker.taskTrackers().size());
    
    // Tracker 1 gets the map task
    tid[1] = job.findMapTask(trackers[1]);
    // Finish the task and also the job
    job.finishTask(tid[1]);


    // check if job is successful
    assertEquals("Job not successful", 
                 JobStatus.SUCCEEDED, job.getStatus().getRunState());
    
    // check if the trackers 1 got blacklisted
    assertTrue("Tracker 0 not blacklisted", 
               jobTracker.getBlacklistedTrackers()[0].getTaskTrackerName()
                 .equals(trackers[0]));

View Full Code Here

        trackers[1],JobInProgress.convertTrackerNameToHostName(
            trackers[1]),0,new ArrayList<TaskStatus>(), 0, 2, 2);
    tt1.setStatus(status1);
    tt2.setStatus(status2);
    
    fakeJob = new FakeJobInProgress(new JobConf(job.getConfiguration()),
                    jobTracker);
    fakeJob.setClusterSize(3);
    fakeJob.initTasks();
    
    FakeObjectUtilities.sendHeartBeat(jobTracker, status1, false,

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobInProgress

org.apache.hadoop.mapred.TestClusterStatus

org.apache.hadoop.mapred.TestLostTracker

org.apache.hadoop.mapred.TestSetupTaskScheduling

org.apache.hadoop.mapred.TestSpeculativeExecution

org.apache.hadoop.mapred.TestTrackerReservation

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.