Examples of org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobInProgress

Package org.apache.hadoop.mapred.FakeObjectUtilities

Examples of org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobInProgress

org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobInProgress

    conf.setFloat(JobInProgress.SPECULATIVE_STDDEVMEANRATIO_MAX, 0.33f);
    conf.setNumMapTasks(7);
    conf.setNumReduceTasks(0);
    conf.setFloat(JobInProgress.SPECULATIVE_MAP_UNFINISHED_THRESHOLD_KEY, 0);
    conf.setFloat(JobInProgress.SPECULATIVE_REDUCE_UNFINISHED_THRESHOLD_KEY, 0);
    FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker);
    job.initTasks();


    // all but one tasks start off
    taskAttemptID[0] = job.findMapTask(trackers[0]);
    taskAttemptID[1] = job.findMapTask(trackers[1]);
    taskAttemptID[2] = job.findMapTask(trackers[2]);
    taskAttemptID[3] = job.findMapTask(trackers[0]);
    taskAttemptID[4] = job.findMapTask(trackers[1]);
    taskAttemptID[5] = job.findMapTask(trackers[2]);


    // 3 tasks finish really fast in 15s
    clock.advance (15 * 1000);
    job.finishTask(taskAttemptID[0]);
    job.finishTask(taskAttemptID[1]);
    job.finishTask(taskAttemptID[2]);


    // advance to 600s and schedule last mapper
    clock.advance (585 * 1000);
    taskAttemptID[6] = job.findMapTask(trackers[0]);


    // advance to 700s and report progress
    clock.advance (10 * 60 * 1000);


    // set progress rates
    job.progressMade(taskAttemptID[3], 0.2f);
    job.progressMade(taskAttemptID[4], 0.5f);
    job.progressMade(taskAttemptID[5], 0.6f);
    job.progressMade(taskAttemptID[6], 0.02f);


    // the progress has been set in such a way that
    // stddev > mean. now we depend on stddev capping
    // for speculation.


    job.refresh(clock.getTime());
    taskAttemptID[7] = job.findMapTask(trackers[1]);


    // no new map task should be found
    if(taskAttemptID[7] ==  null)
      Assert.fail();
  }

View Full Code Here

    conf.setNumMapTasks(5);
    conf.setNumReduceTasks(0);
    conf.setFloat(JobInProgress.SPECULATIVE_MAP_UNFINISHED_THRESHOLD_KEY, 0);
    conf.setFloat(JobInProgress.SPECULATIVE_REDUCE_UNFINISHED_THRESHOLD_KEY, 0);
    conf.setFloat(JobInProgress.SPECULATIVE_SLOWTASK_THRESHOLD, 0.5f);
    FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker);
    job.initTasks();


    taskAttemptID[0] = job.findMapTask(trackers[0]);
    taskAttemptID[1] = job.findMapTask(trackers[1]);
    taskAttemptID[2] = job.findMapTask(trackers[2]);
    taskAttemptID[3] = job.findMapTask(trackers[3]);
    clock.advance(2000);
    job.finishTask(taskAttemptID[0]);
    job.finishTask(taskAttemptID[1]);
    job.finishTask(taskAttemptID[2]);
    clock.advance(250000);
    taskAttemptID[4] = job.findMapTask(trackers[3]);
    clock.advanceBySpeculativeLag();
    //by doing the above clock adjustments, we bring the progress rate of
    //taskID 3 lower than 4. For taskID 3, the rate is 85/317000
    //and for taskID 4, the rate is 20/65000. But when we ask for a spec task
    //now, we should get back taskID 4 (since that is expected to complete
    //later than taskID 3).
    job.refresh(clock.getTime());
    job.progressMade(taskAttemptID[3], 0.85f);
    job.progressMade(taskAttemptID[4], 0.20f);
    taskAttemptID[5] = job.findMapTask(trackers[4]);
    assertEquals(taskAttemptID[5].getTaskID().getId(),4);
  }

View Full Code Here

    conf.setNumReduceTasks(0);
    conf.setFloat(JobInProgress.SPECULATIVE_MAP_UNFINISHED_THRESHOLD_KEY, 0);
    conf.setFloat(JobInProgress.SPECULATIVE_REDUCE_UNFINISHED_THRESHOLD_KEY, 0);
    conf.setFloat(JobInProgress.SPECULATIVE_SLOWTASK_THRESHOLD, 0.5f);
    conf.setMapSpeculativeDuration(300L);
    FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker);
    job.initTasks();


    // a really fast task #1
    taskAttemptID[0] = job.findMapTask(trackers[0]);
    clock.advance(2000);
    job.finishTask(taskAttemptID[0]);


    // task #2 is slow
    taskAttemptID[1] = job.findMapTask(trackers[1]);
    clock.advanceBySpeculativeLag();
    clock.advance(5000);
    // 65 secs have elapsed since task scheduling
    // set progress so that it will complete within
    // 300 seconds
    job.progressMade(taskAttemptID[1], 0.7f);


    // no new map task should be found
    job.refresh(clock.getTime());
    assertEquals(job.findMapTask(trackers[2]), null);
  }

View Full Code Here

    JobConf conf = new JobConf();
    conf.setSpeculativeExecution(true);
    conf.setNumMapTasks(totalTasks);
    conf.setNumReduceTasks(0);
    jobTracker.setNumSlots(slots);
    FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker);
    job.initTasks();
    int i;
    for (i = 0; i < totalTasks; i++) {
      taskAttemptID[i] = job.findMapTask(trackers[0]);
    }
    clock.advance(5000);
    for (i = 0; i < numEarlyComplete; i++) {
      job.finishTask(taskAttemptID[i]);
    }


    clock.advanceBySpeculativeLag();


    for (i = numEarlyComplete; i < totalTasks; i++) {
      job.progressMade(taskAttemptID[i], 0.85f);
    }
    clock.advance(50000);
    for (i = 0; i < (totalTasks - numEarlyComplete); i++) {
      job.refresh(clock.getTime());
      taskAttemptID[i] = job.findMapTask(trackers[1]);
      clock.advance(2000);
      if (taskAttemptID[i] != null) {
        //add some good progress constantly for the different
        //task-attempts so that
        //the tasktracker doesn't get into the slow trackers category
        job.progressMade(taskAttemptID[i], 0.99f);
      } else {
        break;
      }
    }
    return i;

View Full Code Here

    TaskAttemptID[] taskAttemptID = new TaskAttemptID[8];
    JobConf conf = new JobConf();
    conf.setSpeculativeExecution(true);
    conf.setNumMapTasks(3);
    conf.setNumReduceTasks(3);
    FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker);
    job.initTasks();


    taskAttemptID[0] = job.findMapTask(trackers[0]);
    taskAttemptID[1] = job.findMapTask(trackers[1]);
    taskAttemptID[2] = job.findMapTask(trackers[2]);


    clock.advanceBySpeculativeLag();
    job.finishTask(taskAttemptID[0]);
    job.finishTask(taskAttemptID[1]);


    // Speculative last unfinised task
    job.refresh(clock.getTime());
    taskAttemptID[3] = job.findMapTask(trackers[3]);
    Assert.assertNotNull(taskAttemptID[3]);


    job.finishTask(taskAttemptID[2]);
    job.finishTask(taskAttemptID[3]);


    taskAttemptID[4] = job.findReduceTask(trackers[0]);
    taskAttemptID[5] = job.findReduceTask(trackers[1]);
    taskAttemptID[6] = job.findReduceTask(trackers[2]);


    clock.advanceBySpeculativeLag();
    job.finishTask(taskAttemptID[4]);
    job.finishTask(taskAttemptID[5]);


    // Speculative last unfinised task
    job.refresh(clock.getTime());
    taskAttemptID[7] = job.findReduceTask(trackers[3]);
    Assert.assertNotNull(taskAttemptID[7]);


    job.finishTask(taskAttemptID[6]);
    job.finishTask(taskAttemptID[7]);
  }

View Full Code Here

    conf.setSpeculativeExecution(true);
    conf.setNumMapTasks(3);
    conf.setNumReduceTasks(0);
    //use processing rate for speculation
    conf.setBoolean("mapreduce.job.speculative.using.processing.rate", true);
    FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker);
    job.initTasks();


    //schedule maps
    taskAttemptID[0] = job.findMapTask(trackers[0]);
    taskAttemptID[1] = job.findMapTask(trackers[1]);
    taskAttemptID[2] = job.findMapTask(trackers[2]);


    clock.advance(1000);
    job.finishTask(taskAttemptID[0]);
    //if consider the progress rate, we should speculate task 1
    //but if consider the processing rate, which is map_input_bytes/time
    //then we should speculate task 2
    job.processingRate(taskAttemptID[1], Task.Counter.MAP_INPUT_BYTES,
        100000000, 0.1f, TaskStatus.Phase.MAP);
    job.processingRate(taskAttemptID[2], Task.Counter.MAP_INPUT_BYTES,
        1000, 0.5f, TaskStatus.Phase.MAP);
    clock.advanceBySpeculativeLag();
    //we should get a speculative task now
    job.refresh(clock.getTime());
    taskAttemptID[3] = job.findMapTask(trackers[0]);


    assertEquals(taskAttemptID[3].getTaskID().getId(),2);
  }

View Full Code Here

    conf.setSpeculativeExecution(true);
    conf.setNumMapTasks(4);
    conf.setNumReduceTasks(4);
    //use processing rate for speculation
    conf.setBoolean("mapreduce.job.speculative.using.processing.rate", true);
    FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker);
    job.initTasks();


    //schedule reduces
    taskAttemptID[0] = job.findReduceTask(trackers[0]);
    taskAttemptID[1] = job.findReduceTask(trackers[1]);
    taskAttemptID[2] = job.findReduceTask(trackers[2]);
    taskAttemptID[3] = job.findReduceTask(trackers[3]);


    clock.advance(1000);


    //task 0 just starts copying, while task 1, 2, 3 are already in the reducing
    //phase. If we compared the progress rate, then we should speculate 0.
    //However, by comparing the processing rate in the copy phase, among all 4
    //tasks, task 0 is fast, and we should not speculate it.
    //for task 1, 2, 3, they are all in the reducing phase, with same progress,
    //however, task 1 has smaller processing rate(the statistics of the reduce
    //phase for all the tasks will also include statistics for task 0, whose
    //processing rate is 0)
    job.finishCopy(taskAttemptID[1], clock.getTime(), 10000);
    job.finishCopy(taskAttemptID[2], clock.getTime(), 10000);
    job.finishCopy(taskAttemptID[3], clock.getTime(), 10000);
    clock.advance(1000);
    job.finishSort(taskAttemptID[1], clock.getTime());
    job.finishSort(taskAttemptID[2], clock.getTime());
    job.finishSort(taskAttemptID[3], clock.getTime());
    job.processingRate(taskAttemptID[0], Task.Counter.REDUCE_SHUFFLE_BYTES,
        100000000, 0.1f, TaskStatus.Phase.SHUFFLE);
    job.processingRate(taskAttemptID[1], Task.Counter.REDUCE_INPUT_BYTES,
        1000, 0.8f, TaskStatus.Phase.REDUCE);
    job.processingRate(taskAttemptID[2], Task.Counter.REDUCE_INPUT_BYTES,
        100000000, 0.8f, TaskStatus.Phase.REDUCE);
    job.processingRate(taskAttemptID[3], Task.Counter.REDUCE_INPUT_BYTES,
        100000000, 0.8f, TaskStatus.Phase.REDUCE);
    clock.advanceBySpeculativeLag();
    //we should get a speculative task now
    job.refresh(clock.getTime());
    taskAttemptID[4] = job.findReduceTask(trackers[4]);
    assertEquals(taskAttemptID[4].getTaskID().getId(),1);
  }

View Full Code Here

    conf.setNumMapTasks(3);
    conf.setNumReduceTasks(3);
    conf.setFloat(JobInProgress.SPECULATIVE_SLOWTASK_THRESHOLD, 0.5f);
    conf.setFloat(JobInProgress.SPECULATIVE_MAP_UNFINISHED_THRESHOLD_KEY, 0);
    conf.setFloat(JobInProgress.SPECULATIVE_REDUCE_UNFINISHED_THRESHOLD_KEY, 0);
    FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker);
    job.initTasks();


    //Check for runningMap counts first
    //schedule maps
    taskAttemptID[0] = job.findMapTask(trackers[0]);
    taskAttemptID[1] = job.findMapTask(trackers[1]);
    taskAttemptID[2] = job.findMapTask(trackers[2]);


    clock.advance(5000);
    job.finishTask(taskAttemptID[0]);
    clock.advance(1000);
    job.finishTask(taskAttemptID[1]);
    clock.advanceBySpeculativeLag();


    //we should get a speculative task now
    job.refresh(clock.getTime());
    taskAttemptID[3] = job.findMapTask(trackers[3]);
    job.refresh(clock.getTime());
    int oldRunningMap = job.runningMaps();
    LOG.info("No of running maps before fail was " + oldRunningMap);
    job.failTask(taskAttemptID[2]);
    job.refresh(clock.getTime());
    assertEquals(
      "Running maps count should be updated from " + oldRunningMap + " to " +
        (oldRunningMap - 1), job.runningMaps(), oldRunningMap - 1);
    LOG.info(" Job running maps after fail " + job.runningMaps());


    clock.advance(5000);
    job.finishTask(taskAttemptID[3]);


    //check for runningReduce count.
    taskAttemptID[4] = job.findReduceTask(trackers[0]);
    taskAttemptID[5] = job.findReduceTask(trackers[1]);
    taskAttemptID[6] = job.findReduceTask(trackers[2]);


    clock.advance(5000);
    job.finishTask(taskAttemptID[4]);
    clock.advance(1000);
    job.finishTask(taskAttemptID[5]);


    job.refresh(clock.getTime());
    clock.advanceBySpeculativeLag();
    taskAttemptID[7] = job.findReduceTask(trackers[4]);


    job.refresh(clock.getTime());
    int oldRunningReduces = job.runningReduces();
    job.failTask(taskAttemptID[6]);
    job.refresh(clock.getTime());
    LOG.info(
      " No of running Reduces before fail " + oldRunningReduces);
    LOG.info(
      " No of runing reduces after fail " + job.runningReduces());
    assertEquals(
      "Running reduces count should be updated from " + oldRunningReduces +
        " to " + (oldRunningReduces - 1), job.runningReduces(),
      oldRunningReduces - 1);


    job.finishTask(taskAttemptID[7]);
  }

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobInProgress

org.apache.hadoop.mapred.TestClusterStatus

org.apache.hadoop.mapred.TestLostTracker

org.apache.hadoop.mapred.TestSetupTaskScheduling

org.apache.hadoop.mapred.TestSpeculativeExecution

org.apache.hadoop.mapred.TestTrackerReservation

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.