Package org.apache.hadoop.mapred.TaskTracker

Examples of org.apache.hadoop.mapred.TaskTracker.TaskInProgress


      return;
    }

    task.setTaskCleanupTask();
    // register task
    tip = tracker.new TaskInProgress(task, trackerFConf);

    // localize the job again.
    RunningJob rjob = tracker.localizeJob(tip);
    localizedJobConf = rjob.getJobConf();
    checkJobLocalization();
View Full Code Here


                    + currentMemUsage + "bytes. Limit : " + limit
                    + "bytes. Killing task. \nDump of the process-tree for "
                    + tid + " : \n" + pTree.getProcessTreeDump();
            LOG.warn(msg);
            // kill the task
            TaskInProgress tip = taskTracker.runningTasks.get(tid);
            if (tip != null) {
              String[] diag = msg.split("\n");
              tip.getStatus().setDiagnosticInfo(diag[0]);
            }
            taskTracker.cleanUpOverMemoryTask(tid, true, msg);

            it.remove();
            LOG.info("Removed ProcessTree with root " + pId);
View Full Code Here

    List<TaskAttemptID> tasksToExclude = new ArrayList<TaskAttemptID>();
    // Find tasks to kill so as to get memory usage under limits.
    while (memoryStillInUsage > maxMemoryAllowedForAllTasks) {
      // Exclude tasks that are already marked for
      // killing.
      TaskInProgress task = taskTracker.findTaskToKill(tasksToExclude);
      if (task == null) {
        break; // couldn't find any more tasks to kill.
      }

      TaskAttemptID tid = task.getTask().getTaskID();
      if (processTreeInfoMap.containsKey(tid)) {
        ProcessTreeInfo ptInfo = processTreeInfoMap.get(tid);
        ProcfsBasedProcessTree pTree = ptInfo.getProcessTree();
        memoryStillInUsage -= pTree.getCumulativeVmem();
        tasksToKill.add(tid);
      }
      // Exclude this task from next search because it is already
      // considered.
      tasksToExclude.add(tid);
    }

    // Now kill the tasks.
    if (!tasksToKill.isEmpty()) {
      for (TaskAttemptID tid : tasksToKill) {
        String msg =
            "Killing one of the least progress tasks - " + tid
                + ", as the cumulative memory usage of all the tasks on "
                + "the TaskTracker " + taskTracker.localHostname
                + " exceeds virtual memory limit "
                + maxMemoryAllowedForAllTasks + ".";
        TaskInProgress tip = taskTracker.runningTasks.get(tid);
        if (tip != null) {
           tip.getStatus().setDiagnosticInfo(msg);
        }
        LOG.warn(msg);
        // Kill the task and mark it as killed.
        taskTracker.cleanUpOverMemoryTask(tid, false, msg);
        // Now destroy the ProcessTree, remove it from monitoring map.
View Full Code Here

    String user = "test";
    JobConf taskConf = new JobConf(ttConf);
    TaskAttemptID attemptID = new TaskAttemptID("test", 0, true, 0, 0);
    Task task = new MapTask(null, attemptID, 0, null, MAP_SLOTS);
    task.setConf(taskConf);
    TaskInProgress tip = tt.new TaskInProgress(task, taskConf);
    RunningJob rjob = new RunningJob(attemptID.getJobID());
    TaskController taskController = new DefaultTaskController();
    taskController.setConf(ttConf);
    rjob.distCacheMgr =
      new TrackerDistributedCacheManager(ttConf, taskController).
View Full Code Here

    String jtId = "test";
    TaskAttemptID attemptID = new TaskAttemptID(jtId, 1, true, 0, 0);
    Task task = new MapTask(null, attemptID, 0, null, 5);
    mapLauncher.addToTaskQueue(new LaunchTaskAction(task));
    // verify that task is added to runningTasks
    TaskInProgress killTip = tt.runningTasks.get(attemptID);
    assertNotNull(killTip);

    // wait for a while for launcher to pick up the task
    // this loop waits atmost for 30 seconds
    for (int i = 0; i < 300; i++) {
      if (mapLauncher.getNumWaitingTasksToLaunch() == 0) {
        break;
      }
      UtilsForTests.waitFor(100);
    }
    assertEquals("Launcher didnt pick up the task " + attemptID + "to launch",
        0, mapLauncher.getNumWaitingTasksToLaunch());

    // Now, that launcher has picked up the task, it waits until all five slots
    // are available. i.e. it waits for-ever
    // lets kill the task so that map launcher comes out
    tt.processKillTaskAction(new KillTaskAction(attemptID));
    assertEquals(TaskStatus.State.KILLED, killTip.getRunState());

    // launch another attempt which requires only one slot
    TaskAttemptID runningAttemptID = new TaskAttemptID(jtId, 1, true,
        0, expectedLaunchAttemptId);
    mapLauncher.addToTaskQueue(new LaunchTaskAction(new MapTask(null,
        runningAttemptID, 0, null, 1)));
    TaskInProgress runningTip = tt.runningTasks.get(runningAttemptID);
    assertNotNull(runningTip);

    // wait for a while for the task to be launched
    // this loop waits at most for 30 seconds
    for (int i = 0; i < 300; i++) {
      if (runningTip.getRunState().equals(TaskStatus.State.RUNNING)) {
        break;
      }
      UtilsForTests.waitFor(100);
    }

    // verify that the task went to running
    assertEquals(TaskStatus.State.RUNNING, runningTip.getRunState());
  }
View Full Code Here

    // launch a jvm
    JobConf taskConf = new JobConf(ttConf);
    TaskAttemptID attemptID = new TaskAttemptID("test", 0, TaskType.MAP, 0, 0);
    MapTask task = new MapTask(null, attemptID, 0, null, 1);
    task.setConf(taskConf);
    TaskInProgress tip = tt.new TaskInProgress(task, taskConf);
    File pidFile = new File(TEST_DIR, "pid");
    final TaskRunner taskRunner = new MapTaskRunner(tip, tt, taskConf);
    // launch a jvm which sleeps for 60 seconds
    final Vector<String> vargs = new Vector<String>(2);
    vargs.add(writeScript("SLEEP", "sleep 60\n", pidFile).getAbsolutePath());
    final File workDir = new File(TEST_DIR, "work");
    workDir.mkdir();
    final File stdout = new File(TEST_DIR, "stdout");
    final File stderr = new File(TEST_DIR, "stderr");

    // launch the process and wait in a thread, till it finishes
    Thread launcher = new Thread() {
      public void run() {
        try {
          taskRunner.launchJvmAndWait(null, vargs, stdout, stderr, 100,
              workDir, null);
        } catch (InterruptedException e) {
          e.printStackTrace();
          return;
        }
      }
    };
    launcher.start();
    // wait till the jvm is launched
    // this loop waits for at most 1 second
    for (int i = 0; i < 10; i++) {
      if (pidFile.exists()) {
        break;
      }
      UtilsForTests.waitFor(100);
    }
    // assert that the process is launched
    assertTrue("pidFile is not present", pidFile.exists());
   
    // imitate Child code.
    // set pid in jvmManager
    BufferedReader in = new  BufferedReader(new FileReader(pidFile));
    String pid = in.readLine();
    in.close();
    JVMId jvmid = mapJvmManager.runningTaskToJvm.get(taskRunner);
    jvmManager.setPidToJvm(jvmid, pid);

    // kill JvmRunner
    final JvmRunner jvmRunner = mapJvmManager.jvmIdToRunner.get(jvmid);
    Thread killer = new Thread() {
      public void run() {
        jvmRunner.kill();
      }
    };
    killer.start();
   
    //wait for a while so that killer thread is started.
    Thread.sleep(100);

    // kill the jvm externally
    taskRunner.kill();

    assertTrue(jvmRunner.killed);

    // launch another jvm and see it finishes properly
    attemptID = new TaskAttemptID("test", 0, TaskType.MAP, 0, 1);
    task = new MapTask(null, attemptID, 0, null, 1);
    task.setConf(taskConf);
    tip = tt.new TaskInProgress(task, taskConf);
    TaskRunner taskRunner2 = new MapTaskRunner(tip, tt, taskConf);
    // build dummy vargs to call ls
    Vector<String> vargs2 = new Vector<String>(1);
    vargs2.add(writeScript("LS", "ls", pidFile).getAbsolutePath());
    File workDir2 = new File(TEST_DIR, "work2");
View Full Code Here

    for (int i = 0; i < MAP_SLOTS*5; i++) {
      JobConf taskConf = new JobConf(ttConf);
      TaskAttemptID attemptID = new TaskAttemptID("test", 0, TaskType.MAP, i, 0);
      Task task = new MapTask(null, attemptID, i, null, 1);
      task.setConf(taskConf);
      TaskInProgress tip = tt.new TaskInProgress(task, taskConf);
      File pidFile = new File(TEST_DIR, "pid_" + i);
      final TaskRunner taskRunner = new MapTaskRunner(tip, tt, taskConf);
      // launch a jvm which sleeps for 60 seconds
      final Vector<String> vargs = new Vector<String>(2);
      vargs.add(writeScript("script_" + i, "echo hi\n", pidFile).getAbsolutePath());
View Full Code Here

    String jtId = "test";
    TaskAttemptID attemptID = new TaskAttemptID(jtId, 1, true, 0, 0);
    Task task = new MapTask(null, attemptID, 0, null, 2);
    mapLauncher.addToTaskQueue(new LaunchTaskAction(task));
    // verify that task is added to runningTasks
    TaskInProgress runningTip = tt.runningTasks.get(attemptID);
    assertNotNull(runningTip);

    // wait for a while for the first task to start initializing
    // this loop waits at most for 30 seconds
    for (int i = 0; i < 300; i++) {
      if (firstJobStarted == true) {
        break;
      }
      UtilsForTests.waitFor(100);
    }

    // Now start a second task and make sure it doesn't wait while first one initializes
    String secondjtId = "test2";
    TaskAttemptID secondAttemptID = new TaskAttemptID(secondjtId, 1, true, 0, 0);
    Task secondTask = new MapTask(null, secondAttemptID, 0, null, 2);
    mapLauncher.addToTaskQueue(new LaunchTaskAction(secondTask));
    // verify that task is added to runningTasks
    TaskInProgress secondRunningTip = tt.runningTasks.get(secondAttemptID);
    assertNotNull(secondRunningTip);

    // wait for a while for the second task to be launched
    // this loop waits at most for 30 seconds
    for (int i = 0; i < 300; i++) {
View Full Code Here

          if (pId == null) {
            continue; // processTree cannot be tracked
          }

          TaskInProgress tip = taskTracker.getRunningTask(tid);
          if (tip == null) {
            continue;
          }
          if (tip.wasKilled()) {
            continue;
          }

          LOG.debug("Constructing ProcessTree for : PID = " + pId + " TID = "
              + tid);
View Full Code Here

    // Find tasks to kill so as to get memory usage under limits.
    while (memoryStillInUsage > maxMemoryAllowedForAllTasks) {
      // Exclude tasks that are already marked for killing.
      // Note that we do not need to call isKillable() here because the logic
      // is contained in taskTracker.findTaskToKill()
      TaskInProgress task = taskTracker.findTaskToKill(tasksToExclude);
      if (task == null) {
        break; // couldn't find any more tasks to kill.
      }

      TaskAttemptID tid = task.getTask().getTaskID();
      if (processTreeInfoMap.containsKey(tid)) {
        ProcessTreeInfo ptInfo = processTreeInfoMap.get(tid);
        ProcfsBasedProcessTree pTree = ptInfo.getProcessTree();
        memoryStillInUsage -= pTree.getCumulativeVmem();
        tasksToKill.add(tid);
View Full Code Here

TOP

Related Classes of org.apache.hadoop.mapred.TaskTracker.TaskInProgress

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.