Examples of org.apache.hadoop.util.ProcfsBasedProcessTree$SigKillThread

org.apache.hadoop.util.ProcfsBasedProcessTree
Helper thread class that kills process-tree with SIGKILL in background

            // or if the tip is finished and we removed pidFile, but the TIP
            // itself is still retained in runningTasks till successful
            // transmission to JT


            // create process tree object
            ProcfsBasedProcessTree pt = new ProcfsBasedProcessTree(pId);
            LOG.debug("Tracking ProcessTree " + pId + " for the first time");


            ptInfo.setPid(pId);
            ptInfo.setProcessTree(pt);
            processTreeInfoMap.put(tid, ptInfo);
          }
        }
        // End of initializing any uninitialized processTrees


        if (pId == null) {
          continue; // processTree cannot be tracked
        }


        LOG.debug("Constructing ProcessTree for : PID = " + pId + " TID = "
            + tid);
        ProcfsBasedProcessTree pTree = ptInfo.getProcessTree();
        pTree = pTree.getProcessTree(); // get the updated process-tree
        ptInfo.setProcessTree(pTree); // update ptInfo with proces-tree of
                                      // updated state
        long currentMemUsage = pTree.getCumulativeVmem();
        long limit = ptInfo.getMemLimit();
        LOG.info("Memory usage of ProcessTree " + pId + " :" + currentMemUsage
            + "kB. Limit : " + limit + "kB");


        if (limit != JobConf.DISABLED_VIRTUAL_MEMORY_LIMIT
            && currentMemUsage > limit) {
          // Task (the root process) is still alive and overflowing memory.
          // Clean up.
          String msg = "TaskTree [pid=" + pId + ",tipID=" + tid
              + "] is running beyond memory-limits. Current usage : "
              + currentMemUsage + "kB. Limit : " + limit + "kB. Killing task.";
          LOG.warn(msg);
          taskTracker.cleanUpOverMemoryTask(tid, msg);


          // Now destroy the ProcessTree, remove it from monitoring map.
          pTree.destroy();
          it.remove();
          LOG.info("Removed ProcessTree with root " + pId);
        }
      }

View Full Code Here

              // or if the tip is finished and we removed pidFile, but the TIP
              // itself is still retained in runningTasks till successful
              // transmission to JT


              // create process tree object
              ProcfsBasedProcessTree pt = new ProcfsBasedProcessTree(pId);
              LOG.debug("Tracking ProcessTree " + pId + " for the first time");


              ptInfo.setPid(pId);
              ptInfo.setProcessTree(pt);
            }
          }
          // End of initializing any uninitialized processTrees


          if (pId == null) {
            continue; // processTree cannot be tracked
          }


          LOG.debug("Constructing ProcessTree for : PID = " + pId + " TID = "
              + tid);
          ProcfsBasedProcessTree pTree = ptInfo.getProcessTree();
          pTree = pTree.getProcessTree(); // get the updated process-tree
          ptInfo.setProcessTree(pTree); // update ptInfo with proces-tree of
          // updated state
          long currentMemUsage = pTree.getCumulativeVmem();
          long limit = ptInfo.getMemLimit();
          LOG.info("Memory usage of ProcessTree " + pId + " :"
              + currentMemUsage + "bytes. Limit : " + limit + "bytes");


          if (limit > taskTracker.getLimitMaxVMemPerTask()) {
            // TODO: With monitoring enabled and no scheduling based on
            // memory,users can seriously hijack the system by specifying memory
            // requirements well above the cluster wide limit. Ideally these
            // jobs
            // should have been rejected by JT/scheduler. Because we can't do
            // that, in the minimum we should fail the tasks and hence the job.
            LOG.warn("Task " + tid
                + " 's maxVmemPerTask is greater than TT's limitMaxVmPerTask");
          }


          if (limit != JobConf.DISABLED_MEMORY_LIMIT
              && currentMemUsage > limit) {
            // Task (the root process) is still alive and overflowing memory.
            // Clean up.
            String msg =
                "TaskTree [pid=" + pId + ",tipID=" + tid
                    + "] is running beyond memory-limits. Current usage : "
                    + currentMemUsage + "bytes. Limit : " + limit
                    + "bytes. Killing task.";
            LOG.warn(msg);
            taskTracker.cleanUpOverMemoryTask(tid, true, msg);


            // Now destroy the ProcessTree, remove it from monitoring map.
            pTree.destroy();
            it.remove();
            LOG.info("Removed ProcessTree with root " + pId);
          } else {
            // Accounting the total memory in usage for all tasks that are still
            // alive and within limits.

View Full Code Here

      }


      TaskAttemptID tid = task.getTask().getTaskID();
      if (processTreeInfoMap.containsKey(tid)) {
        ProcessTreeInfo ptInfo = processTreeInfoMap.get(tid);
        ProcfsBasedProcessTree pTree = ptInfo.getProcessTree();
        memoryStillInUsage -= pTree.getCumulativeVmem();
        tasksToKill.add(tid);
      }
      // Exclude this task from next search because it is already
      // considered.
      tasksToExclude.add(tid);
    }


    // Now kill the tasks.
    if (!tasksToKill.isEmpty()) {
      for (TaskAttemptID tid : tasksToKill) {
        String msg =
            "Killing one of the least progress tasks - " + tid
                + ", as the cumulative memory usage of all the tasks on "
                + "the TaskTracker exceeds virtual memory limit "
                + maxMemoryAllowedForAllTasks + ".";
        LOG.warn(msg);
        // Kill the task and mark it as killed.
        taskTracker.cleanUpOverMemoryTask(tid, false, msg);
        // Now destroy the ProcessTree, remove it from monitoring map.
        ProcessTreeInfo ptInfo = processTreeInfoMap.get(tid);
        ProcfsBasedProcessTree pTree = ptInfo.getProcessTree();
        pTree.destroy();
        processTreeInfoMap.remove(tid);
        LOG.info("Removed ProcessTree with root " + ptInfo.getPID());
      }
    } else {
      LOG.info("The total memory usage is overflowing TTs limits. "

View Full Code Here

            // or if the tip is finished and we removed pidFile, but the TIP
            // itself is still retained in runningTasks till successful
            // transmission to JT


            // create process tree object
            ProcfsBasedProcessTree pt = new ProcfsBasedProcessTree(pId);
            LOG.debug("Tracking ProcessTree " + pId + " for the first time");


            ptInfo.setPid(pId);
            ptInfo.setProcessTree(pt);
            processTreeInfoMap.put(tid, ptInfo);
          }
        }
        // End of initializing any uninitialized processTrees


        if (pId == null) {
          continue; // processTree cannot be tracked
        }


        LOG.debug("Constructing ProcessTree for : PID = " + pId + " TID = "
            + tid);
        ProcfsBasedProcessTree pTree = ptInfo.getProcessTree();
        pTree = pTree.getProcessTree(); // get the updated process-tree
        ptInfo.setProcessTree(pTree); // update ptInfo with proces-tree of
                                      // updated state
        long currentMemUsage = pTree.getCumulativeVmem();
        long limit = ptInfo.getMemLimit();
        LOG.info("Memory usage of ProcessTree " + pId + " :" + currentMemUsage
            + "kB. Limit : " + limit + "kB");


        if (limit != JobConf.DISABLED_VIRTUAL_MEMORY_LIMIT
            && currentMemUsage > limit) {
          // Task (the root process) is still alive and overflowing memory.
          // Clean up.
          String msg = "TaskTree [pid=" + pId + ",tipID=" + tid
              + "] is running beyond memory-limits. Current usage : "
              + currentMemUsage + "kB. Limit : " + limit + "kB. Killing task.";
          LOG.warn(msg);
          taskTracker.cleanUpOverMemoryTask(tid, msg);


          // Now destroy the ProcessTree, remove it from monitoring map.
          pTree.destroy();
          it.remove();
          LOG.info("Removed ProcessTree with root " + pId);
        }
      }

View Full Code Here

              // PID will be null, either if the pid file is yet to be created
              // or if the tip is finished and we removed pidFile, but the TIP
              // itself is still retained in runningTasks till successful
              // transmission to JT


              ProcfsBasedProcessTree pt = 
                new ProcfsBasedProcessTree(pId, ProcessTree.isSetsidAvailable);
              LOG.debug("Tracking ProcessTree " + pId + " for the first time");


              ptInfo.setPid(pId);
              ptInfo.setProcessTree(pt);
            }
          }
          // End of initializing any uninitialized processTrees


          if (pId == null) {
            continue; // processTree cannot be tracked
          }


          LOG.debug("Constructing ProcessTree for : PID = " + pId + " TID = "
              + tid);
          ProcfsBasedProcessTree pTree = ptInfo.getProcessTree();
          pTree = pTree.getProcessTree(); // get the updated process-tree
          ptInfo.setProcessTree(pTree); // update ptInfo with proces-tree of
          // updated state
          long currentMemUsage = pTree.getCumulativeVmem();
          // as processes begin with an age 1, we want to see if there 
          // are processes more than 1 iteration old.
          long curMemUsageOfAgedProcesses = pTree.getCumulativeVmem(1);
          long limit = ptInfo.getMemLimit();
          LOG.info(String.format(MEMORY_USAGE_STRING, 
                                pId, tid.toString(), currentMemUsage, limit));


          if (isProcessTreeOverLimit(tid.toString(), currentMemUsage, 
                                      curMemUsageOfAgedProcesses, limit)) {
            // Task (the root process) is still alive and overflowing memory.
            // Dump the process-tree and then clean it up.
            String msg =
                "TaskTree [pid=" + pId + ",tipID=" + tid
                    + "] is running beyond memory-limits. Current usage : "
                    + currentMemUsage + "bytes. Limit : " + limit
                    + "bytes. Killing task. \nDump of the process-tree for "
                    + tid + " : \n" + pTree.getProcessTreeDump();
            LOG.warn(msg);
            // kill the task
            TaskInProgress tip = taskTracker.runningTasks.get(tid);
            if (tip != null) {
              String[] diag = msg.split("\n");

View Full Code Here

      }


      TaskAttemptID tid = task.getTask().getTaskID();
      if (processTreeInfoMap.containsKey(tid)) {
        ProcessTreeInfo ptInfo = processTreeInfoMap.get(tid);
        ProcfsBasedProcessTree pTree = ptInfo.getProcessTree();
        memoryStillInUsage -= pTree.getCumulativeVmem();
        tasksToKill.add(tid);
      }
      // Exclude this task from next search because it is already
      // considered.
      tasksToExclude.add(tid);

View Full Code Here

      TaskMemoryManagerThread test = new TaskMemoryManagerThread(1000000L,
                                                                5000L);
      // create process trees
      // tree rooted at 100 is over limit immediately, as it is
      // twice over the mem limit.
      ProcfsBasedProcessTree pTree = new ProcfsBasedProcessTree(
                                          "100", 
                                          procfsRootDir.getAbsolutePath());
      pTree.getProcessTree();
      assertTrue("tree rooted at 100 should be over limit " +
                    "after first iteration.",
                  test.isProcessTreeOverLimit(pTree, "dummyId", limit));
      
      // the tree rooted at 200 is initially below limit.
      pTree = new ProcfsBasedProcessTree("200", 
                                          procfsRootDir.getAbsolutePath());
      pTree.getProcessTree();
      assertFalse("tree rooted at 200 shouldn't be over limit " +
                    "after one iteration.",
                  test.isProcessTreeOverLimit(pTree, "dummyId", limit));
      // second iteration - now the tree has been over limit twice,
      // hence it should be declared over limit.
      pTree.getProcessTree();
      assertTrue("tree rooted at 200 should be over limit after 2 iterations",
                  test.isProcessTreeOverLimit(pTree, "dummyId", limit));
      
      // the tree rooted at 600 is never over limit.
      pTree = new ProcfsBasedProcessTree("600", 
                                          procfsRootDir.getAbsolutePath());
      pTree.getProcessTree();
      assertFalse("tree rooted at 600 should never be over limit.",
                    test.isProcessTreeOverLimit(pTree, "dummyId", limit));
      
      // another iteration does not make any difference.
      pTree.getProcessTree();
      assertFalse("tree rooted at 600 should never be over limit.",
                    test.isProcessTreeOverLimit(pTree, "dummyId", limit));
    } finally {
      FileUtil.fullyDelete(procfsRootDir);
    }

View Full Code Here

      TaskMemoryManagerThread test = new TaskMemoryManagerThread(1000000L,
                                                                5000L);
      // create process trees
      // tree rooted at 100 is over limit immediately, as it is
      // twice over the mem limit.
      ProcfsBasedProcessTree pTree = new ProcfsBasedProcessTree(
                                          "100", true, 100L,
                                          procfsRootDir.getAbsolutePath());
      pTree.getProcessTree();
      assertTrue("tree rooted at 100 should be over limit " +
                    "after first iteration.",
                  test.isProcessTreeOverLimit(pTree, "dummyId", limit));
      
      // the tree rooted at 200 is initially below limit.
      pTree = new ProcfsBasedProcessTree("200", true, 100L,
                                          procfsRootDir.getAbsolutePath());
      pTree.getProcessTree();
      assertFalse("tree rooted at 200 shouldn't be over limit " +
                    "after one iteration.",
                  test.isProcessTreeOverLimit(pTree, "dummyId", limit));
      // second iteration - now the tree has been over limit twice,
      // hence it should be declared over limit.
      pTree.getProcessTree();
      assertTrue("tree rooted at 200 should be over limit after 2 iterations",
                  test.isProcessTreeOverLimit(pTree, "dummyId", limit));
      
      // the tree rooted at 600 is never over limit.
      pTree = new ProcfsBasedProcessTree("600", true, 100L,
                                          procfsRootDir.getAbsolutePath());
      pTree.getProcessTree();
      assertFalse("tree rooted at 600 should never be over limit.",
                    test.isProcessTreeOverLimit(pTree, "dummyId", limit));
      
      // another iteration does not make any difference.
      pTree.getProcessTree();
      assertFalse("tree rooted at 600 should never be over limit.",
                    test.isProcessTreeOverLimit(pTree, "dummyId", limit));
    } finally {
      FileUtil.fullyDelete(procfsRootDir);
    }

View Full Code Here

      TaskMemoryManagerThread test = new TaskMemoryManagerThread(1000000L,
                                                                5000L);
      // create process trees
      // tree rooted at 100 is over limit immediately, as it is
      // twice over the mem limit.
      ProcfsBasedProcessTree pTree = new ProcfsBasedProcessTree(
                                          "100", true, 100L,
                                          procfsRootDir.getAbsolutePath());
      pTree.getProcessTree();
      assertTrue("tree rooted at 100 should be over limit " +
                    "after first iteration.",
                  test.isProcessTreeOverLimit(pTree, "dummyId", limit));
      
      // the tree rooted at 200 is initially below limit.
      pTree = new ProcfsBasedProcessTree("200", true, 100L,
                                          procfsRootDir.getAbsolutePath());
      pTree.getProcessTree();
      assertFalse("tree rooted at 200 shouldn't be over limit " +
                    "after one iteration.",
                  test.isProcessTreeOverLimit(pTree, "dummyId", limit));
      // second iteration - now the tree has been over limit twice,
      // hence it should be declared over limit.
      pTree.getProcessTree();
      assertTrue("tree rooted at 200 should be over limit after 2 iterations",
                  test.isProcessTreeOverLimit(pTree, "dummyId", limit));
      
      // the tree rooted at 600 is never over limit.
      pTree = new ProcfsBasedProcessTree("600", true, 100L,
                                          procfsRootDir.getAbsolutePath());
      pTree.getProcessTree();
      assertFalse("tree rooted at 600 should never be over limit.",
                    test.isProcessTreeOverLimit(pTree, "dummyId", limit));
      
      // another iteration does not make any difference.
      pTree.getProcessTree();
      assertFalse("tree rooted at 600 should never be over limit.",
                    test.isProcessTreeOverLimit(pTree, "dummyId", limit));
    } finally {
      FileUtil.fullyDelete(procfsRootDir);
    }

View Full Code Here

                // create process tree object
                long sleeptimeBeforeSigkill = taskTracker.getJobConf().getLong(
                    JvmManager.SLEEPTIME_BEFORE_SIGKILL_KEY,
                    ProcessTree.DEFAULT_SLEEPTIME_BEFORE_SIGKILL);


                ProcfsBasedProcessTree pt = new ProcfsBasedProcessTree(
                    pId,ProcessTree.isSetsidAvailable, sleeptimeBeforeSigkill);
                LOG.debug("Tracking ProcessTree " + pId + " for the first time");


                ptInfo.setPid(pId);
                ptInfo.setProcessTree(pt);
              }
            }
            // End of initializing any uninitialized processTrees


            if (pId == null) {
              continue; // processTree cannot be tracked
            }


            LOG.debug("Constructing ProcessTree for : PID = " + pId + " TID = "
                + tid);
            ProcfsBasedProcessTree pTree = ptInfo.getProcessTree();
            pTree = pTree.getProcessTree(); // get the updated process-tree
            ptInfo.setProcessTree(pTree); // update ptInfo with process-tree of
            // updated state
            long currentMemUsage = pTree.getCumulativeVmem();
            long currentRssMemUsage = pTree.getCumulativeRssmem();
            // as processes begin with an age 1, we want to see if there 
            // are processes more than 1 iteration old.
            long curMemUsageOfAgedProcesses = pTree.getCumulativeVmem(1);
            long limit = ptInfo.getMemLimit();
            String user = taskTracker.getUserName(ptInfo.tid);
            if (user == null) {
              // If user is null the task is deleted from the TT memory
              continue;
            }
            // Log RSS and virtual memory usage of all tasks
            LOG.debug((String.format("Memory usage of ProcessTree %s : " +
                               "[USER,TID,RSS,VMEM,VLimit,TotalRSSLimit]"
                     + "=[%s,%s,%s,%s,%s,%s]",
                     pId, user, ptInfo.tid, currentRssMemUsage,
                     currentMemUsage, limit, maxRssMemoryAllowedForAllTasks)));


            if (doCheckVirtualMemory() &&
                isProcessTreeOverLimit(tid.toString(), currentMemUsage,
                                        curMemUsageOfAgedProcesses, limit)) {
              // Task (the root process) is still alive and overflowing memory.
               // Dump the process-tree and then clean it up.
              String msg =
                  "TaskTree [pid=" + pId + ",tipID=" + tid
                      + "] is running beyond memory-limits. Current usage : "
                      + currentMemUsage + "bytes. Limit : " + limit
                      + "bytes. Killing task. \nDump of the process-tree for "
                      + tid + " : \n" + pTree.getProcessTreeDump();
              LOG.warn(msg);
              taskTracker.cleanUpOverMemoryTask(tid, true, msg);


              LinuxSystemCall.killProcessGroup(Integer.parseInt(pId));
              it.remove();

View Full Code Here

0 1 2

TOP

Related Classes of org.apache.hadoop.util.ProcfsBasedProcessTree$SigKillThread

org.apache.hadoop.mapred.TaskMemoryManagerThread

org.apache.hadoop.mapred.TestTaskTrackerMemoryManager

org.apache.hadoop.util.Shell.ShellCommandExecutor

java.math.BigInteger

java.io.IOException

java.util.regex.Matcher

java.io.FileReader

java.io.BufferedReader

java.io.File

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.