Examples of org.apache.hadoop.yarn.util.ProcfsBasedProcessTree

org.apache.hadoop.yarn.util.ProcfsBasedProcessTree
A Proc file-system based ProcessTree. Works only on Linux.

      ContainersMonitorImpl test = new ContainersMonitorImpl(null, null, null);


      // create process trees
      // tree rooted at 100 is over limit immediately, as it is
      // twice over the mem limit.
      ProcfsBasedProcessTree pTree = new ProcfsBasedProcessTree(
                                          "100",
                                          procfsRootDir.getAbsolutePath());
      pTree.updateProcessTree();
      assertTrue("tree rooted at 100 should be over limit " +
                    "after first iteration.",
                  test.isProcessTreeOverLimit(pTree, "dummyId", limit));


      // the tree rooted at 200 is initially below limit.
      pTree = new ProcfsBasedProcessTree("200",
                                          procfsRootDir.getAbsolutePath());
      pTree.updateProcessTree();
      assertFalse("tree rooted at 200 shouldn't be over limit " +
                    "after one iteration.",
                  test.isProcessTreeOverLimit(pTree, "dummyId", limit));
      // second iteration - now the tree has been over limit twice,
      // hence it should be declared over limit.
      pTree.updateProcessTree();
      assertTrue(
          "tree rooted at 200 should be over limit after 2 iterations",
                  test.isProcessTreeOverLimit(pTree, "dummyId", limit));


      // the tree rooted at 600 is never over limit.
      pTree = new ProcfsBasedProcessTree("600",
                                            procfsRootDir.getAbsolutePath());
      pTree.updateProcessTree();
      assertFalse("tree rooted at 600 should never be over limit.",
                    test.isProcessTreeOverLimit(pTree, "dummyId", limit));


      // another iteration does not make any difference.
      pTree.updateProcessTree();
      assertFalse("tree rooted at 600 should never be over limit.",
                    test.isProcessTreeOverLimit(pTree, "dummyId", limit));
    } finally {
      FileUtil.fullyDelete(procfsRootDir);
    }

View Full Code Here


    Thread t = new RogueTaskThread();
    t.start();
    String pid = getRogueTaskPID();
    LOG.info("Root process pid: " + pid);
    ProcfsBasedProcessTree p = createProcessTree(pid);
    p.getProcessTree(); // initialize
    LOG.info("ProcessTree: " + p.toString());


    File leaf = new File(lowestDescendant);
    //wait till lowest descendant process of Rougue Task starts execution
    while (!leaf.exists()) {
      try {
        Thread.sleep(500);
      } catch (InterruptedException ie) {
        break;
      }
    }


    p.getProcessTree(); // reconstruct
    LOG.info("ProcessTree: " + p.toString());


    // Get the process-tree dump
    String processTreeDump = p.getProcessTreeDump();


    // destroy the process and all its subprocesses
    destroyProcessTree(pid);


    if (isSetsidAvailable()) { // whole processtree should be gone
      Assert.assertFalse("Proceesses in process group live",
          isAnyProcessInTreeAlive(p));
    } else {// process should be gone
      Assert.assertFalse("ProcessTree must have been gone", isAlive(pid));
    }


    LOG.info("Process-tree dump follows: \n" + processTreeDump);
    Assert.assertTrue("Process-tree dump doesn't start with a proper header",
        processTreeDump.startsWith("\t|- PID PPID PGRPID SESSID CMD_NAME " +
        "USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) " +
        "RSSMEM_USAGE(PAGES) FULL_CMD_LINE\n"));
    for (int i = N; i >= 0; i--) {
      String cmdLineDump = "\\|- [0-9]+ [0-9]+ [0-9]+ [0-9]+ \\(sh\\)" +
          " [0-9]+ [0-9]+ [0-9]+ [0-9]+ sh " + shellScript + " " + i;
      Pattern pat = Pattern.compile(cmdLineDump);
      Matcher mat = pat.matcher(processTreeDump);
      Assert.assertTrue("Process-tree dump doesn't contain the cmdLineDump of " + i
          + "th process!", mat.find());
    }


    // Not able to join thread sometimes when forking with large N.
    try {
      t.join(2000);
      LOG.info("RogueTaskThread successfully joined.");
    } catch (InterruptedException ie) {
      LOG.info("Interrupted while joining RogueTaskThread.");
    }


    // ProcessTree is gone now. Any further calls should be sane.
    p.getProcessTree();
    Assert.assertFalse("ProcessTree must have been gone", isAlive(pid));
    Assert.assertTrue("Cumulative vmem for the gone-process is "
        + p.getCumulativeVmem() + " . It should be zero.", p
        .getCumulativeVmem() == 0);
    Assert.assertTrue(p.toString().equals("[ ]"));
  }

View Full Code Here

        .getCumulativeVmem() == 0);
    Assert.assertTrue(p.toString().equals("[ ]"));
  }


  protected ProcfsBasedProcessTree createProcessTree(String pid) {
    return new ProcfsBasedProcessTree(pid);
  }

View Full Code Here

  protected ProcfsBasedProcessTree createProcessTree(String pid) {
    return new ProcfsBasedProcessTree(pid);
  }


  protected ProcfsBasedProcessTree createProcessTree(String pid, String procfsRootDir) {
    return new ProcfsBasedProcessTree(pid, procfsRootDir);
  }

View Full Code Here

          {"400", "proc4", "1", "400", "400", "400000", "400", "4000", "800"});


      writeStatFiles(procfsRootDir, pids, procInfos);


      // crank up the process tree class.
      ProcfsBasedProcessTree processTree =
          createProcessTree("100", procfsRootDir.getAbsolutePath());
      // build the process tree.
      processTree.getProcessTree();


      // verify cumulative memory
      Assert.assertEquals("Cumulative virtual memory does not match", 600000L,
                   processTree.getCumulativeVmem());


      // verify rss memory
      long cumuRssMem = ProcfsBasedProcessTree.PAGE_SIZE > 0 ?
                        600L * ProcfsBasedProcessTree.PAGE_SIZE : 0L;
      Assert.assertEquals("Cumulative rss memory does not match",
                   cumuRssMem, processTree.getCumulativeRssmem());


      // verify cumulative cpu time
      long cumuCpuTime = ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS > 0 ?
             7200L * ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS : 0L;
      Assert.assertEquals("Cumulative cpu time does not match",
                   cumuCpuTime, processTree.getCumulativeCpuTime());


      // test the cpu time again to see if it cumulates
      procInfos[0] = new ProcessStatInfo(new String[]
          {"100", "proc1", "1", "100", "100", "100000", "100", "2000", "300"});
      procInfos[1] = new ProcessStatInfo(new String[]
          {"200", "proc2", "100", "100", "100", "200000", "200", "3000", "500"});
      writeStatFiles(procfsRootDir, pids, procInfos);


      // build the process tree.
      processTree.getProcessTree();


      // verify cumulative cpu time again
      cumuCpuTime = ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS > 0 ?
             9400L * ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS : 0L;
      Assert.assertEquals("Cumulative cpu time does not match",
                   cumuCpuTime, processTree.getCumulativeCpuTime());
    } finally {
      FileUtil.fullyDelete(procfsRootDir);
    }
  }

View Full Code Here

                        {"400", "proc4", "100", "100", "100", "400000", "400"});


      writeStatFiles(procfsRootDir, pids, procInfos);


      // crank up the process tree class.
      ProcfsBasedProcessTree processTree =
          createProcessTree("100", procfsRootDir.getAbsolutePath());
      // build the process tree.
      processTree.getProcessTree();


      // verify cumulative memory
      Assert.assertEquals("Cumulative memory does not match",
                   700000L, processTree.getCumulativeVmem());


      // write one more process as child of 100.
      String[] newPids = { "500" };
      setupPidDirs(procfsRootDir, newPids);


      ProcessStatInfo[] newProcInfos = new ProcessStatInfo[1];
      newProcInfos[0] = new ProcessStatInfo(new String[]
                      {"500", "proc5", "100", "100", "100", "500000", "500"});
      writeStatFiles(procfsRootDir, newPids, newProcInfos);


      // check memory includes the new process.
      processTree.getProcessTree();
      Assert.assertEquals("Cumulative vmem does not include new process",
                   1200000L, processTree.getCumulativeVmem());
      long cumuRssMem = ProcfsBasedProcessTree.PAGE_SIZE > 0 ?
                        1200L * ProcfsBasedProcessTree.PAGE_SIZE : 0L;
      Assert.assertEquals("Cumulative rssmem does not include new process",
                   cumuRssMem, processTree.getCumulativeRssmem());


      // however processes older than 1 iteration will retain the older value
      Assert.assertEquals("Cumulative vmem shouldn't have included new process",
                   700000L, processTree.getCumulativeVmem(1));
      cumuRssMem = ProcfsBasedProcessTree.PAGE_SIZE > 0 ?
                   700L * ProcfsBasedProcessTree.PAGE_SIZE : 0L;
      Assert.assertEquals("Cumulative rssmem shouldn't have included new process",
                   cumuRssMem, processTree.getCumulativeRssmem(1));


      // one more process
      newPids = new String[]{ "600" };
      setupPidDirs(procfsRootDir, newPids);


      newProcInfos = new ProcessStatInfo[1];
      newProcInfos[0] = new ProcessStatInfo(new String[]
                      {"600", "proc6", "100", "100", "100", "600000", "600"});
      writeStatFiles(procfsRootDir, newPids, newProcInfos);


      // refresh process tree
      processTree.getProcessTree();


      // processes older than 2 iterations should be same as before.
      Assert.assertEquals("Cumulative vmem shouldn't have included new processes",
                   700000L, processTree.getCumulativeVmem(2));
      cumuRssMem = ProcfsBasedProcessTree.PAGE_SIZE > 0 ?
                   700L * ProcfsBasedProcessTree.PAGE_SIZE : 0L;
      Assert.assertEquals("Cumulative rssmem shouldn't have included new processes",
                   cumuRssMem, processTree.getCumulativeRssmem(2));


      // processes older than 1 iteration should not include new process,
      // but include process 500
      Assert.assertEquals("Cumulative vmem shouldn't have included new processes",
                   1200000L, processTree.getCumulativeVmem(1));
      cumuRssMem = ProcfsBasedProcessTree.PAGE_SIZE > 0 ?
                   1200L * ProcfsBasedProcessTree.PAGE_SIZE : 0L;
      Assert.assertEquals("Cumulative rssmem shouldn't have included new processes",
                   cumuRssMem, processTree.getCumulativeRssmem(1));


      // no processes older than 3 iterations, this should be 0
      Assert.assertEquals("Getting non-zero vmem for processes older than 3 iterations",
                    0L, processTree.getCumulativeVmem(3));
      Assert.assertEquals("Getting non-zero rssmem for processes older than 3 iterations",
                    0L, processTree.getCumulativeRssmem(3));
    } finally {
      FileUtil.fullyDelete(procfsRootDir);
    }
  }

View Full Code Here

      cmdLines[5] = "proc6 arg11 arg12";


      writeStatFiles(procfsRootDir, pids, procInfos);
      writeCmdLineFiles(procfsRootDir, pids, cmdLines);


      ProcfsBasedProcessTree processTree = createProcessTree(
          "100", procfsRootDir.getAbsolutePath());
      // build the process tree.
      processTree.getProcessTree();


      // Get the process-tree dump
      String processTreeDump = processTree.getProcessTreeDump();


      LOG.info("Process-tree dump follows: \n" + processTreeDump);
      Assert.assertTrue("Process-tree dump doesn't start with a proper header",
          processTreeDump.startsWith("\t|- PID PPID PGRPID SESSID CMD_NAME " +
          "USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) " +

View Full Code Here

      ContainersMonitorImpl test = new ContainersMonitorImpl(null, null, null);


      // create process trees
      // tree rooted at 100 is over limit immediately, as it is
      // twice over the mem limit.
      ProcfsBasedProcessTree pTree = new ProcfsBasedProcessTree(
                                          "100",
                                          procfsRootDir.getAbsolutePath());
      pTree.updateProcessTree();
      assertTrue("tree rooted at 100 should be over limit " +
                    "after first iteration.",
                  test.isProcessTreeOverLimit(pTree, "dummyId", limit));


      // the tree rooted at 200 is initially below limit.
      pTree = new ProcfsBasedProcessTree("200",
                                          procfsRootDir.getAbsolutePath());
      pTree.updateProcessTree();
      assertFalse("tree rooted at 200 shouldn't be over limit " +
                    "after one iteration.",
                  test.isProcessTreeOverLimit(pTree, "dummyId", limit));
      // second iteration - now the tree has been over limit twice,
      // hence it should be declared over limit.
      pTree.updateProcessTree();
      assertTrue(
          "tree rooted at 200 should be over limit after 2 iterations",
                  test.isProcessTreeOverLimit(pTree, "dummyId", limit));


      // the tree rooted at 600 is never over limit.
      pTree = new ProcfsBasedProcessTree("600",
                                            procfsRootDir.getAbsolutePath());
      pTree.updateProcessTree();
      assertFalse("tree rooted at 600 should never be over limit.",
                    test.isProcessTreeOverLimit(pTree, "dummyId", limit));


      // another iteration does not make any difference.
      pTree.updateProcessTree();
      assertFalse("tree rooted at 600 should never be over limit.",
                    test.isProcessTreeOverLimit(pTree, "dummyId", limit));
    } finally {
      FileUtil.fullyDelete(procfsRootDir);
    }

View Full Code Here

                // pId will be null, either if the container is not spawned yet
                // or if the container's pid is removed from ContainerExecutor
                LOG.debug("Tracking ProcessTree " + pId
                    + " for the first time");


                ProcfsBasedProcessTree pt =
                    new ProcfsBasedProcessTree(pId,
                        ContainerExecutor.isSetsidAvailable);
                ptInfo.setPid(pId);
                ptInfo.setProcessTree(pt);
              }
            }
            // End of initializing any uninitialized processTrees


            if (pId == null) {
              continue; // processTree cannot be tracked
            }


            LOG.debug("Constructing ProcessTree for : PID = " + pId
                + " ContainerId = " + containerId);
            ProcfsBasedProcessTree pTree = ptInfo.getProcessTree();
            pTree = pTree.getProcessTree(); // get the updated process-tree
            ptInfo.setProcessTree(pTree); // update ptInfo with proces-tree of
                                          // updated state
            long currentVmemUsage = pTree.getCumulativeVmem();
            long currentPmemUsage = pTree.getCumulativeRssmem();
            // as processes begin with an age 1, we want to see if there
            // are processes more than 1 iteration old.
            long curMemUsageOfAgedProcesses = pTree.getCumulativeVmem(1);
            long curRssMemUsageOfAgedProcesses = pTree.getCumulativeRssmem(1);
            long vmemLimit = ptInfo.getVmemLimit();
            long pmemLimit = ptInfo.getPmemLimit();
            LOG.info(String.format(
                "Memory usage of ProcessTree %s for container-id %s: ",
                     pId, containerId.toString()) +
                formatUsageString(currentVmemUsage, vmemLimit, currentPmemUsage, pmemLimit));


            boolean isMemoryOverLimit = false;
            String msg = "";
            if (isVirtualMemoryCheckEnabled()
                && isProcessTreeOverLimit(containerId.toString(),
                    currentVmemUsage, curMemUsageOfAgedProcesses, vmemLimit)) {
              // Container (the root process) is still alive and overflowing
              // memory.
              // Dump the process-tree and then clean it up.
              msg = formatErrorMessage("virtual",
                  currentVmemUsage, vmemLimit,
                  currentPmemUsage, pmemLimit,
                  pId, containerId, pTree);
              isMemoryOverLimit = true;
            } else if (isPhysicalMemoryCheckEnabled()
                && isProcessTreeOverLimit(containerId.toString(),
                    currentPmemUsage, curRssMemUsageOfAgedProcesses,
                    pmemLimit)) {
              // Container (the root process) is still alive and overflowing
              // memory.
              // Dump the process-tree and then clean it up.
              msg = formatErrorMessage("physical",
                  currentVmemUsage, vmemLimit,
                  currentPmemUsage, pmemLimit,
                  pId, containerId, pTree);
              isMemoryOverLimit = true;
            }


            if (isMemoryOverLimit) {
              // Virtual or physical memory over limit. Fail the container and
              // remove
              // the corresponding process tree
              LOG.warn(msg);
              // warn if not a leader
              if (!pTree.checkPidPgrpidForMatch()) {
                LOG.error("Killed container process with PID " + pId
                    + " but it is not a process group leader.");
              }
              // kill the container
              eventDispatcher.getEventHandler().handle(

View Full Code Here


    Thread t = new RogueTaskThread();
    t.start();
    String pid = getRogueTaskPID();
    LOG.info("Root process pid: " + pid);
    ProcfsBasedProcessTree p = createProcessTree(pid);
    p = p.getProcessTree(); // initialize
    LOG.info("ProcessTree: " + p.toString());


    File leaf = new File(lowestDescendant);
    //wait till lowest descendant process of Rougue Task starts execution
    while (!leaf.exists()) {
      try {
        Thread.sleep(500);
      } catch (InterruptedException ie) {
        break;
      }
    }


    p = p.getProcessTree(); // reconstruct
    LOG.info("ProcessTree: " + p.toString());


    // Get the process-tree dump
    String processTreeDump = p.getProcessTreeDump();


    // destroy the process and all its subprocesses
    destroyProcessTree(pid);


    if (isSetsidAvailable()) { // whole processtree should be gone
      Assert.assertFalse("Proceesses in process group live",
          isAnyProcessInTreeAlive(p));
    } else {// process should be gone
      Assert.assertFalse("ProcessTree must have been gone", isAlive(pid));
    }


    LOG.info("Process-tree dump follows: \n" + processTreeDump);
    Assert.assertTrue("Process-tree dump doesn't start with a proper header",
        processTreeDump.startsWith("\t|- PID PPID PGRPID SESSID CMD_NAME " +
        "USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) " +
        "RSSMEM_USAGE(PAGES) FULL_CMD_LINE\n"));
    for (int i = N; i >= 0; i--) {
      String cmdLineDump = "\\|- [0-9]+ [0-9]+ [0-9]+ [0-9]+ \\(sh\\)" +
          " [0-9]+ [0-9]+ [0-9]+ [0-9]+ sh " + shellScript + " " + i;
      Pattern pat = Pattern.compile(cmdLineDump);
      Matcher mat = pat.matcher(processTreeDump);
      Assert.assertTrue("Process-tree dump doesn't contain the cmdLineDump of " + i
          + "th process!", mat.find());
    }


    // Not able to join thread sometimes when forking with large N.
    try {
      t.join(2000);
      LOG.info("RogueTaskThread successfully joined.");
    } catch (InterruptedException ie) {
      LOG.info("Interrupted while joining RogueTaskThread.");
    }


    // ProcessTree is gone now. Any further calls should be sane.
    p = p.getProcessTree();
    Assert.assertFalse("ProcessTree must have been gone", isAlive(pid));
    Assert.assertTrue("Cumulative vmem for the gone-process is "
        + p.getCumulativeVmem() + " . It should be zero.", p
        .getCumulativeVmem() == 0);
    Assert.assertTrue(p.toString().equals("[ ]"));
  }

View Full Code Here

0 1 2 3 4

TOP

Related Classes of org.apache.hadoop.yarn.util.ProcfsBasedProcessTree

org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl$MonitoringThread

org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.TestContainersMonitor

org.apache.hadoop.yarn.util.TestProcfsBasedProcessTree

java.io.FileInputStream

java.math.BigInteger

java.io.FileReader

java.io.BufferedReader

java.util.regex.Matcher

java.io.File

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.