Examples of org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics

Package org.apache.hadoop.yarn.server.resourcemanager.scheduler

Examples of org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics

org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics

    //Yes this is a hack, but there is no other way to insert
    //CSS in the correct spot
    html.style(".metrics {margin-bottom:5px}"); 
    
    ResourceScheduler rs = rm.getResourceScheduler();
    QueueMetrics metrics = rs.getRootQueueMetrics();
    ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();
    
    int appsSubmitted = metrics.getAppsSubmitted();
    int reservedGB = metrics.getReservedGB();
    int availableGB = metrics.getAvailableGB();
    int allocatedGB = metrics.getAllocatedGB();
    int containersAllocated = metrics.getAllocatedContainers();
    int totalGB = availableGB + reservedGB + allocatedGB;


    int totalNodes = clusterMetrics.getNumNMs();
    int lostNodes = clusterMetrics.getNumLostNMs();
    int unhealthyNodes = clusterMetrics.getUnhealthyNMs();
    int decommissionedNodes = clusterMetrics.getNumDecommisionedNMs();
    int rebootedNodes = clusterMetrics.getNumRebootedNMs();


    
    DIV<Hamlet> div = html.div().$class("metrics");
    
    div.table("#metricsoverview").
    thead().$class("ui-widget-header").
      tr().
        th().$class("ui-state-default")._("Apps Submitted")._().
        th().$class("ui-state-default")._("Containers Running")._().
        th().$class("ui-state-default")._("Memory Used")._().
        th().$class("ui-state-default")._("Memopry Total")._().
        th().$class("ui-state-default")._("Memory Reserved")._().
        th().$class("ui-state-default")._("Total Nodes")._().
        th().$class("ui-state-default")._("Decommissioned Nodes")._().
        th().$class("ui-state-default")._("Lost Nodes")._().
        th().$class("ui-state-default")._("Unhealthy Nodes")._().
        th().$class("ui-state-default")._("Rebooted Nodes")._().
      _().
    _().
    tbody().$class("ui-widget-content").
      tr().
        td(String.valueOf(appsSubmitted)).
        td(String.valueOf(containersAllocated)).
        td(StringUtils.byteDesc(allocatedGB * BYTES_IN_GB)).
        td(StringUtils.byteDesc(totalGB * BYTES_IN_GB)).
        td(StringUtils.byteDesc(reservedGB * BYTES_IN_GB)).
        td().a(url("nodes"),String.valueOf(totalNodes))._(). 
        td().a(url("nodes/decommissioned"),String.valueOf(decommissionedNodes))._(). 
        td().a(url("nodes/lost"),String.valueOf(lostNodes))._().
        td().a(url("nodes/unhealthy"),String.valueOf(unhealthyNodes))._().
        td().a(url("nodes/rebooted"),String.valueOf(rebootedNodes))._().
      _().
    _()._();
    
    String user = request().getRemoteUser();
    if (user != null) {
      QueueMetrics userMetrics = metrics.getUserMetrics(user);
      if(userMetrics != null) {
        int myAppsSubmitted = userMetrics.getAppsSubmitted();
        int myRunningContainers = userMetrics.getAllocatedContainers();
        int myPendingContainers = userMetrics.getPendingContainers();
        int myReservedContainers = userMetrics.getReservedContainers();
        int myReservedGB = userMetrics.getReservedGB();
        int myPendingGB = userMetrics.getPendingGB();
        int myAllocatedGB = userMetrics.getAllocatedGB();
        div.table("#usermetricsoverview").
        thead().$class("ui-widget-header").
          tr().
            th().$class("ui-state-default")._("Apps Submitted ("+user+")")._().
            th().$class("ui-state-default")._("Containers Running ("+user+")")._().

View Full Code Here

    assertEquals(1, a.getMetrics().getAppsSubmitted());
    assertEquals(0, a.getMetrics().getAppsPending());
    assertEquals(0, a.getMetrics().getAppsFailed());
    assertEquals(1, a.getMetrics().getAppsCompleted());


    QueueMetrics userMetrics = a.getMetrics().getUserMetrics(user_0);
    assertEquals(1, userMetrics.getAppsSubmitted());
  }

View Full Code Here

    assertEquals(1, a.getMetrics().getAppsSubmitted());
    assertEquals(0, a.getMetrics().getAppsPending());
    assertEquals(0, a.getMetrics().getAppsFailed());
    assertEquals(1, a.getMetrics().getAppsCompleted());


    QueueMetrics userMetrics = a.getMetrics().getUserMetrics(user_0);
    assertEquals(1, userMetrics.getAppsSubmitted());
  }

View Full Code Here

      int allocMB, int containersAlloc, int totalMB, int totalNodes,
      int lostNodes, int unhealthyNodes, int decommissionedNodes,
      int rebootedNodes, int activeNodes) throws JSONException, Exception {


    ResourceScheduler rs = rm.getResourceScheduler();
    QueueMetrics metrics = rs.getRootQueueMetrics();
    ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();


    long totalMBExpect = 
        metrics.getAvailableMB() + metrics.getAllocatedMB();


    assertEquals("appsSubmitted doesn't match", 
        metrics.getAppsSubmitted(), submittedApps);
    assertEquals("appsCompleted doesn't match", 
        metrics.getAppsCompleted(), completedApps);
    assertEquals("reservedMB doesn't match",
        metrics.getReservedMB(), reservedMB);
    assertEquals("availableMB doesn't match", 
        metrics.getAvailableMB(), availableMB);
    assertEquals("allocatedMB doesn't match", 
        metrics.getAllocatedMB(), allocMB);
    assertEquals("containersAllocated doesn't match", 0, containersAlloc);
    assertEquals("totalMB doesn't match", totalMBExpect, totalMB);
    assertEquals(
        "totalNodes doesn't match",
        clusterMetrics.getNumActiveNMs() + clusterMetrics.getNumLostNMs()

View Full Code Here

  
  private void verifyClusterMetrics(int activeNodes, int appsSubmitted,
      int appsPending, int containersPending, int availableMB,
      int activeApplications) throws Exception {
    int timeoutSecs = 0;
    QueueMetrics metrics = rm.getResourceScheduler().getRootQueueMetrics();
    ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();
    boolean isAllMetricAssertionDone = false;
    String message = null;
    while (timeoutSecs++ < 5) {
      try {
        // verify queue metrics
        assertMetric("appsSubmitted", appsSubmitted, metrics.getAppsSubmitted());
        assertMetric("appsPending", appsPending, metrics.getAppsPending());
        assertMetric("containersPending", containersPending,
            metrics.getPendingContainers());
        assertMetric("availableMB", availableMB, metrics.getAvailableMB());
        assertMetric("activeApplications", activeApplications,
            metrics.getActiveApps());
        // verify node metric
        assertMetric("activeNodes", activeNodes,
            clusterMetrics.getNumActiveNMs());
        isAllMetricAssertionDone = true;
        break;

View Full Code Here

    MockRM rm1 = new MockRM(conf, memStore);
    rm1.start();
    MockNM nm1 =
        new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
    nm1.registerNode();
    QueueMetrics qm1 = rm1.getResourceScheduler().getRootQueueMetrics();
    resetQueueMetrics(qm1);
    assertQueueMetrics(qm1, 0, 0, 0, 0);


    // create app that gets launched and does allocate before RM restart
    RMApp app1 = rm1.submitApp(200);
    assertQueueMetrics(qm1, 1, 1, 0, 0);
    nm1.nodeHeartbeat(true);
    RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
    ApplicationAttemptId attemptId1 = attempt1.getAppAttemptId();
    rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED);
    MockAM am1 = rm1.sendAMLaunched(attempt1.getAppAttemptId());
    am1.registerAppAttempt();
    am1.allocate("127.0.0.1" , 1000, 1, new ArrayList<ContainerId>()); 
    nm1.nodeHeartbeat(true);
    List<Container> conts = am1.allocate(new ArrayList<ResourceRequest>(),
        new ArrayList<ContainerId>()).getAllocatedContainers();
    while (conts.size() == 0) {
      nm1.nodeHeartbeat(true);
      conts.addAll(am1.allocate(new ArrayList<ResourceRequest>(),
          new ArrayList<ContainerId>()).getAllocatedContainers());
      Thread.sleep(500);
    }
    assertQueueMetrics(qm1, 1, 0, 1, 0);


    // PHASE 2: create new RM and start from old state
    // create new RM to represent restart and recover state
    MockRM rm2 = new MockRM(conf, memStore);
    rm2.start();
    nm1.setResourceTrackerService(rm2.getResourceTrackerService());
    QueueMetrics qm2 = rm2.getResourceScheduler().getRootQueueMetrics();
    resetQueueMetrics(qm2);
    assertQueueMetrics(qm2, 0, 0, 0, 0);
    // recover app
    RMApp loadedApp1 = rm2.getRMContext().getRMApps().get(app1.getApplicationId());
    am1.setAMRMProtocol(rm2.getApplicationMasterService());

View Full Code Here

        return dispatcher;
      }
    };


    // test metrics
    QueueMetrics metrics = rm.getResourceScheduler().getRootQueueMetrics();
    int appsKilled = metrics.getAppsKilled();
    int appsSubmitted = metrics.getAppsSubmitted();


    rm.start();
    
    MockNM nm1 =
        new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
    nm1.registerNode();


    // a failed app
    RMApp application = rm.submitApp(200);
    MockAM am = MockRM.launchAM(application, rm, nm1);
    am.waitForState(RMAppAttemptState.LAUNCHED);
    nm1.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.RUNNING);
    rm.waitForState(application.getApplicationId(), RMAppState.ACCEPTED);


    // Now kill the application before new attempt is launched, the app report
    // returns the invalid AM host and port.
    KillApplicationRequest request =
        KillApplicationRequest.newInstance(application.getApplicationId());
    rm.getClientRMService().forceKillApplication(request);


    // Specific test for YARN-1689 follows
    // Now let's say a race causes AM to register now. This should not crash RM.
    am.registerAppAttempt(false);


    // We explicitly intercepted the kill-event to RMAppAttempt, so app should
    // still be in KILLING state.
    rm.waitForState(application.getApplicationId(), RMAppState.KILLING);
    // AM should now be in running
    rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.RUNNING);


    // Simulate that appAttempt is killed.
    rm.getRMContext().getDispatcher().getEventHandler().handle(
        new RMAppEvent(application.getApplicationId(),
          RMAppEventType.ATTEMPT_KILLED));
    rm.waitForState(application.getApplicationId(), RMAppState.KILLED);


    // test metrics
    metrics = rm.getResourceScheduler().getRootQueueMetrics();
    Assert.assertEquals(appsKilled + 1, metrics.getAppsKilled());
    Assert.assertEquals(appsSubmitted + 1, metrics.getAppsSubmitted());
  }

View Full Code Here

    nm1.nodeHeartbeat(true);
    nm2.nodeHeartbeat(false);
    dispatcher.await();
    checkUnealthyNMCount(rm, nm2, true, 1);
    final int expectedNMs = ClusterMetrics.getMetrics().getNumActiveNMs();
    QueueMetrics metrics = rm.getResourceScheduler().getRootQueueMetrics();
    // TODO Metrics incorrect in case of the FifoScheduler
    Assert.assertEquals(5120, metrics.getAvailableMB());


    // reconnect of healthy node
    nm1 = rm.registerNode("host1:1234", 5120);
    NodeHeartbeatResponse response = nm1.nodeHeartbeat(true);
    Assert.assertTrue(NodeAction.NORMAL.equals(response.getNodeAction()));
    dispatcher.await();
    Assert.assertEquals(expectedNMs, ClusterMetrics.getMetrics().getNumActiveNMs());
    checkUnealthyNMCount(rm, nm2, true, 1);


    // reconnect of unhealthy node
    nm2 = rm.registerNode("host2:5678", 5120);
    response = nm2.nodeHeartbeat(false);
    Assert.assertTrue(NodeAction.NORMAL.equals(response.getNodeAction()));
    dispatcher.await();
    Assert.assertEquals(expectedNMs, ClusterMetrics.getMetrics().getNumActiveNMs());
    checkUnealthyNMCount(rm, nm2, true, 1);
    
    // unhealthy node changed back to healthy
    nm2 = rm.registerNode("host2:5678", 5120);
    dispatcher.await();
    response = nm2.nodeHeartbeat(true);
    response = nm2.nodeHeartbeat(true);
    dispatcher.await();
    Assert.assertEquals(5120 + 5120, metrics.getAvailableMB());


    // reconnect of node with changed capability
    nm1 = rm.registerNode("host2:5678", 10240);
    dispatcher.await();
    response = nm2.nodeHeartbeat(true);
    dispatcher.await();
    Assert.assertTrue(NodeAction.NORMAL.equals(response.getNodeAction()));
    Assert.assertEquals(5120 + 10240, metrics.getAvailableMB());
  }

View Full Code Here

    RMContext rmContext =
        new RMContextImpl(null, dispatcher, null, null, null, null, null, null);


    FifoScheduler schedular = new FifoScheduler();
    schedular.reinitialize(new Configuration(), rmContext);
    QueueMetrics metrics = schedular.getRootQueueMetrics();
    int beforeAppsSubmitted = metrics.getAppsSubmitted();


    ApplicationId appId = BuilderUtils.newApplicationId(200, 1);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
        appId, 1);


    SchedulerEvent event = new AppAddedSchedulerEvent(appAttemptId, "queue",
        "user");
    schedular.handle(event);


    appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 2);


    event = new AppAddedSchedulerEvent(appAttemptId, "queue", "user");
    schedular.handle(event);


    int afterAppsSubmitted = metrics.getAppsSubmitted();
    Assert.assertEquals(1, afterAppsSubmitted - beforeAppsSubmitted);
  }

View Full Code Here

    appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 2);


    event = new AppAddedSchedulerEvent(appAttemptId, "queue", "user");
    schedular.handle(event);


    QueueMetrics metrics = schedular.getRootQueueMetrics();
    Assert.assertEquals(1, metrics.getAppsSubmitted());
  }

View Full Code Here

0 1 2 3 4 5 6 7

TOP

Related Classes of org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics

org.apache.hadoop.metrics2.lib.MetricsRegistry

org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestLeafQueue

org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueueMetrics

org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.TestFairScheduler

org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.TestFifoScheduler

org.apache.hadoop.yarn.server.resourcemanager.TestResourceTrackerService

org.apache.hadoop.yarn.server.resourcemanager.TestRM

org.apache.hadoop.yarn.server.resourcemanager.TestRMHA

org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart

org.apache.hadoop.yarn.server.resourcemanager.TestWorkPreservingRMRestart

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.