Package org.apache.hadoop.yarn.server.resourcemanager.scheduler

Examples of org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics

343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
    nm1.nodeHeartbeat(true);
    nm2.nodeHeartbeat(false);
    dispatcher.await();
    checkUnealthyNMCount(rm, nm2, true, 1);
    final int expectedNMs = ClusterMetrics.getMetrics().getNumActiveNMs();
    QueueMetrics metrics = rm.getResourceScheduler().getRootQueueMetrics();
    // TODO Metrics incorrect in case of the FifoScheduler
    Assert.assertEquals(5120, metrics.getAvailableMB());

    // reconnect of healthy node
    nm1 = rm.registerNode("host1:1234", 5120);
    HeartbeatResponse response = nm1.nodeHeartbeat(true);
    Assert.assertTrue(NodeAction.NORMAL.equals(response.getNodeAction()));
    dispatcher.await();
    Assert.assertEquals(expectedNMs, ClusterMetrics.getMetrics().getNumActiveNMs());
    checkUnealthyNMCount(rm, nm2, true, 1);

    // reconnect of unhealthy node
    nm2 = rm.registerNode("host2:5678", 5120);
    response = nm2.nodeHeartbeat(false);
    Assert.assertTrue(NodeAction.NORMAL.equals(response.getNodeAction()));
    dispatcher.await();
    Assert.assertEquals(expectedNMs, ClusterMetrics.getMetrics().getNumActiveNMs());
    checkUnealthyNMCount(rm, nm2, true, 1);

    // reconnect of node with changed capability
    nm1 = rm.registerNode("host2:5678", 10240);
    dispatcher.await();
    response = nm2.nodeHeartbeat(true);
    dispatcher.await();
    Assert.assertTrue(NodeAction.NORMAL.equals(response.getNodeAction()));
    Assert.assertEquals(5120 + 10240, metrics.getAvailableMB());
  }
View Full Code Here
105
106
107
108
109
110
111
112
    appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 2);

    event = new AppAddedSchedulerEvent(appAttemptId, "queue", "user");
    schedular.handle(event);

    QueueMetrics metrics = schedular.getRootQueueMetrics();
    Assert.assertEquals(1, metrics.getAppsSubmitted());
  }
View Full Code Here
359
360
361
362
363
364
365
366
    assertEquals(1, a.getMetrics().getAppsSubmitted());
    assertEquals(0, a.getMetrics().getAppsPending());
    assertEquals(0, a.getMetrics().getAppsFailed());
    assertEquals(1, a.getMetrics().getAppsCompleted());

    QueueMetrics userMetrics = a.getMetrics().getUserMetrics(user_0);
    assertEquals(1, userMetrics.getAppsSubmitted());
  }
View Full Code Here
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
      int allocMB, int containersAlloc, int totalMB, int totalNodes,
      int lostNodes, int unhealthyNodes, int decommissionedNodes,
      int rebootedNodes, int activeNodes) throws JSONException, Exception {

    ResourceScheduler rs = rm.getResourceScheduler();
    QueueMetrics metrics = rs.getRootQueueMetrics();
    ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();

    long totalMBExpect =
        metrics.getReservedMB()+ metrics.getAvailableMB()
        + metrics.getAllocatedMB();

    assertEquals("appsSubmitted doesn't match",
        metrics.getAppsSubmitted(), submittedApps);
    assertEquals("appsCompleted doesn't match",
        metrics.getAppsCompleted(), completedApps);
    assertEquals("reservedMB doesn't match",
        metrics.getReservedMB(), reservedMB);
    assertEquals("availableMB doesn't match",
        metrics.getAvailableMB(), availableMB);
    assertEquals("allocatedMB doesn't match",
        metrics.getAllocatedMB(), allocMB);
    assertEquals("containersAllocated doesn't match", 0, containersAlloc);
    assertEquals("totalMB doesn't match", totalMBExpect, totalMB);
    assertEquals(
        "totalNodes doesn't match",
        clusterMetrics.getNumActiveNMs() + clusterMetrics.getNumLostNMs()
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
 
  public synchronized
  static FSQueueMetrics forQueue(String queueName, Queue parent,
      boolean enableUserMetrics, Configuration conf) {
    MetricsSystem ms = DefaultMetricsSystem.instance();
    QueueMetrics metrics = queueMetrics.get(queueName);
    if (metrics == null) {
      metrics = new FSQueueMetrics(ms, queueName, parent, enableUserMetrics, conf)
          .tag(QUEUE_INFO, queueName);
     
      // Register with the MetricsSystems
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
  } // JAXB needs this

  public UserMetricsInfo(final ResourceManager rm, final RMContext rmContext,
      final String user) {
    ResourceScheduler rs = rm.getResourceScheduler();
    QueueMetrics metrics = rs.getRootQueueMetrics();
    QueueMetrics userMetrics = metrics.getUserMetrics(user);
    this.userMetricsAvailable = false;

    if (userMetrics != null) {
      this.userMetricsAvailable = true;
     
      this.appsSubmitted = userMetrics.getAppsSubmitted();
      this.appsCompleted = metrics.getAppsCompleted();
      this.appsPending = metrics.getAppsPending();
      this.appsRunning = metrics.getAppsRunning();
      this.appsFailed = metrics.getAppsFailed();
      this.appsKilled = metrics.getAppsKilled();

      this.runningContainers = userMetrics.getAllocatedContainers();
      this.pendingContainers = userMetrics.getPendingContainers();
      this.reservedContainers = userMetrics.getReservedContainers();
     
      this.reservedMB = userMetrics.getReservedMB();
      this.pendingMB = userMetrics.getPendingMB();
      this.allocatedMB = userMetrics.getAllocatedMB();
    }
  }
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
  public ClusterMetricsInfo() {
  } // JAXB needs this

  public ClusterMetricsInfo(final ResourceManager rm, final RMContext rmContext) {
    ResourceScheduler rs = rm.getResourceScheduler();
    QueueMetrics metrics = rs.getRootQueueMetrics();
    ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();

    this.appsSubmitted = metrics.getAppsSubmitted();
    this.appsCompleted = metrics.getAppsCompleted();
    this.appsPending = metrics.getAppsPending();
    this.appsRunning = metrics.getAppsRunning();
    this.appsFailed = metrics.getAppsFailed();
    this.appsKilled = metrics.getAppsKilled();
   
    this.reservedMB = metrics.getReservedMB();
    this.availableMB = metrics.getAvailableMB();
    this.allocatedMB = metrics.getAllocatedMB();
   
    this.containersAllocated = metrics.getAllocatedContainers();
    this.containersPending = metrics.getPendingContainers();
    this.containersReserved = metrics.getReservedContainers();
   
    this.totalMB = availableMB + allocatedMB;
    this.activeNodes = clusterMetrics.getNumActiveNMs();
    this.lostNodes = clusterMetrics.getNumLostNMs();
    this.unhealthyNodes = clusterMetrics.getUnhealthyNMs();
View Full Code Here
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
    RMContext rmContext = new RMContextImpl(dispatcher, null,
        null, null, null, null, null, null, null, writer);

    FifoScheduler schedular = new FifoScheduler();
    schedular.reinitialize(new Configuration(), rmContext);
    QueueMetrics metrics = schedular.getRootQueueMetrics();
    int beforeAppsSubmitted = metrics.getAppsSubmitted();

    ApplicationId appId = BuilderUtils.newApplicationId(200, 1);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
        appId, 1);

    SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "queue", "user");
    schedular.handle(appEvent);
    SchedulerEvent attemptEvent =
        new AppAttemptAddedSchedulerEvent(appAttemptId, false);
    schedular.handle(attemptEvent);

    appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 2);
    SchedulerEvent attemptEvent2 =
        new AppAttemptAddedSchedulerEvent(appAttemptId, false);
    schedular.handle(attemptEvent2);

    int afterAppsSubmitted = metrics.getAppsSubmitted();
    Assert.assertEquals(1, afterAppsSubmitted - beforeAppsSubmitted);
  }
View Full Code Here
550
551
552
553
554
555
556
557
558
559
560
561
562
563
      getResourceUsage().getMemory());
    assertEquals(2, scheduler.getQueueManager().getQueue("queue1").
      getResourceUsage().getVirtualCores());

    // verify metrics
    QueueMetrics queue1Metrics = scheduler.getQueueManager().getQueue("queue1")
        .getMetrics();
    assertEquals(1024, queue1Metrics.getAllocatedMB());
    assertEquals(2, queue1Metrics.getAllocatedVirtualCores());
    assertEquals(1024, scheduler.getRootQueueMetrics().getAllocatedMB());
    assertEquals(2, scheduler.getRootQueueMetrics().getAllocatedVirtualCores());
    assertEquals(512, scheduler.getRootQueueMetrics().getAvailableMB());
    assertEquals(4, scheduler.getRootQueueMetrics().getAvailableVirtualCores());
  }
View Full Code Here
349
350
351
352
353
354
355
356
    assertEquals(1, a.getMetrics().getAppsSubmitted());
    assertEquals(0, a.getMetrics().getAppsPending());
    assertEquals(0, a.getMetrics().getAppsFailed());
    assertEquals(1, a.getMetrics().getAppsCompleted());

    QueueMetrics userMetrics = a.getMetrics().getUserMetrics(user_0);
    assertEquals(1, userMetrics.getAppsSubmitted());
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics

  • org.apache.hadoop.metrics2.lib.MetricsRegistry
  • org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestLeafQueue
  • org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueueMetrics
  • org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.TestFairScheduler
  • org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.TestFifoScheduler
  • org.apache.hadoop.yarn.server.resourcemanager.TestResourceTrackerService
  • org.apache.hadoop.yarn.server.resourcemanager.TestRM
  • org.apache.hadoop.yarn.server.resourcemanager.TestRMHA
  • org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart
  • org.apache.hadoop.yarn.server.resourcemanager.TestWorkPreservingRMRestart
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.