Package org.apache.hadoop.yarn.server.resourcemanager.scheduler

Examples of org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics

144
145
146
147
148
149
150
151
    appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 2);

    event = new AppAddedSchedulerEvent(appAttemptId, "queue", "user");
    schedular.handle(event);

    QueueMetrics metrics = schedular.getRootQueueMetrics();
    Assert.assertEquals(1, metrics.getAppsSubmitted());
  }
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
  public ClusterMetricsInfo() {
  } // JAXB needs this

  public ClusterMetricsInfo(final ResourceManager rm, final RMContext rmContext) {
    ResourceScheduler rs = rm.getResourceScheduler();
    QueueMetrics metrics = rs.getRootQueueMetrics();
    ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();

    this.appsSubmitted = metrics.getAppsSubmitted();
    this.appsCompleted = metrics.getAppsCompleted();
    this.appsPending = metrics.getAppsPending();
    this.appsRunning = metrics.getAppsRunning();
    this.appsFailed = metrics.getAppsFailed();
    this.appsKilled = metrics.getAppsKilled();

    this.reservedMB = metrics.getReservedMB();
    this.availableMB = metrics.getAvailableMB();
    this.allocatedMB = metrics.getAllocatedMB();

    this.reservedVirtualCores = metrics.getReservedVirtualCores();
    this.availableVirtualCores = metrics.getAvailableVirtualCores();
    this.allocatedVirtualCores = metrics.getAllocatedVirtualCores();

    this.containersAllocated = metrics.getAllocatedContainers();
    this.containersPending = metrics.getPendingContainers();
    this.containersReserved = metrics.getReservedContainers();

    this.totalMB = availableMB + allocatedMB;
    this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
    this.activeNodes = clusterMetrics.getNumActiveNMs();
    this.lostNodes = clusterMetrics.getNumLostNMs();
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
  } // JAXB needs this

  public UserMetricsInfo(final ResourceManager rm, final RMContext rmContext,
      final String user) {
    ResourceScheduler rs = rm.getResourceScheduler();
    QueueMetrics metrics = rs.getRootQueueMetrics();
    QueueMetrics userMetrics = metrics.getUserMetrics(user);
    this.userMetricsAvailable = false;

    if (userMetrics != null) {
      this.userMetricsAvailable = true;

      this.appsSubmitted = userMetrics.getAppsSubmitted();
      this.appsCompleted = metrics.getAppsCompleted();
      this.appsPending = metrics.getAppsPending();
      this.appsRunning = metrics.getAppsRunning();
      this.appsFailed = metrics.getAppsFailed();
      this.appsKilled = metrics.getAppsKilled();

      this.runningContainers = userMetrics.getAllocatedContainers();
      this.pendingContainers = userMetrics.getPendingContainers();
      this.reservedContainers = userMetrics.getReservedContainers();

      this.reservedMB = userMetrics.getReservedMB();
      this.pendingMB = userMetrics.getPendingMB();
      this.allocatedMB = userMetrics.getAllocatedMB();

      this.reservedVirtualCores = userMetrics.getReservedVirtualCores();
      this.pendingVirtualCores = userMetrics.getPendingVirtualCores();
      this.allocatedVirtualCores = userMetrics.getAllocatedVirtualCores();
    }
  }
View Full Code Here
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
 
  private void verifyClusterMetrics(int activeNodes, int appsSubmitted,
      int appsPending, int containersPending, int availableMB,
      int activeApplications) throws Exception {
    int timeoutSecs = 0;
    QueueMetrics metrics = rm.getResourceScheduler().getRootQueueMetrics();
    ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();
    boolean isAllMetricAssertionDone = false;
    String message = null;
    while (timeoutSecs++ < 5) {
      try {
        // verify queue metrics
        assertMetric("appsSubmitted", appsSubmitted, metrics.getAppsSubmitted());
        assertMetric("appsPending", appsPending, metrics.getAppsPending());
        assertMetric("containersPending", containersPending,
            metrics.getPendingContainers());
        assertMetric("availableMB", availableMB, metrics.getAvailableMB());
        assertMetric("activeApplications", activeApplications,
            metrics.getActiveApps());
        // verify node metric
        assertMetric("activeNodes", activeNodes,
            clusterMetrics.getNumActiveNMs());
        isAllMetricAssertionDone = true;
        break;
View Full Code Here
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
    Configuration conf = new Configuration();
    scheduler.setRMContext(rmContext);
    scheduler.init(conf);
    scheduler.start();
    scheduler.reinitialize(conf, rmContext);
    QueueMetrics metrics = scheduler.getRootQueueMetrics();
    int beforeAppsSubmitted = metrics.getAppsSubmitted();

    ApplicationId appId = BuilderUtils.newApplicationId(200, 1);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
        appId, 1);

    SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "queue", "user");
    scheduler.handle(appEvent);
    SchedulerEvent attemptEvent =
        new AppAttemptAddedSchedulerEvent(appAttemptId, false);
    scheduler.handle(attemptEvent);

    appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 2);
    SchedulerEvent attemptEvent2 =
        new AppAttemptAddedSchedulerEvent(appAttemptId, false);
    scheduler.handle(attemptEvent2);

    int afterAppsSubmitted = metrics.getAppsSubmitted();
    Assert.assertEquals(1, afterAppsSubmitted - beforeAppsSubmitted);
    scheduler.stop();
  }
View Full Code Here
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
    nm1.nodeHeartbeat(true);
    nm2.nodeHeartbeat(false);
    dispatcher.await();
    checkUnealthyNMCount(rm, nm2, true, 1);
    final int expectedNMs = ClusterMetrics.getMetrics().getNumActiveNMs();
    QueueMetrics metrics = rm.getResourceScheduler().getRootQueueMetrics();
    // TODO Metrics incorrect in case of the FifoScheduler
    Assert.assertEquals(5120, metrics.getAvailableMB());

    // reconnect of healthy node
    nm1 = rm.registerNode("host1:1234", 5120);
    NodeHeartbeatResponse response = nm1.nodeHeartbeat(true);
    Assert.assertTrue(NodeAction.NORMAL.equals(response.getNodeAction()));
    dispatcher.await();
    Assert.assertEquals(expectedNMs, ClusterMetrics.getMetrics().getNumActiveNMs());
    checkUnealthyNMCount(rm, nm2, true, 1);

    // reconnect of unhealthy node
    nm2 = rm.registerNode("host2:5678", 5120);
    response = nm2.nodeHeartbeat(false);
    Assert.assertTrue(NodeAction.NORMAL.equals(response.getNodeAction()));
    dispatcher.await();
    Assert.assertEquals(expectedNMs, ClusterMetrics.getMetrics().getNumActiveNMs());
    checkUnealthyNMCount(rm, nm2, true, 1);
   
    // unhealthy node changed back to healthy
    nm2 = rm.registerNode("host2:5678", 5120);
    dispatcher.await();
    response = nm2.nodeHeartbeat(true);
    response = nm2.nodeHeartbeat(true);
    dispatcher.await();
    Assert.assertEquals(5120 + 5120, metrics.getAvailableMB());

    // reconnect of node with changed capability
    nm1 = rm.registerNode("host2:5678", 10240);
    dispatcher.await();
    response = nm2.nodeHeartbeat(true);
    dispatcher.await();
    Assert.assertTrue(NodeAction.NORMAL.equals(response.getNodeAction()));
    Assert.assertEquals(5120 + 10240, metrics.getAvailableMB());
  }
View Full Code Here
388
389
390
391
392
393
394
395
    assertEquals(1, a.getMetrics().getAppsSubmitted());
    assertEquals(0, a.getMetrics().getAppsPending());
    assertEquals(0, a.getMetrics().getAppsFailed());
    assertEquals(1, a.getMetrics().getAppsCompleted());

    QueueMetrics userMetrics = a.getMetrics().getUserMetrics(user_0);
    assertEquals(1, userMetrics.getAppsSubmitted());
  }
View Full Code Here
350
351
352
353
354
355
356
357
    assertEquals(1, a.getMetrics().getAppsSubmitted());
    assertEquals(0, a.getMetrics().getAppsPending());
    assertEquals(0, a.getMetrics().getAppsFailed());
    assertEquals(1, a.getMetrics().getAppsCompleted());

    QueueMetrics userMetrics = a.getMetrics().getUserMetrics(user_0);
    assertEquals(1, userMetrics.getAppsSubmitted());
  }
View Full Code Here
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
    nm1.nodeHeartbeat(true);
    nm2.nodeHeartbeat(false);
    dispatcher.await();
    checkUnealthyNMCount(rm, nm2, true, 1);
    final int expectedNMs = ClusterMetrics.getMetrics().getNumActiveNMs();
    QueueMetrics metrics = rm.getResourceScheduler().getRootQueueMetrics();
    // TODO Metrics incorrect in case of the FifoScheduler
    Assert.assertEquals(5120, metrics.getAvailableMB());

    // reconnect of healthy node
    nm1 = rm.registerNode("host1:1234", 5120);
    HeartbeatResponse response = nm1.nodeHeartbeat(true);
    Assert.assertTrue(NodeAction.NORMAL.equals(response.getNodeAction()));
    dispatcher.await();
    Assert.assertEquals(expectedNMs, ClusterMetrics.getMetrics().getNumActiveNMs());
    checkUnealthyNMCount(rm, nm2, true, 1);

    // reconnect of unhealthy node
    nm2 = rm.registerNode("host2:5678", 5120);
    response = nm2.nodeHeartbeat(false);
    Assert.assertTrue(NodeAction.NORMAL.equals(response.getNodeAction()));
    dispatcher.await();
    Assert.assertEquals(expectedNMs, ClusterMetrics.getMetrics().getNumActiveNMs());
    checkUnealthyNMCount(rm, nm2, true, 1);

    // reconnect of node with changed capability
    nm1 = rm.registerNode("host2:5678", 10240);
    dispatcher.await();
    response = nm2.nodeHeartbeat(true);
    dispatcher.await();
    Assert.assertTrue(NodeAction.NORMAL.equals(response.getNodeAction()));
    Assert.assertEquals(5120 + 10240, metrics.getAvailableMB());
  }
View Full Code Here
108
109
110
111
112
113
114
115
    appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 2);

    event = new AppAddedSchedulerEvent(appAttemptId, "queue", "user");
    schedular.handle(event);

    QueueMetrics metrics = schedular.getRootQueueMetrics();
    Assert.assertEquals(1, metrics.getAppsSubmitted());
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics

  • org.apache.hadoop.metrics2.lib.MetricsRegistry
  • org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestLeafQueue
  • org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueueMetrics
  • org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.TestFairScheduler
  • org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.TestFifoScheduler
  • org.apache.hadoop.yarn.server.resourcemanager.TestResourceTrackerService
  • org.apache.hadoop.yarn.server.resourcemanager.TestRM
  • org.apache.hadoop.yarn.server.resourcemanager.TestRMHA
  • org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart
  • org.apache.hadoop.yarn.server.resourcemanager.TestWorkPreservingRMRestart
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.