Package org.apache.hadoop.yarn.server.resourcemanager.scheduler

Examples of org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler

170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
      RMContainerState.RUNNING);
    rm2.waitForState(nm1, runningContainer.getContainerId(),
      RMContainerState.RUNNING);
    rm2.waitForContainerToComplete(loadedAttempt1, completedContainer);

    AbstractYarnScheduler scheduler =
        (AbstractYarnScheduler) rm2.getResourceScheduler();
    SchedulerNode schedulerNode1 = scheduler.getSchedulerNode(nm1.getNodeId());

    // ********* check scheduler node state.*******
    // 2 running containers.
    Resource usedResources = Resources.multiply(containerResource, 2);
    Resource nmResource =
        Resource.newInstance(nm1.getMemory(), nm1.getvCores());

    assertTrue(schedulerNode1.isValidContainer(amContainer.getContainerId()));
    assertTrue(schedulerNode1.isValidContainer(runningContainer
      .getContainerId()));
    assertFalse(schedulerNode1.isValidContainer(completedContainer
      .getContainerId()));
    // 2 launched containers, 1 completed container
    assertEquals(2, schedulerNode1.getNumContainers());

    assertEquals(Resources.subtract(nmResource, usedResources),
      schedulerNode1.getAvailableResource());
    assertEquals(usedResources, schedulerNode1.getUsedResource());
    Resource availableResources = Resources.subtract(nmResource, usedResources);

    // ***** check queue state based on the underlying scheduler ********
    Map<ApplicationId, SchedulerApplication> schedulerApps =
        ((AbstractYarnScheduler) rm2.getResourceScheduler())
          .getSchedulerApplications();
    SchedulerApplication schedulerApp =
        schedulerApps.get(recoveredApp1.getApplicationId());

    if (schedulerClass.equals(CapacityScheduler.class)) {
      checkCSQueue(rm2, schedulerApp, nmResource, nmResource, usedResources, 2);
    } else if (schedulerClass.equals(FifoScheduler.class)) {
      checkFifoQueue(schedulerApp, usedResources, availableResources);
    }

    // *********** check scheduler attempt state.********
    SchedulerApplicationAttempt schedulerAttempt =
        schedulerApp.getCurrentAppAttempt();
    assertTrue(schedulerAttempt.getLiveContainers().contains(
      scheduler.getRMContainer(amContainer.getContainerId())));
    assertTrue(schedulerAttempt.getLiveContainers().contains(
      scheduler.getRMContainer(runningContainer.getContainerId())));
    assertEquals(schedulerAttempt.getCurrentConsumption(), usedResources);
    assertEquals(availableResources, schedulerAttempt.getHeadroom());

    // *********** check appSchedulingInfo state ***********
    assertEquals((1 << 22) + 1, schedulerAttempt.getNewContainerId());
View Full Code Here
486
487
488
489
490
491
492
493
494
495
496
497
      completedContainer), null);
    rm2.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.FAILED);
    // Wait for RM to settle down on recovering containers;
    Thread.sleep(3000);

    AbstractYarnScheduler scheduler =
        (AbstractYarnScheduler) rm2.getResourceScheduler();
    // Previous AM failed, The failed AM should once again release the
    // just-recovered containers.
    assertNull(scheduler.getRMContainer(runningContainer.getContainerId()));
    assertNull(scheduler.getRMContainer(completedContainer.getContainerId()));
  }
View Full Code Here
526
527
528
529
530
531
532
533
534
535
536
537
    assertEquals(RMAppState.FINISHED, recoveredApp1.getState());

    // Wait for RM to settle down on recovering containers;
    Thread.sleep(3000);

    AbstractYarnScheduler scheduler =
        (AbstractYarnScheduler) rm2.getResourceScheduler();

    // scheduler should not recover containers for finished apps.
    assertNull(scheduler.getRMContainer(runningContainer.getContainerId()));
    assertNull(scheduler.getRMContainer(completedContainer.getContainerId()));
  }
View Full Code Here
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
    nm1.registerNode();
    RMApp app1_1 = rm1.submitApp(1024);
    MockAM am1_1 = MockRM.launchAndRegisterAM(app1_1, rm1, nm1);
   
    RMAppAttempt attempt0 = app1_1.getCurrentAppAttempt();
    AbstractYarnScheduler scheduler =
        ((AbstractYarnScheduler) rm1.getResourceScheduler());
   
    Assert.assertTrue(scheduler.getRMContainer(
        attempt0.getMasterContainer().getId()).isAMContainer());

    // Re-start RM
    rm2 = new MockRM(conf, memStore);
    rm2.start();
    nm1.setResourceTrackerService(rm2.getResourceTrackerService());

    List<NMContainerStatus> am1_1Containers =
        createNMContainerStatusForApp(am1_1);
    nm1.registerNode(am1_1Containers, null);

    // Wait for RM to settle down on recovering containers;
    waitForNumContainersToRecover(2, rm2, am1_1.getApplicationAttemptId());

    scheduler = ((AbstractYarnScheduler) rm2.getResourceScheduler());
    Assert.assertTrue(scheduler.getRMContainer(
        attempt0.getMasterContainer().getId()).isAMContainer());
  }
View Full Code Here
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
    assertEquals(allocatedVirtualCores, qm.getAllocatedVirtualCores());
  }

  private void waitForNumContainersToRecover(int num, MockRM rm,
      ApplicationAttemptId attemptId) throws Exception {
    AbstractYarnScheduler scheduler =
        (AbstractYarnScheduler) rm.getResourceScheduler();
    SchedulerApplicationAttempt attempt =
        scheduler.getApplicationAttempt(attemptId);
    while (attempt == null) {
      System.out.println("Wait for scheduler attempt " + attemptId
          + " to be created");
      Thread.sleep(200);
      attempt = scheduler.getApplicationAttempt(attemptId);
    }
    while (attempt.getLiveContainers().size() < num) {
      System.out.println("Wait for " + num + " containers to recover.");
      Thread.sleep(200);
    }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler

  • org.apache.hadoop.yarn.api.records.ApplicationAttemptId
  • org.apache.hadoop.yarn.api.records.ApplicationId
  • org.apache.hadoop.yarn.api.records.Container
  • org.apache.hadoop.yarn.api.records.ContainerId
  • org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt
  • org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp
  • org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer
  • org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl
  • org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerRecoverEvent
  • org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.