Package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo

Examples of org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler

233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
    Configuration conf = new Configuration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
        ResourceScheduler.class);
    MockRM rm = new MockRM(conf);
    rm.start();
    FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler();

    // Add a node
    RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1);
    fs.handle(new NodeAddedSchedulerEvent(n1));
   
    // Add two applications
    ApplicationId appId1 = BuilderUtils.newApplicationId(100, 1);
    ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId(
        appId1, 1);
    SchedulerEvent event1 = new AppAddedSchedulerEvent(appAttemptId1, "queue",
        "user");
    fs.handle(event1);

    ApplicationId appId2 = BuilderUtils.newApplicationId(200, 2);
    ApplicationAttemptId appAttemptId2 = BuilderUtils.newApplicationAttemptId(
        appId2, 1);
    SchedulerEvent event2 = new AppAddedSchedulerEvent(appAttemptId2, "queue",
        "user");
    fs.handle(event2);

    List<ContainerStatus> emptyStatus = new ArrayList<ContainerStatus>();
    List<ContainerId> emptyId = new ArrayList<ContainerId>();
    List<ResourceRequest> emptyAsk = new ArrayList<ResourceRequest>();

    // Set up resource requests
   
    // Ask for a 1 GB container for app 1
    List<ResourceRequest> ask1 = new ArrayList<ResourceRequest>();
    ask1.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), "*",
        BuilderUtils.newResource(GB, 1), 1));
    fs.allocate(appAttemptId1, ask1, emptyId);

    // Ask for a 2 GB container for app 2
    List<ResourceRequest> ask2 = new ArrayList<ResourceRequest>();
    ask2.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), "*",
        BuilderUtils.newResource(2 * GB, 1), 1));
    fs.allocate(appAttemptId2, ask2, emptyId);
   
    // Trigger container assignment
    fs.handle(new NodeUpdateSchedulerEvent(n1, emptyStatus, emptyStatus));
   
    // Get the allocation for the applications and verify headroom
    Allocation allocation1 = fs.allocate(appAttemptId1, emptyAsk, emptyId);
    Assert.assertEquals("Allocation headroom", 1 * GB,
        allocation1.getResourceLimit().getMemory());

    Allocation allocation2 = fs.allocate(appAttemptId2, emptyAsk, emptyId);
    Assert.assertEquals("Allocation headroom", 1 * GB,
        allocation2.getResourceLimit().getMemory());

    rm.stop();
  }
View Full Code Here
216
217
218
219
220
221
222
223
224
  public static FifoScheduler mockFifoScheduler() throws Exception {
    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
    setupFifoQueueConfiguration(conf);

    FifoScheduler fs = new FifoScheduler();
    fs.reinitialize(conf, null, null);
    return fs;
  }
View Full Code Here
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
    Configuration conf = new Configuration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
        ResourceScheduler.class);
    MockRM rm = new MockRM(conf);
    rm.start();
    FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler();

    // Add a node
    RMNode n1 =
        MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, "127.0.0.2");
    fs.handle(new NodeAddedSchedulerEvent(n1));
   
    // Add two applications
    ApplicationId appId1 = BuilderUtils.newApplicationId(100, 1);
    ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId(
        appId1, 1);
    SchedulerEvent appEvent =
        new AppAddedSchedulerEvent(appId1, "queue", "user");
    fs.handle(appEvent);
    SchedulerEvent attemptEvent =
        new AppAttemptAddedSchedulerEvent(appAttemptId1, false);
    fs.handle(attemptEvent);

    ApplicationId appId2 = BuilderUtils.newApplicationId(200, 2);
    ApplicationAttemptId appAttemptId2 = BuilderUtils.newApplicationAttemptId(
        appId2, 1);
    SchedulerEvent appEvent2 =
        new AppAddedSchedulerEvent(appId2, "queue", "user");
    fs.handle(appEvent2);
    SchedulerEvent attemptEvent2 =
        new AppAttemptAddedSchedulerEvent(appAttemptId2, false);
    fs.handle(attemptEvent2);

    List<ContainerId> emptyId = new ArrayList<ContainerId>();
    List<ResourceRequest> emptyAsk = new ArrayList<ResourceRequest>();

    // Set up resource requests
   
    // Ask for a 1 GB container for app 1
    List<ResourceRequest> ask1 = new ArrayList<ResourceRequest>();
    ask1.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0),
        ResourceRequest.ANY, BuilderUtils.newResource(GB, 1), 1));
    fs.allocate(appAttemptId1, ask1, emptyId, null, null);

    // Ask for a 2 GB container for app 2
    List<ResourceRequest> ask2 = new ArrayList<ResourceRequest>();
    ask2.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0),
        ResourceRequest.ANY, BuilderUtils.newResource(2 * GB, 1), 1));
    fs.allocate(appAttemptId2, ask2, emptyId, null, null);
   
    // Trigger container assignment
    fs.handle(new NodeUpdateSchedulerEvent(n1));
   
    // Get the allocation for the applications and verify headroom
    Allocation allocation1 = fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null);
    Assert.assertEquals("Allocation headroom", 1 * GB,
        allocation1.getResourceLimit().getMemory());

    Allocation allocation2 = fs.allocate(appAttemptId2, emptyAsk, emptyId, null, null);
    Assert.assertEquals("Allocation headroom", 1 * GB,
        allocation2.getResourceLimit().getMemory());

    rm.stop();
  }
View Full Code Here
277
278
279
280
281
282
283
284
285
286
  public static FifoScheduler mockFifoScheduler() throws Exception {
    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
    setupFifoQueueConfiguration(conf);

    FifoScheduler fs = new FifoScheduler();
    fs.setConf(new YarnConfiguration());
    fs.reinitialize(conf, null);
    return fs;
  }
View Full Code Here
31
32
33
34
35
36
37
38
39
40
41
42
43
44
  }

  @Override
  public void trackQueue(String queueName) {
    trackedQueues.add(queueName);
    FifoScheduler fifo = (FifoScheduler) scheduler;
    // for FifoScheduler, only DEFAULT_QUEUE
    // here the three parameters doesn't affect results
    final QueueInfo queue = fifo.getQueueInfo(queueName, false, false);
    // track currentCapacity, maximumCapacity (always 1.0f)
    metrics.register("variable.queue." + queueName + ".currentcapacity",
      new Gauge<Float>() {
        @Override
        public Float getValue() {
View Full Code Here
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
    Configuration conf = new Configuration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
        ResourceScheduler.class);
    MockRM rm = new MockRM(conf);
    rm.start();
    FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler();

    // Add a node
    RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1);
    fs.handle(new NodeAddedSchedulerEvent(n1));
   
    // Add two applications
    ApplicationId appId1 = BuilderUtils.newApplicationId(100, 1);
    ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId(
        appId1, 1);
    SchedulerEvent event1 = new AppAddedSchedulerEvent(appAttemptId1, "queue",
        "user");
    fs.handle(event1);

    ApplicationId appId2 = BuilderUtils.newApplicationId(200, 2);
    ApplicationAttemptId appAttemptId2 = BuilderUtils.newApplicationAttemptId(
        appId2, 1);
    SchedulerEvent event2 = new AppAddedSchedulerEvent(appAttemptId2, "queue",
        "user");
    fs.handle(event2);

    List<ContainerStatus> emptyStatus = new ArrayList<ContainerStatus>();
    List<ContainerId> emptyId = new ArrayList<ContainerId>();
    List<ResourceRequest> emptyAsk = new ArrayList<ResourceRequest>();

    // Set up resource requests
   
    // Ask for a 1 GB container for app 1
    List<ResourceRequest> ask1 = new ArrayList<ResourceRequest>();
    ask1.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), "*",
        BuilderUtils.newResource(GB), 1));
    fs.allocate(appAttemptId1, ask1, emptyId);

    // Ask for a 2 GB container for app 2
    List<ResourceRequest> ask2 = new ArrayList<ResourceRequest>();
    ask2.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), "*",
        BuilderUtils.newResource(2 * GB), 1));
    fs.allocate(appAttemptId2, ask2, emptyId);
   
    // Trigger container assignment
    fs.handle(new NodeUpdateSchedulerEvent(n1, emptyStatus, emptyStatus));
   
    // Get the allocation for the applications and verify headroom
    Allocation allocation1 = fs.allocate(appAttemptId1, emptyAsk, emptyId);
    Assert.assertEquals("Allocation headroom", 1 * GB,
        allocation1.getResourceLimit().getMemory());

    Allocation allocation2 = fs.allocate(appAttemptId2, emptyAsk, emptyId);
    Assert.assertEquals("Allocation headroom", 1 * GB,
        allocation2.getResourceLimit().getMemory());

  }
View Full Code Here
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
  @Test
  public void testReconnectedNode() throws Exception {
    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
    conf.setQueues("default", new String[] {"default"});
    conf.setCapacity("default", 100);
    FifoScheduler fs = new FifoScheduler();
    fs.reinitialize(conf, null);

    RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1);
    RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2);

    fs.handle(new NodeAddedSchedulerEvent(n1));
    fs.handle(new NodeAddedSchedulerEvent(n2));
    List<ContainerStatus> emptyList = new ArrayList<ContainerStatus>();
    fs.handle(new NodeUpdateSchedulerEvent(n1, emptyList, emptyList));
    Assert.assertEquals(6 * GB, fs.getRootQueueMetrics().getAvailableMB());

    // reconnect n1 with downgraded memory
    n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 1);
    fs.handle(new NodeRemovedSchedulerEvent(n1));
    fs.handle(new NodeAddedSchedulerEvent(n1));
    fs.handle(new NodeUpdateSchedulerEvent(n1, emptyList, emptyList));

    Assert.assertEquals(4 * GB, fs.getRootQueueMetrics().getAvailableMB());
  }
View Full Code Here
274
275
276
277
278
279
280
281
282
283
  public static FifoScheduler mockFifoScheduler() throws Exception {
    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
    setupFifoQueueConfiguration(conf);

    FifoScheduler fs = new FifoScheduler();
    fs.setConf(new YarnConfiguration());
    fs.reinitialize(conf, null);
    return fs;
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler

  • org.apache.hadoop.yarn.api.records.ApplicationAttemptId
  • org.apache.hadoop.yarn.api.records.ApplicationId
  • org.apache.hadoop.yarn.api.records.Container
  • org.apache.hadoop.yarn.api.records.ContainerId
  • org.apache.hadoop.yarn.api.records.ContainerToken
  • org.apache.hadoop.yarn.api.records.NodeId
  • org.apache.hadoop.yarn.api.records.Resource
  • org.apache.hadoop.yarn.api.records.ResourceRequest
  • org.apache.hadoop.yarn.api.records.Token
  • org.apache.hadoop.yarn.security.ContainerTokenIdentifier
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.