Examples of org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest

Package org.apache.hadoop.yarn.client.api.AMRMClient

Examples of org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest

org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest

      amClient.registerApplicationMaster("Host", 10000, "");
      
      assertTrue(amClient.ask.size() == 0);
      assertTrue(amClient.release.size() == 0);
      
      ContainerRequest storedContainer1 = 
          new ContainerRequest(capability, nodes, racks, priority);
      amClient.addContainerRequest(storedContainer1);
      assertTrue(amClient.ask.size() == 3);
      assertTrue(amClient.release.size() == 0);
      
      List<String> localNodeBlacklist = new ArrayList<String>();
      localNodeBlacklist.add(node);
      
      // put node in black list, so no container assignment
      amClient.updateBlacklist(localNodeBlacklist, null);


      int allocatedContainerCount = getAllocatedContainersNumber(amClient,
        DEFAULT_ITERATION);
      // the only node is in blacklist, so no allocation
      assertTrue(allocatedContainerCount == 0);


      // Remove node from blacklist, so get assigned with 2
      amClient.updateBlacklist(null, localNodeBlacklist);
      ContainerRequest storedContainer2 = 
              new ContainerRequest(capability, nodes, racks, priority);
      amClient.addContainerRequest(storedContainer2);
      allocatedContainerCount = getAllocatedContainersNumber(amClient,
          DEFAULT_ITERATION);
      assertEquals(allocatedContainerCount, 2);
      
      // Test in case exception in allocate(), blacklist is kept
      assertTrue(amClient.blacklistAdditions.isEmpty());
      assertTrue(amClient.blacklistRemovals.isEmpty());
      
      // create a invalid ContainerRequest - memory value is minus
      ContainerRequest invalidContainerRequest = 
          new ContainerRequest(Resource.newInstance(-1024, 1),
              nodes, racks, priority);
      amClient.addContainerRequest(invalidContainerRequest);
      amClient.updateBlacklist(localNodeBlacklist, null);
      try {
        // allocate() should complain as ContainerRequest is invalid.

View Full Code Here

    
    assertTrue(amClient.ask.size() == 0);
    assertTrue(amClient.release.size() == 0);
    
    amClient.addContainerRequest(
        new ContainerRequest(capability, nodes, racks, priority));
    amClient.addContainerRequest(
        new ContainerRequest(capability, nodes, racks, priority));
    amClient.addContainerRequest(
        new ContainerRequest(capability, nodes, racks, priority));
    amClient.addContainerRequest(
        new ContainerRequest(capability, nodes, racks, priority));
    amClient.removeContainerRequest(
        new ContainerRequest(capability, nodes, racks, priority));
    amClient.removeContainerRequest(
        new ContainerRequest(capability, nodes, racks, priority));
    
    int containersRequestedNode = amClient.remoteRequestsTable.get(priority)
        .get(node).get(capability).remoteRequest.getNumContainers();
    int containersRequestedRack = amClient.remoteRequestsTable.get(priority)
        .get(rack).get(capability).remoteRequest.getNumContainers();
    int containersRequestedAny = amClient.remoteRequestsTable.get(priority)
    .get(ResourceRequest.ANY).get(capability).remoteRequest.getNumContainers();


    assertTrue(containersRequestedNode == 2);
    assertTrue(containersRequestedRack == 2);
    assertTrue(containersRequestedAny == 2);
    assertTrue(amClient.ask.size() == 3);
    assertTrue(amClient.release.size() == 0);


    // RM should allocate container within 2 calls to allocate()
    int allocatedContainerCount = 0;
    int iterationsLeft = 3;
    Set<ContainerId> releases = new TreeSet<ContainerId>();
    
    NMTokenCache.clearCache();
    Assert.assertEquals(0, NMTokenCache.numberOfNMTokensInCache());
    HashMap<String, Token> receivedNMTokens = new HashMap<String, Token>();
    
    while (allocatedContainerCount < containersRequestedAny
        && iterationsLeft-- > 0) {
      AllocateResponse allocResponse = amClient.allocate(0.1f);
      assertTrue(amClient.ask.size() == 0);
      assertTrue(amClient.release.size() == 0);
      
      assertTrue(nodeCount == amClient.getClusterNodeCount());
      allocatedContainerCount += allocResponse.getAllocatedContainers().size();
      for(Container container : allocResponse.getAllocatedContainers()) {
        ContainerId rejectContainerId = container.getId();
        releases.add(rejectContainerId);
        amClient.releaseAssignedContainer(rejectContainerId);
      }
      
      for (NMToken token : allocResponse.getNMTokens()) {
        String nodeID = token.getNodeId().toString();
        if (receivedNMTokens.containsKey(nodeID)) {
          Assert.fail("Received token again for : " + nodeID);          
        }
        receivedNMTokens.put(nodeID, token.getToken());
      }
      
      if(allocatedContainerCount < containersRequestedAny) {
        // sleep to let NM's heartbeat to RM and trigger allocations
        sleep(100);
      }
    }
    
    // Should receive atleast 1 token
    Assert.assertTrue(receivedNMTokens.size() > 0
        && receivedNMTokens.size() <= nodeCount);
    
    assertTrue(allocatedContainerCount == containersRequestedAny);
    assertTrue(amClient.release.size() == 2);
    assertTrue(amClient.ask.size() == 0);
    
    // need to tell the AMRMClient that we dont need these resources anymore
    amClient.removeContainerRequest(
        new ContainerRequest(capability, nodes, racks, priority));
    amClient.removeContainerRequest(
        new ContainerRequest(capability, nodes, racks, priority));
    assertTrue(amClient.ask.size() == 3);
    // send 0 container count request for resources that are no longer needed
    ResourceRequest snoopRequest = amClient.ask.iterator().next();
    assertTrue(snoopRequest.getNumContainers() == 0);
    
    // test RPC exception handling
    amClient.addContainerRequest(new ContainerRequest(capability, nodes,
        racks, priority));
    amClient.addContainerRequest(new ContainerRequest(capability, nodes,
        racks, priority));
    snoopRequest = amClient.ask.iterator().next();
    assertTrue(snoopRequest.getNumContainers() == 2);
    
    ApplicationMasterProtocol realRM = amClient.rmClient;
    try {
      ApplicationMasterProtocol mockRM = mock(ApplicationMasterProtocol.class);
      when(mockRM.allocate(any(AllocateRequest.class))).thenAnswer(
          new Answer<AllocateResponse>() {
            public AllocateResponse answer(InvocationOnMock invocation)
                throws Exception {
              amClient.removeContainerRequest(
                             new ContainerRequest(capability, nodes, 
                                                          racks, priority));
              amClient.removeContainerRequest(
                  new ContainerRequest(capability, nodes, racks, priority));
              throw new Exception();
            }
          });
      amClient.rmClient = mockRM;
      amClient.allocate(0.1f);

View Full Code Here

      int askCount = numTotalContainers - numRequestedContainers.get();
      numRequestedContainers.addAndGet(askCount);


      if (askCount > 0) {
        for (int i = 0; i < askCount; ++i) {
          ContainerRequest containerAsk = setupContainerAskForRM();
          amRMClient.addContainerRequest(containerAsk);
        }
      }
      
      if (numCompletedContainers.get() == numTotalContainers) {

View Full Code Here

    // Until we get our fully allocated quota, we keep on polling RM for
    // containers
    // Keep looping until all the containers are launched and shell script
    // executed on them ( regardless of success/failure).
    for (int i = 0; i < numTotalContainers; ++i) {
      ContainerRequest containerAsk = setupContainerAskForRM();
      amRMClient.addContainerRequest(containerAsk);
    }
    numRequestedContainers.set(numTotalContainers);


    while (!done

View Full Code Here

    // Set up resource type requirements
    // For now, only memory is supported so we set memory requirements
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(containerMemory);


    ContainerRequest request = new ContainerRequest(capability, null, null,
        pri);
    LOG.info("Requested container ask: " + request.toString());
    return request;
  }

View Full Code Here

    String rack = nodeReports.get(0).getRackName();
    String[] nodes = new String[] {node};
    String[] racks = new String[] {rack};


    for (int i = 0; i < num; ++i) {
      rmClient.addContainerRequest(new ContainerRequest(capability, nodes,
          racks, priority));
    }


    int containersRequestedAny = rmClient.remoteRequestsTable.get(priority)
        .get(ResourceRequest.ANY).get(capability).remoteRequest

View Full Code Here

        CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
        MyResolver.class, DNSToSwitchMapping.class);
    client.init(conf);
 
    Resource capability = Resource.newInstance(1024, 1);
    ContainerRequest request =
        new ContainerRequest(capability, new String[] {"host1", "host2"},
            new String[] {"/rack2"}, Priority.newInstance(1));
    client.addContainerRequest(request);
    verifyResourceRequest(client, request, "host1", true);
    verifyResourceRequest(client, request, "host2", true);
    verifyResourceRequest(client, request, "/rack1", true);

View Full Code Here

        CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
        MyResolver.class, DNSToSwitchMapping.class);
    client.init(conf);
    
    Resource capability = Resource.newInstance(1024, 1);
    ContainerRequest nodeLevelRequest =
        new ContainerRequest(capability, new String[] {"host1", "host2"},
            null, Priority.newInstance(1), false);
    client.addContainerRequest(nodeLevelRequest);


    verifyResourceRequest(client, nodeLevelRequest, ResourceRequest.ANY, false);
    verifyResourceRequest(client, nodeLevelRequest, "/rack1", false);
    verifyResourceRequest(client, nodeLevelRequest, "host1", true);
    verifyResourceRequest(client, nodeLevelRequest, "host2", true);
    
    // Make sure we don't get any errors with two node-level requests at the
    // same priority
    ContainerRequest nodeLevelRequest2 =
        new ContainerRequest(capability, new String[] {"host2", "host3"},
            null, Priority.newInstance(1), false);
    client.addContainerRequest(nodeLevelRequest2);
    
    AMRMClient.ContainerRequest rackLevelRequest =
        new AMRMClient.ContainerRequest(capability, null,
            new String[] {"/rack3", "/rack4"}, Priority.newInstance(2), false);
    client.addContainerRequest(rackLevelRequest);
    
    verifyResourceRequest(client, rackLevelRequest, ResourceRequest.ANY, false);
    verifyResourceRequest(client, rackLevelRequest, "/rack3", true);
    verifyResourceRequest(client, rackLevelRequest, "/rack4", true);
    
    // Make sure we don't get any errors with two rack-level requests at the
    // same priority
    AMRMClient.ContainerRequest rackLevelRequest2 =
        new AMRMClient.ContainerRequest(capability, null,
            new String[] {"/rack4", "/rack5"}, Priority.newInstance(2), false);
    client.addContainerRequest(rackLevelRequest2);
    
    ContainerRequest bothLevelRequest =
        new ContainerRequest(capability, new String[] {"host3", "host4"},
            new String[] {"rack1", "/otherrack"},
            Priority.newInstance(3), false);
    client.addContainerRequest(bothLevelRequest);


    verifyResourceRequest(client, bothLevelRequest, ResourceRequest.ANY, false);
    verifyResourceRequest(client, bothLevelRequest, "rack1",
        true);
    verifyResourceRequest(client, bothLevelRequest, "/otherrack",
        true);
    verifyResourceRequest(client, bothLevelRequest, "host3", true);
    verifyResourceRequest(client, bothLevelRequest, "host4", true);
    
    // Make sure we don't get any errors with two both-level requests at the
    // same priority
    ContainerRequest bothLevelRequest2 =
        new ContainerRequest(capability, new String[] {"host4", "host5"},
            new String[] {"rack1", "/otherrack2"},
            Priority.newInstance(3), false);
    client.addContainerRequest(bothLevelRequest2);
  }

View Full Code Here

        CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
        MyResolver.class, DNSToSwitchMapping.class);
    client.init(conf);
    
    Resource capability = Resource.newInstance(1024, 1);
    ContainerRequest request1 =
        new ContainerRequest(capability, new String[] {"host1", "host2"},
            null, Priority.newInstance(1), false);
    client.addContainerRequest(request1);
    ContainerRequest request2 =
        new ContainerRequest(capability, new String[] {"host3"},
            null, Priority.newInstance(1), true);
    client.addContainerRequest(request2);
  }

View Full Code Here

        CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
        MyResolver.class, DNSToSwitchMapping.class);
    client.init(conf);
    
    Resource capability = Resource.newInstance(1024, 1);
    ContainerRequest request1 =
        new ContainerRequest(capability, new String[] {"host1", "host2"},
            null, Priority.newInstance(1), false);
    client.addContainerRequest(request1);
    
    client.removeContainerRequest(request1);


    ContainerRequest request2 =
        new ContainerRequest(capability, new String[] {"host3"},
            null, Priority.newInstance(1), true);
    client.addContainerRequest(request2);
    
    client.removeContainerRequest(request2);
    
    ContainerRequest request3 =
        new ContainerRequest(capability, new String[] {"host1", "host2"},
            null, Priority.newInstance(1), false);
    client.addContainerRequest(request3);
    
    client.removeContainerRequest(request3);
    
    ContainerRequest request4 =
        new ContainerRequest(capability, null,
            new String[] {"rack1"}, Priority.newInstance(1), true);
    client.addContainerRequest(request4);


  }

View Full Code Here

0 1 2 3 4 5 6

TOP

Related Classes of org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest

com.alibaba.jstorm.yarn.StormAMRMClient

com.hortonworks.simpleyarnapp.ApplicationMaster

com.hortonworks.simpleyarnapp.ApplicationMasterAsync

com.yahoo.storm.yarn.StormAMRMClient

eu.stratosphere.yarn.ApplicationMaster

org.apache.flink.yarn.appMaster.ApplicationMaster

org.apache.giraph.yarn.GiraphApplicationMaster

org.apache.hadoop.yarn.applications.distributedshell.ApplicationMaster

org.apache.hadoop.yarn.applications.distributedshell.ApplicationMaster$RMCallbackHandler

org.apache.hadoop.yarn.client.api.impl.TestAMRMClient

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.