Examples of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse

Package org.apache.hadoop.yarn.api.protocolrecords

Examples of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse

org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse
The response sent by the ResourceManager the ApplicationMaster during resource negotiation.

The response, includes:
- Response ID to track duplicate responses.
- An AMCommand sent by ResourceManager to let the ApplicationMaster take some actions (resync, shutdown etc.).
- A list of newly allocated {@link Container}.
- A list of completed {@link Container}s' statuses.
- The available headroom for resources in the cluster for the application.
- A list of nodes whose status has been updated.
- The number of available nodes in a cluster.
- A description of resources requested back by the cluster
@see ApplicationMasterProtocol#allocate(AllocateRequest)

    Configuration conf = new Configuration();
    TestCallbackHandler callbackHandler = new TestCallbackHandler();
    @SuppressWarnings("unchecked")
    AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);


    final AllocateResponse shutDownResponse = createAllocateResponse(
        new ArrayList<ContainerStatus>(), new ArrayList<Container>(), null);
    shutDownResponse.setAMCommand(AMCommand.AM_SHUTDOWN);
    when(client.allocate(anyFloat())).thenReturn(shutDownResponse);


    AMRMClientAsync<ContainerRequest> asyncClient =
        AMRMClientAsync.createAMRMClientAsync(client, 10, callbackHandler);
    asyncClient.init(conf);

View Full Code Here

    AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);


    List<ContainerStatus> completed = Arrays.asList(
        ContainerStatus.newInstance(newContainerId(0, 0, 0, 0),
            ContainerState.COMPLETE, "", 0));
    final AllocateResponse response = createAllocateResponse(completed,
        new ArrayList<Container>(), null);


    when(client.allocate(anyFloat())).thenReturn(response);


    AMRMClientAsync<ContainerRequest> asyncClient =

View Full Code Here

    AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);


    List<ContainerStatus> completed = Arrays.asList(
        ContainerStatus.newInstance(newContainerId(0, 0, 0, 0),
            ContainerState.COMPLETE, "", 0));
    final AllocateResponse response = createAllocateResponse(completed,
        new ArrayList<Container>(), null);


    when(client.allocate(anyFloat())).thenReturn(response);
    AMRMClientAsync<ContainerRequest> asyncClient =
        AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler);

View Full Code Here

  }


  private AllocateResponse createAllocateResponse(
      List<ContainerStatus> completed, List<Container> allocated,
      List<NMToken> nmTokens) {
    AllocateResponse response =
        AllocateResponse.newInstance(0, completed, allocated,
            new ArrayList<NodeReport>(), null, null, 1, null, nmTokens);
    return response;
  }

View Full Code Here

    AllocateRequest request = AllocateRequest.newInstance(0, 50f,
        new ArrayList<ResourceRequest>(),
        new ArrayList<ContainerId>(),
        ResourceBlacklistRequest.newInstance(new ArrayList<String>(),
            new ArrayList<String>()));
    AllocateResponse response = amClient.allocate(request);
    Assert.assertEquals(response, this.cluster.createFakeAllocateResponse());
  }

View Full Code Here

            UserGroupInformation.createRemoteUser(appAttemptId.toString());
    Token<AMRMTokenIdentifier> token = rm.getRMContext().getRMApps()
            .get(appAttemptId.getApplicationId())
            .getRMAppAttempt(appAttemptId).getAMRMToken();
    ugi.addTokenIdentifier(token.decodeIdentifier());
    AllocateResponse response = ugi.doAs(
            new PrivilegedExceptionAction<AllocateResponse>() {
      @Override
      public AllocateResponse run() throws Exception {
        return rm.getApplicationMasterService().allocate(request);
      }
    });


    // waiting until the AM container is allocated
    while (true) {
      if (response != null && ! response.getAllocatedContainers().isEmpty()) {
        // get AM container
        Container container = response.getAllocatedContainers().get(0);
        se.getNmMap().get(container.getNodeId())
                .addNewContainer(container, -1L);
        // start AM container
        amContainer = container;
        LOG.debug(MessageFormat.format("Application {0} starts its " +

View Full Code Here

  @Override
  @SuppressWarnings("unchecked")
  protected void processResponseQueue()
          throws InterruptedException, YarnException, IOException {
    while (! responseQueue.isEmpty()) {
      AllocateResponse response = responseQueue.take();


      // check completed containers
      if (! response.getCompletedContainersStatuses().isEmpty()) {
        for (ContainerStatus cs : response.getCompletedContainersStatuses()) {
          ContainerId containerId = cs.getContainerId();
          if (cs.getExitStatus() == ContainerExitStatus.SUCCESS) {
            if (assignedMaps.containsKey(containerId)) {
              LOG.debug(MessageFormat.format("Application {0} has one" +
                      "mapper finished ({1}).", appId, containerId));
              assignedMaps.remove(containerId);
              mapFinished ++;
              finishedContainers ++;
            } else if (assignedReduces.containsKey(containerId)) {
              LOG.debug(MessageFormat.format("Application {0} has one" +
                      "reducer finished ({1}).", appId, containerId));
              assignedReduces.remove(containerId);
              reduceFinished ++;
              finishedContainers ++;
            } else {
              // am container released event
              isFinished = true;
              LOG.info(MessageFormat.format("Application {0} goes to " +
                      "finish.", appId));
            }
          } else {
            // container to be killed
            if (assignedMaps.containsKey(containerId)) {
              LOG.debug(MessageFormat.format("Application {0} has one " +
                      "mapper killed ({1}).", appId, containerId));
              pendingFailedMaps.add(assignedMaps.remove(containerId));
            } else if (assignedReduces.containsKey(containerId)) {
              LOG.debug(MessageFormat.format("Application {0} has one " +
                      "reducer killed ({1}).", appId, containerId));
              pendingFailedReduces.add(assignedReduces.remove(containerId));
            } else {
              LOG.info(MessageFormat.format("Application {0}'s AM is " +
                      "going to be killed. Restarting...", appId));
              restart();
            }
          }
        }
      }
      
      // check finished
      if (isAMContainerRunning &&
              (mapFinished == mapTotal) &&
              (reduceFinished == reduceTotal)) {
        // to release the AM container
        se.getNmMap().get(amContainer.getNodeId())
                .cleanupContainer(amContainer.getId());
        isAMContainerRunning = false;
        LOG.debug(MessageFormat.format("Application {0} sends out event " +
                "to clean up its AM container.", appId));
        isFinished = true;
      }


      // check allocated containers
      for (Container container : response.getAllocatedContainers()) {
        if (! scheduledMaps.isEmpty()) {
          ContainerSimulator cs = scheduledMaps.remove();
          LOG.debug(MessageFormat.format("Application {0} starts a " +
                  "launch a mapper ({1}).", appId, container.getId()));
          assignedMaps.put(container.getId(), cs);

View Full Code Here

            UserGroupInformation.createRemoteUser(appAttemptId.toString());
    Token<AMRMTokenIdentifier> token = rm.getRMContext().getRMApps()
            .get(appAttemptId.getApplicationId())
            .getRMAppAttempt(appAttemptId).getAMRMToken();
    ugi.addTokenIdentifier(token.decodeIdentifier());
    AllocateResponse response = ugi.doAs(
            new PrivilegedExceptionAction<AllocateResponse>() {
      @Override
      public AllocateResponse run() throws Exception {
        return rm.getApplicationMasterService().allocate(request);
      }

View Full Code Here

  @Test
  public void testPbRecordFactory() {
    RecordFactory pbRecordFactory = RecordFactoryPBImpl.get();
    
    try {
      AllocateResponse response =
          pbRecordFactory.newRecordInstance(AllocateResponse.class);
      Assert.assertEquals(AllocateResponsePBImpl.class, response.getClass());
    } catch (YarnRuntimeException e) {
      e.printStackTrace();
      Assert.fail("Failed to crete record");
    }
    
    try {
      AllocateRequest response =
          pbRecordFactory.newRecordInstance(AllocateRequest.class);
      Assert.assertEquals(AllocateRequestPBImpl.class, response.getClass());
    } catch (YarnRuntimeException e) {
      e.printStackTrace();
      Assert.fail("Failed to crete record");
    }
  }

View Full Code Here

    for (int i = 0; i < 5; i++) {
      decContainers.add(ContainerResourceDecrease.newInstance(null,
          Resource.newInstance(1024, i)));
    }


    AllocateResponse r =
        AllocateResponse.newInstance(3, new ArrayList<ContainerStatus>(),
            new ArrayList<Container>(), new ArrayList<NodeReport>(), null,
            AMCommand.AM_RESYNC, 3, null, new ArrayList<NMToken>(),
            incContainers, decContainers);


    // serde
    AllocateResponseProto p = ((AllocateResponsePBImpl) r).getProto();
    r = new AllocateResponsePBImpl(p);


    // check value
    Assert
        .assertEquals(incContainers.size(), r.getIncreasedContainers().size());
    Assert
        .assertEquals(decContainers.size(), r.getDecreasedContainers().size());


    for (int i = 0; i < incContainers.size(); i++) {
      Assert.assertEquals(i, r.getIncreasedContainers().get(i).getCapability()
          .getVirtualCores());
    }


    for (int i = 0; i < decContainers.size(); i++) {
      Assert.assertEquals(i, r.getDecreasedContainers().get(i).getCapability()
          .getVirtualCores());
    }
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse

com.alibaba.jstorm.yarn.MasterServer

com.cloudera.iterativereduce.yarn.ResourceManagerHandler

com.hortonworks.simpleyarnapp.ApplicationMaster

com.yahoo.storm.yarn.MasterServer

eu.stratosphere.yarn.ApplicationMaster

org.apache.flink.yarn.appMaster.ApplicationMaster

org.apache.giraph.yarn.GiraphApplicationMaster

org.apache.hadoop.mapreduce.v2.app.local.LocalContainerAllocator

org.apache.hadoop.mapreduce.v2.app.MRAppBenchmark

org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.