Examples of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse

Package org.apache.hadoop.yarn.api.protocolrecords

Examples of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse

org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse
The response sent by the ResourceManager the ApplicationMaster during resource negotiation.

The response, includes:
- Response ID to track duplicate responses.
- An AMCommand sent by ResourceManager to let the ApplicationMaster take some actions (resync, shutdown etc.).
- A list of newly allocated {@link Container}.
- A list of completed {@link Container}s' statuses.
- The available headroom for resources in the cluster for the application.
- A list of nodes whose status has been updated.
- The number of available nodes in a cluster.
- A description of resources requested back by the cluster
@see ApplicationMasterProtocol#allocate(AllocateRequest)

  protected synchronized void heartbeat() throws Exception {
    AllocateRequest allocateRequest = BuilderUtils.newAllocateRequest(
        this.applicationAttemptId, this.lastResponseID, super
            .getApplicationProgress(), new ArrayList<ResourceRequest>(),
        new ArrayList<ContainerId>());
    AllocateResponse allocateResponse = scheduler.allocate(allocateRequest);
    AMResponse response;
    try {
      response = allocateResponse.getAMResponse();
      // Reset retry count if no exception occurred.
      retrystartTime = System.currentTimeMillis();
    } catch (Exception e) {
      // This can happen when the connection to the RM has gone down. Keep
      // re-trying until the retryInterval has expired.

View Full Code Here


              @Override
              public AllocateResponse allocate(AllocateRequest request)
                  throws YarnRemoteException {


                AllocateResponse response =
                    Records.newRecord(AllocateResponse.class);
                List<ResourceRequest> askList = request.getAskList();
                List<Container> containers = new ArrayList<Container>();
                for (ResourceRequest req : askList) {
                  if (req.getHostName() != "*") {
                    continue;
                  }
                  int numContainers = req.getNumContainers();
                  for (int i = 0; i < numContainers; i++) {
                    ContainerId containerId =
                        BuilderUtils.newContainerId(
                          request.getApplicationAttemptId(),
                          request.getResponseId() + i);
                    containers.add(BuilderUtils
                      .newContainer(containerId, BuilderUtils.newNodeId("host"
                          + containerId.getId(), 2345),
                        "host" + containerId.getId() + ":5678", req
                          .getCapability(), req.getPriority(), null));
                  }
                }


                AMResponse amResponse = Records.newRecord(AMResponse.class);
                amResponse.setAllocatedContainers(containers);
                amResponse.setResponseId(request.getResponseId() + 1);
                response.setAMResponse(amResponse);
                response.setNumClusterNodes(350);
                return response;
              }
            };
          }
        };

View Full Code Here

    }
    for (ContainerId id : releasedContainers) {
      LOG.info("Released container, id=" + id.getId());
    }


    AllocateResponse resp = resourceManager.allocate(req);
    return resp.getAMResponse();
  }

View Full Code Here

    RegisterApplicationMasterResponse response;
    try {
      response = registerApplicationMaster("localhost", 10080, "http://localhost:1234");


      // If the number of cluster nodes is ZERO, it waits for available nodes.
      AllocateResponse allocateResponse = allocate(0.0f);
      while(allocateResponse.getNumClusterNodes() < 1) {
        try {
          Thread.sleep(WAIT_INTERVAL_AVAILABLE_NODES);
          LOG.info("Waiting for Available Cluster Nodes");
          allocateResponse = allocate(0);
        } catch (InterruptedException e) {
          LOG.error(e);
        }
      }
      context.getQueryMasterContext().getWorkerContext().setNumClusterNodes(allocateResponse.getNumClusterNodes());
    } catch (IOException e) {
      LOG.error(e);
    } catch (YarnException e) {
      LOG.error(e);
    }

View Full Code Here


  private AtomicLong prevReportTime = new AtomicLong(0);
  private int reportInterval = 5 * 1000; // second


  public void heartbeat() throws Exception {
    AllocateResponse allocateResponse = allocate(context.getProgress());


    List<Container> allocatedContainers = allocateResponse.getAllocatedContainers();


    long currentTime = System.currentTimeMillis();
    if ((currentTime - prevReportTime.longValue()) >= reportInterval) {
      LOG.debug("Available Cluster Nodes: " + allocateResponse.getNumClusterNodes());
      LOG.debug("Num of Allocated Containers: " + allocatedContainers.size());
      LOG.info("Available Resource: " + allocateResponse.getAvailableResources());
      prevReportTime.set(currentTime);
    }


    if (allocatedContainers.size() > 0) {
      LOG.info("================================================================");
      for (Container container : allocateResponse.getAllocatedContainers()) {
        LOG.info("> Container Id: " + container.getId());
        LOG.info("> Node Id: " + container.getNodeId());
        LOG.info("> Resource (Mem): " + container.getResource().getMemory());
        LOG.info("> Priority: " + container.getPriority());
      }

View Full Code Here

   * @return the AllocateResponse, which we may or may not need.
   */
  private AllocateResponse sendHeartbeat() {
    heartbeat.setProgress(progress);
    heartbeat.setResponseId(lastResponseId.incrementAndGet());
    AllocateResponse allocateResponse = null;
    try {
      allocateResponse = resourceManager.allocate(heartbeat);
      final int responseId = allocateResponse.getAMResponse().getResponseId();
      if (responseId != lastResponseId.get()) {
        lastResponseId.set(responseId);
      }
      checkForRebootFlag(allocateResponse.getAMResponse());
      return allocateResponse;
    } catch (YarnRemoteException yre) {
      throw new IllegalStateException("sendHeartbeat() failed with " +
        "YarnRemoteException: ", yre);
    }

View Full Code Here

      List<ContainerId> releasedContainers = Lists.newArrayListWithCapacity(0);
      allocRequest.setResponseId(lastResponseId.get());
      allocRequest.setApplicationAttemptId(appAttemptId);
      allocRequest.addAllReleases(releasedContainers);
      allocRequest.setProgress(progress);
      AllocateResponse allocResponse = resourceManager.allocate(allocRequest);
      AMResponse amResponse = allocResponse.getAMResponse();
      if (amResponse.getResponseId() != lastResponseId.get()) {
        lastResponseId.set(amResponse.getResponseId());
      }
      checkForRebootFlag(amResponse);
      // now, make THIS our new HEARTBEAT object, but with ZERO new requests!

View Full Code Here

      super("AMRM Heartbeater thread");
    }
    
    public void run() {
      while (true) {
        AllocateResponse response = null;
        // synchronization ensures we don't send heartbeats after unregistering
        synchronized (unregisterHeartbeatLock) {
          if (!keepRunning) {
            return;
          }
            
          try {
            response = client.allocate(progress);
          } catch (Throwable ex) {
            LOG.error("Exception on heartbeat", ex);
            savedException = ex;
            // interrupt handler thread in case it waiting on the queue
            handlerThread.interrupt();
            return;
          }
        }
        if (response != null) {
          while (true) {
            try {
              responseQueue.put(response);
              if (response.getAMCommand() == AMCommand.AM_RESYNC
                  || response.getAMCommand() == AMCommand.AM_SHUTDOWN) {
                return;
              }
              break;
            } catch (InterruptedException ex) {
              LOG.info("Interrupted while waiting to put on response queue", ex);

View Full Code Here

      while (true) {
        if (!keepRunning) {
          return;
        }
        try {
          AllocateResponse response;
          if(savedException != null) {
            LOG.error("Stopping callback due to: ", savedException);
            handler.onError(savedException);
            return;
          }
          try {
            response = responseQueue.take();
          } catch (InterruptedException ex) {
            LOG.info("Interrupted while waiting for queue", ex);
            continue;
          }


          if (response.getAMCommand() != null) {
            switch(response.getAMCommand()) {
            case AM_RESYNC:
            case AM_SHUTDOWN:
              handler.onShutdownRequest();
              LOG.info("Shutdown requested. Stopping callback.");
              return;
            default:
              String msg =
                    "Unhandled value of RM AMCommand: " + response.getAMCommand();
              LOG.error(msg);
              throw new YarnRuntimeException(msg);
            }
          }
          List<NodeReport> updatedNodes = response.getUpdatedNodes();
          if (!updatedNodes.isEmpty()) {
            handler.onNodesUpdated(updatedNodes);
          }


          List<ContainerStatus> completed =
              response.getCompletedContainersStatuses();
          if (!completed.isEmpty()) {
            handler.onContainersCompleted(completed);
          }


          List<Container> allocated = response.getAllocatedContainers();
          if (!allocated.isEmpty()) {
            handler.onContainersAllocated(allocated);
          }


          progress = handler.getProgress();

View Full Code Here

  @Override
  public AllocateResponse allocate(float progressIndicator) 
      throws YarnException, IOException {
    Preconditions.checkArgument(progressIndicator >= 0,
        "Progress indicator should not be negative");
    AllocateResponse allocateResponse = null;
    List<ResourceRequest> askList = null;
    List<ContainerId> releaseList = null;
    AllocateRequest allocateRequest = null;
    List<String> blacklistToAdd = new ArrayList<String>();
    List<String> blacklistToRemove = new ArrayList<String>();
    
    try {
      synchronized (this) {
        askList = new ArrayList<ResourceRequest>(ask.size());
        for(ResourceRequest r : ask) {
          // create a copy of ResourceRequest as we might change it while the 
          // RPC layer is using it to send info across
          askList.add(ResourceRequest.newInstance(r.getPriority(),
              r.getResourceName(), r.getCapability(), r.getNumContainers(),
              r.getRelaxLocality()));
        }
        releaseList = new ArrayList<ContainerId>(release);
        // optimistically clear this collection assuming no RPC failure
        ask.clear();
        release.clear();


        blacklistToAdd.addAll(blacklistAdditions);
        blacklistToRemove.addAll(blacklistRemovals);
        
        ResourceBlacklistRequest blacklistRequest = 
            (blacklistToAdd != null) || (blacklistToRemove != null) ? 
            ResourceBlacklistRequest.newInstance(blacklistToAdd,
                blacklistToRemove) : null;
        
        allocateRequest =
            AllocateRequest.newInstance(lastResponseId, progressIndicator,
              askList, releaseList, blacklistRequest);
        // clear blacklistAdditions and blacklistRemovals before 
        // unsynchronized part
        blacklistAdditions.clear();
        blacklistRemovals.clear();
      }


      allocateResponse = rmClient.allocate(allocateRequest);


      synchronized (this) {
        // update these on successful RPC
        clusterNodeCount = allocateResponse.getNumClusterNodes();
        lastResponseId = allocateResponse.getResponseId();
        clusterAvailableResources = allocateResponse.getAvailableResources();
        if (!allocateResponse.getNMTokens().isEmpty()) {
          populateNMTokens(allocateResponse);
        }
      }
    } finally {
      // TODO how to differentiate remote yarn exception vs error in rpc

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse

com.alibaba.jstorm.yarn.MasterServer

com.cloudera.iterativereduce.yarn.ResourceManagerHandler

com.hortonworks.simpleyarnapp.ApplicationMaster

com.yahoo.storm.yarn.MasterServer

eu.stratosphere.yarn.ApplicationMaster

org.apache.flink.yarn.appMaster.ApplicationMaster

org.apache.giraph.yarn.GiraphApplicationMaster

org.apache.hadoop.mapreduce.v2.app.local.LocalContainerAllocator

org.apache.hadoop.mapreduce.v2.app.MRAppBenchmark

org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.