Package org.apache.hadoop.yarn.server.resourcemanager.scheduler

Examples of org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApp


  }

  private synchronized void activateApplications() {
    for (Iterator<SchedulerApp> i=pendingApplications.iterator();
         i.hasNext(); ) {
      SchedulerApp application = i.next();
     
      // Check queue limit
      if (getNumActiveApplications() >= getMaximumActiveApplications()) {
        break;
      }
     
      // Check user limit
      User user = getUser(application.getUser());
      if (user.getActiveApplications() < getMaximumActiveApplicationsPerUser()) {
        user.activateApplication();
        activeApplications.add(application);
        i.remove();
        LOG.info("Application " + application.getApplicationId() +
            " from user: " + application.getUser() +
            " activated in queue: " + getQueueName());
      }
    }
  }
View Full Code Here


    }
   
    // Check for reserved resources
    RMContainer reservedContainer = node.getReservedContainer();
    if (reservedContainer != null) {
      SchedulerApp application =
          getApplication(reservedContainer.getApplicationAttemptId());
      return new CSAssignment(
          assignReservedContainer(application, node, reservedContainer,
              clusterResource),
          NodeType.NODE_LOCAL); // Don't care about locality constraints
                                // for reserved containers
    }
   
    // Try to assign containers to applications in order
    for (SchedulerApp application : activeApplications) {
     
      if(LOG.isDebugEnabled()) {
        LOG.debug("pre-assignContainers for application "
        + application.getApplicationId());
        application.showRequests();
      }

      synchronized (application) {
        // Schedule in priority order
        for (Priority priority : application.getPriorities()) {
          // Required resource
          Resource required =
              application.getResourceRequest(priority, RMNode.ANY).getCapability();

          // Do we need containers at this 'priority'?
          if (!needContainers(application, priority, required)) {
            continue;
          }

          // Compute user-limit & set headroom
          // Note: We compute both user-limit & headroom with the highest
          //       priority request as the target.
          //       This works since we never assign lower priority requests
          //       before all higher priority ones are serviced.
          Resource userLimit =
              computeUserLimitAndSetHeadroom(application, clusterResource,
                  required);         
         
          // Check queue max-capacity limit
          if (!assignToQueue(clusterResource, required)) {
            return NULL_ASSIGNMENT;
          }

          // Check user limit
          if (!assignToUser(application.getUser(), userLimit)) {
            break;
          }

          // Inform the application it is about to get a scheduling opportunity
          application.addSchedulingOpportunity(priority);
         
          // Try to schedule
          CSAssignment assignment = 
            assignContainersOnNode(clusterResource, node, application, priority,
                null);

          // Did we schedule or reserve a container?
          Resource assigned = assignment.getResource();
          if (Resources.greaterThan(assigned, Resources.none())) {

            // Book-keeping
            // Note: Update headroom to account for current allocation too...
            allocateResource(clusterResource, application, assigned);
           
            // Reset scheduling opportunities
            application.resetSchedulingOpportunities(priority);
           
            // Done
            return assignment;
          } else {
            // Do not assign out of order w.r.t priorities
            break;
          }
        }
      }

      if(LOG.isDebugEnabled()) {
        LOG.debug("post-assignContainers for application "
          + application.getApplicationId());
      }
      application.showRequests();
    }
 
    return NULL_ASSIGNMENT;

  }
View Full Code Here

      new Allocation(EMPTY_CONTAINER_LIST, Resources.createResource(0));
  @Override
  public Allocation allocate(
      ApplicationAttemptId applicationAttemptId, List<ResourceRequest> ask,
      List<ContainerId> release) {
    SchedulerApp application = getApplication(applicationAttemptId);
    if (application == null) {
      LOG.error("Calling allocate on removed " +
          "or non existant application " + applicationAttemptId);
      return EMPTY_ALLOCATION;
    }

    // Sanity check
    SchedulerUtils.normalizeRequests(ask, minimumAllocation.getMemory());

    // Release containers
    for (ContainerId releasedContainer : release) {
      RMContainer rmContainer = getRMContainer(releasedContainer);
      if (rmContainer == null) {
         RMAuditLogger.logFailure(application.getUser(),
             AuditConstants.RELEASE_CONTAINER,
             "Unauthorized access or invalid container", "FifoScheduler",
             "Trying to release container not owned by app or with invalid id",
             application.getApplicationId(), releasedContainer);
      }
      containerCompleted(rmContainer,
          SchedulerUtils.createAbnormalContainerStatus(
              releasedContainer,
              SchedulerUtils.RELEASED_CONTAINER),
          RMContainerEventType.RELEASED);
    }

    synchronized (application) {
      if (!ask.isEmpty()) {
        LOG.debug("allocate: pre-update" +
            " applicationId=" + applicationAttemptId +
            " application=" + application);
        application.showRequests();

        // Update application requests
        application.updateResourceRequests(ask);

        LOG.debug("allocate: post-update" +
            " applicationId=" + applicationAttemptId +
            " application=" + application);
        application.showRequests();

        LOG.debug("allocate:" +
            " applicationId=" + applicationAttemptId +
            " #ask=" + ask.size());
      }

      return new Allocation(
          application.pullNewlyAllocatedContainers(),
          application.getHeadroom());
    }
  }
View Full Code Here

  }

  @Override
  public SchedulerAppReport getSchedulerAppInfo(
      ApplicationAttemptId applicationAttemptId) {
    SchedulerApp app = getApplication(applicationAttemptId);
    return app == null ? null : new SchedulerAppReport(app);
  }
View Full Code Here

  }
 
  private synchronized void addApplication(ApplicationAttemptId appAttemptId,
      String user) {
    // TODO: Fix store
    SchedulerApp schedulerApp =
        new SchedulerApp(appAttemptId, user, DEFAULT_QUEUE, activeUsersManager,
            this.rmContext, null);
    applications.put(appAttemptId, schedulerApp);
    metrics.submitApp(user, appAttemptId.getAttemptId());
    LOG.info("Application Submission: " + appAttemptId.getApplicationId() +
        " from " + user + ", currently active: " + applications.size());
View Full Code Here

  private synchronized void doneApplication(
      ApplicationAttemptId applicationAttemptId,
      RMAppAttemptState rmAppAttemptFinalState)
      throws IOException {
    SchedulerApp application = getApplication(applicationAttemptId);
    if (application == null) {
      throw new IOException("Unknown application " + applicationAttemptId +
      " has completed!");
    }

    // Kill all 'live' containers
    for (RMContainer container : application.getLiveContainers()) {
      containerCompleted(container,
          SchedulerUtils.createAbnormalContainerStatus(
              container.getContainerId(),
              SchedulerUtils.COMPLETED_APPLICATION),
          RMContainerEventType.KILL);
    }

    // Inform the activeUsersManager
    synchronized (application) {
      activeUsersManager.deactivateApplication(
          application.getUser(), application.getApplicationId());
    }

    // Clean up pending requests, metrics etc.
    application.stop(rmAppAttemptFinalState);

    // Remove the application
    applications.remove(applicationAttemptId);
  }
View Full Code Here

        " #applications=" + applications.size());

    // Try to assign containers to applications in fifo order
    for (Map.Entry<ApplicationAttemptId, SchedulerApp> e : applications
        .entrySet()) {
      SchedulerApp application = e.getValue();
      LOG.debug("pre-assignContainers");
      application.showRequests();
      synchronized (application) {
        for (Priority priority : application.getPriorities()) {
          int maxContainers =
            getMaxAllocatableContainers(application, priority, node,
                NodeType.OFF_SWITCH);
          // Ensure the application needs containers of this priority
          if (maxContainers > 0) {
            int assignedContainers =
              assignContainersOnNode(node, application, priority);
            // Do not assign out of order w.r.t priorities
            if (assignedContainers == 0) {
              break;
            }
          }
        }
      }
     
      LOG.debug("post-assignContainers");
      application.showRequests();

      // Done
      if (Resources.lessThan(node.getAvailableResource(), minimumAllocation)) {
        break;
      }
    }

    // Update the applications' headroom to correctly take into
    // account the containers assigned in this update.
    for (SchedulerApp application : applications.values()) {
      application.setHeadroom(Resources.subtract(clusterResource, usedResource));
    }
  }
View Full Code Here

  }

  private void containerLaunchedOnNode(ContainerId containerId, SchedulerNode node) {
    // Get the application for the finished container
    ApplicationAttemptId applicationAttemptId = containerId.getApplicationAttemptId();
    SchedulerApp application = getApplication(applicationAttemptId);
    if (application == null) {
      LOG.info("Unknown application: " + applicationAttemptId +
          " launched container " + containerId +
          " on node: " + node);
      // Some unknown container sneaked into the system. Kill it.
      this.rmContext.getDispatcher().getEventHandler()
        .handle(new RMNodeCleanContainerEvent(node.getNodeID(), containerId));

      return;
    }
   
    application.containerLaunchedOnNode(containerId, node.getNodeID());
  }
View Full Code Here

    }

    // Get the application for the finished container
    Container container = rmContainer.getContainer();
    ApplicationAttemptId applicationAttemptId = container.getId().getApplicationAttemptId();
    SchedulerApp application = getApplication(applicationAttemptId);
   
    // Get the node on which the container was allocated
    SchedulerNode node = getNode(container.getNodeId());
   
    if (application == null) {
      LOG.info("Unknown application: " + applicationAttemptId +
          " released container " + container.getId() +
          " on node: " + node +
          " with event: " + event);
      return;
    }

    // Inform the application
    application.containerCompleted(rmContainer, containerStatus, event);

    // Inform the node
    node.releaseContainer(container);
   
    // Update total usage
View Full Code Here

    SchedulerNode node = getNode(nodeId);
    return node == null ? null : new SchedulerNodeReport(node);
  }
 
  private RMContainer getRMContainer(ContainerId containerId) {
    SchedulerApp application =
        getApplication(containerId.getApplicationAttemptId());
    return (application == null) ? null : application.getRMContainer(containerId);
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApp

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.