Examples of org.apache.hadoop.yarn.api.records.Resource

Package org.apache.hadoop.yarn.api.records

Examples of org.apache.hadoop.yarn.api.records.Resource

org.apache.hadoop.yarn.api.records.Resource

Resource models a set of computer resources in the cluster.

Currrently it only models memory.

Typically, applications request Resource of suitable capability to run their component tasks.
@see ResourceRequest @see AMRMProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)

    } else {
      taskId = MRBuilderUtils.newTaskId(jobId, 0, TaskType.MAP);
    }
    TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId,
        taskAttemptId);
    Resource containerNeed = Resource.newInstance(memory, 1);
    if (earlierFailedAttempt) {
      return ContainerRequestEvent
          .createContainerRequestEventForFailedContainer(attemptId,
              containerNeed);
    }

View Full Code Here

    acls.put(ApplicationAccessType.VIEW_APP, viewACL.getAclString());
    acls.put(ApplicationAccessType.MODIFY_APP, modifyACL.getAclString());


    ContainerLaunchContext amContainer = recordFactory
        .newRecordInstance(ContainerLaunchContext.class);
    Resource resource = BuilderUtils.newResource(1024, 1);
    context.setResource(resource);
    amContainer.setApplicationACLs(acls);
    context.setAMContainerSpec(amContainer);
    submitRequest.setApplicationSubmissionContext(context);
    rmClient.submitApplication(submitRequest);

View Full Code Here

      public void handle(ContainerAllocatorEvent event) {
        ContainerId cId =
            ContainerId.newInstance(getContext().getApplicationAttemptId(),
              containerCount++);
        NodeId nodeId = NodeId.newInstance(NM_HOST, NM_PORT);
        Resource resource = Resource.newInstance(1234, 2);
        ContainerTokenIdentifier containerTokenIdentifier =
            new ContainerTokenIdentifier(cId, nodeId.toString(), "user",
            resource, System.currentTimeMillis() + 10000, 42, 42,
            Priority.newInstance(0), 0);
        Token containerToken = newContainerToken(nodeId, "password".getBytes(),

View Full Code Here

    ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(appId, 0);
    when(app.getApplicationId()).thenReturn(appId);
    when(app.getApplicationAttemptId()).thenReturn(appAttId);


    int cAlloc = 0;
    Resource unit = Resource.newInstance(gran, 0);
    List<RMContainer> cReserved = new ArrayList<RMContainer>();
    for (int i = 0; i < reserved; i += gran) {
      cReserved.add(mockContainer(appAttId, cAlloc, unit, 1));
      ++cAlloc;
    }

View Full Code Here


    ApplicationClientProtocol submitterClient = getRMClientForUser(submitter);
    ApplicationId applicationId =
        submitterClient.getNewApplication(newAppRequest).getApplicationId();


    Resource resource = BuilderUtils.newResource(1024, 1);
    Map<ApplicationAccessType, String> acls = createACLs(submitter, setupACLs);
    ContainerLaunchContext amContainerSpec =
        ContainerLaunchContext.newInstance(null, null, null, null, null, acls);


    ApplicationSubmissionContext appSubmissionContext =

View Full Code Here

    return getRunningNode(null);
  }


  private RMNodeImpl getRunningNode(String nmVersion) {
    NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
    Resource capability = Resource.newInstance(4096, 4);
    RMNodeImpl node = new RMNodeImpl(nodeId, rmContext,null, 0, 0,
        null, ResourceOption.newInstance(capability,
            RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT), nmVersion);
    node.handle(new RMNodeStartedEvent(node.getNodeID(), null, null));
    Assert.assertEquals(NodeState.RUNNING, node.getState());

View Full Code Here

    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
        appId, 1);
    ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1);
    ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class);


    Resource resource = BuilderUtils.newResource(512, 1);
    Priority priority = BuilderUtils.newPriority(5);


    Container container = BuilderUtils.newContainer(containerId, nodeId,
        "host:3465", resource, priority, null);

View Full Code Here

    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
        appId, 1);
    ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1);
    ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class);


    Resource resource = BuilderUtils.newResource(512, 1);
    Priority priority = BuilderUtils.newPriority(5);


    Container container = BuilderUtils.newContainer(containerId, nodeId,
        "host:3465", resource, priority, null);

View Full Code Here

    when(csContext.getQueueComparator()).
        thenReturn(CapacityScheduler.queueComparator);
    when(csContext.getResourceCalculator()).thenReturn(resourceCalculator);
    
    // Say cluster has 100 nodes of 16G each
    Resource clusterResource = Resources.createResource(100 * 16 * GB, 100 * 16);
    when(csContext.getClusterResource()).thenReturn(clusterResource);
    
    Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
    CSQueue root = 
        CapacityScheduler.parseQueue(csContext, csConf, null, "root", 
            queues, queues, TestUtils.spyHook);


    LeafQueue queue = (LeafQueue)queues.get(A);
    
    LOG.info("Queue 'A' -" +
        " maxActiveApplications=" + queue.getMaximumActiveApplications() + 
        " maxActiveApplicationsPerUser=" + 
        queue.getMaximumActiveApplicationsPerUser());
    int expectedMaxActiveApps = 
        Math.max(1, 
            (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) * 
                   csConf.
                     getMaximumApplicationMasterResourcePerQueuePercent(
                                                        queue.getQueuePath()) *
                   queue.getAbsoluteMaximumCapacity()));
    assertEquals(expectedMaxActiveApps, 
                 queue.getMaximumActiveApplications());
    int expectedMaxActiveAppsUsingAbsCap = 
            Math.max(1, 
                (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) * 
                       csConf.getMaximumApplicationMasterResourcePercent() *
                       queue.getAbsoluteCapacity()));
    assertEquals(
        (int)Math.ceil(
            expectedMaxActiveAppsUsingAbsCap * (queue.getUserLimit() / 100.0f) * 
            queue.getUserLimitFactor()), 
        queue.getMaximumActiveApplicationsPerUser());
    assertEquals(
        (int)(clusterResource.getMemory() * queue.getAbsoluteCapacity()),
        queue.getMetrics().getAvailableMB()
        );
    
    // Add some nodes to the cluster & test new limits
    clusterResource = Resources.createResource(120 * 16 * GB);
    root.updateClusterResource(clusterResource);
    expectedMaxActiveApps = 
        Math.max(1, 
            (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) * 
                   csConf.
                     getMaximumApplicationMasterResourcePerQueuePercent(
                                                        queue.getQueuePath()) *
                   queue.getAbsoluteMaximumCapacity()));
    assertEquals(expectedMaxActiveApps, 
                 queue.getMaximumActiveApplications());
    expectedMaxActiveAppsUsingAbsCap = 
            Math.max(1, 
                (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) * 
                       csConf.getMaximumApplicationMasterResourcePercent() *
                       queue.getAbsoluteCapacity()));
    assertEquals(
        (int)Math.ceil(expectedMaxActiveAppsUsingAbsCap * 
            (queue.getUserLimit() / 100.0f) * queue.getUserLimitFactor()), 
        queue.getMaximumActiveApplicationsPerUser());
    assertEquals(
        (int)(clusterResource.getMemory() * queue.getAbsoluteCapacity()),
        queue.getMetrics().getAvailableMB()
        );


    // should return -1 if per queue setting not set
    assertEquals(
        (int)CapacitySchedulerConfiguration.UNDEFINED, 
        csConf.getMaximumApplicationsPerQueue(queue.getQueuePath()));
    int expectedMaxApps =  
        (int)
        (CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS * 
        queue.getAbsoluteCapacity());
    assertEquals(expectedMaxApps, queue.getMaxApplications());


    int expectedMaxAppsPerUser = (int)(expectedMaxApps *
        (queue.getUserLimit()/100.0f) * queue.getUserLimitFactor());
    assertEquals(expectedMaxAppsPerUser, queue.getMaxApplicationsPerUser());


    // should default to global setting if per queue setting not set
    assertEquals(
        (long)CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT, 
        (long)csConf.getMaximumApplicationMasterResourcePerQueuePercent(
            queue.getQueuePath())
            );


    // Change the per-queue max AM resources percentage.
    csConf.setFloat(
      "yarn.scheduler.capacity." + 
          queue.getQueuePath() + 
          ".maximum-am-resource-percent",
      0.5f);
    // Re-create queues to get new configs.
    queues = new HashMap<String, CSQueue>();
    root = 
        CapacityScheduler.parseQueue(csContext, csConf, null, "root", 
            queues, queues, TestUtils.spyHook);
    clusterResource = Resources.createResource(100 * 16 * GB);


    queue = (LeafQueue)queues.get(A);
    expectedMaxActiveApps = 
        Math.max(1, 
            (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) * 
                   csConf.
                     getMaximumApplicationMasterResourcePerQueuePercent(
                                                        queue.getQueuePath()) *
                   queue.getAbsoluteMaximumCapacity()));

View Full Code Here

    when(csContext.getQueueComparator()).
        thenReturn(CapacityScheduler.queueComparator);
    when(csContext.getResourceCalculator()).thenReturn(resourceCalculator);
    
    // Say cluster has 100 nodes of 16G each
    Resource clusterResource = Resources.createResource(100 * 16 * GB);
    when(csContext.getClusterResource()).thenReturn(clusterResource);
    
    Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
    CapacityScheduler.parseQueue(csContext, csConf, null, "root", 
        queues, queues, TestUtils.spyHook);


    // Manipulate queue 'a'
    LeafQueue queue = TestLeafQueue.stubLeafQueue((LeafQueue)queues.get(A));
    
    String host_0 = "host_0";
    String rack_0 = "rack_0";
    FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, rack_0, 0, 16*GB);


    final String user_0 = "user_0";
    final String user_1 = "user_1";
    
    RecordFactory recordFactory = 
        RecordFactoryProvider.getRecordFactory(null);
    RMContext rmContext = TestUtils.getMockRMContext();


    Priority priority_1 = TestUtils.createMockPriority(1);


    // Submit first application with some resource-requests from user_0, 
    // and check headroom
    final ApplicationAttemptId appAttemptId_0_0 = 
        TestUtils.getMockApplicationAttemptId(0, 0); 
    FiCaSchedulerApp app_0_0 = 
        spy(new FiCaSchedulerApp(appAttemptId_0_0, user_0, queue, 
            queue.getActiveUsersManager(), rmContext));
    queue.submitApplicationAttempt(app_0_0, user_0);


    List<ResourceRequest> app_0_0_requests = new ArrayList<ResourceRequest>();
    app_0_0_requests.add(
        TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2,
            true, priority_1, recordFactory));
    app_0_0.updateResourceRequests(app_0_0_requests);


    // Schedule to compute 
    queue.assignContainers(clusterResource, node_0);
    Resource expectedHeadroom = Resources.createResource(10*16*GB, 1);
    verify(app_0_0).setHeadroom(eq(expectedHeadroom));


    // Submit second application from user_0, check headroom
    final ApplicationAttemptId appAttemptId_0_1 = 
        TestUtils.getMockApplicationAttemptId(1, 0);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.yarn.api.records.Resource

com.alibaba.jstorm.yarn.JStormOnYarn

com.hortonworks.simpleyarnapp.ApplicationMaster

com.hortonworks.simpleyarnapp.Client

eu.stratosphere.yarn.Client

org.apache.flink.yarn.Client

org.apache.giraph.yarn.GiraphApplicationMaster

org.apache.hadoop.hive.ql.exec.tez.DagUtils

org.apache.hadoop.mapred.YARNRunner

org.apache.hadoop.mapreduce.TestTypeConverter

org.apache.hadoop.mapreduce.v2.app.job.impl.TestTaskAttempt

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.