Examples of org.apache.hadoop.yarn.api.records.Resource

Package org.apache.hadoop.yarn.api.records

Examples of org.apache.hadoop.yarn.api.records.Resource

org.apache.hadoop.yarn.api.records.Resource

Resource models a set of computer resources in the cluster.

Currrently it only models memory.

Typically, applications request Resource of suitable capability to run their component tasks.
@see ResourceRequest @see AMRMProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)

    application_0.submit();
    
    application_0.addNodeManager(host_0, 1234, nm_0);
    application_0.addNodeManager(host_1, 1234, nm_1);


    Resource capability_0_0 = Resources.createResource(GB);
    application_0.addResourceRequestSpec(priority_1, capability_0_0);
    
    Resource capability_0_1 = Resources.createResource(2 * GB);
    application_0.addResourceRequestSpec(priority_0, capability_0_1);


    Task task_0_0 = new Task(application_0, priority_1, 
        new String[] {host_0, host_1});
    application_0.addTask(task_0_0);
       
    // Submit another application
    Application application_1 = new Application("user_1", resourceManager);
    application_1.submit();
    
    application_1.addNodeManager(host_0, 1234, nm_0);
    application_1.addNodeManager(host_1, 1234, nm_1);
    
    Resource capability_1_0 = Resources.createResource(3 * GB);
    application_1.addResourceRequestSpec(priority_1, capability_1_0);
    
    Resource capability_1_1 = Resources.createResource(4 * GB);
    application_1.addResourceRequestSpec(priority_0, capability_1_1);


    Task task_1_0 = new Task(application_1, priority_1, 
        new String[] {host_0, host_1});
    application_1.addTask(task_1_0);

View Full Code Here


  private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId,
      String name, String queue) {
    ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class);


    Resource resource = Resources.createResource(
        YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);


    ApplicationSubmissionContext submissionContext =
        recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
    submissionContext.setAMContainerSpec(amContainerSpec);

View Full Code Here

      sub.setQueue(queue);
    }
    sub.setApplicationType(appType);
    ContainerLaunchContext clc = Records
        .newRecord(ContainerLaunchContext.class);
    final Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(masterMemory);
    sub.setResource(capability);
    clc.setApplicationACLs(acls);
    if (ts != null && UserGroupInformation.isSecurityEnabled()) {
      DataOutputBuffer dob = new DataOutputBuffer();
      ts.writeTokenStorageToStream(dob);

View Full Code Here

    application_0.submit();
    
    application_0.addNodeManager(host_0, 1234, nm_0);
    application_0.addNodeManager(host_1, 1234, nm_1);


    Resource capability_0_0 = Resources.createResource(1 * GB, 1);
    application_0.addResourceRequestSpec(priority_1, capability_0_0);
    
    Resource capability_0_1 = Resources.createResource(2 * GB, 1);
    application_0.addResourceRequestSpec(priority_0, capability_0_1);


    Task task_0_0 = new Task(application_0, priority_1, 
        new String[] {host_0, host_1});
    application_0.addTask(task_0_0);
       
    // Submit another application
    Application application_1 = new Application("user_1", "b2", resourceManager);
    application_1.submit();
    
    application_1.addNodeManager(host_0, 1234, nm_0);
    application_1.addNodeManager(host_1, 1234, nm_1);
    
    Resource capability_1_0 = Resources.createResource(3 * GB, 1);
    application_1.addResourceRequestSpec(priority_1, capability_1_0);
    
    Resource capability_1_1 = Resources.createResource(2 * GB, 1);
    application_1.addResourceRequestSpec(priority_0, capability_1_1);


    Task task_1_0 = new Task(application_1, priority_1, 
        new String[] {host_0, host_1});
    application_1.addTask(task_1_0);

View Full Code Here

            ReflectionUtils.invoke(SET_VIRTUAL_CORES, resource, YarnUtils.minVCores(cfg, vCores));
        }
    }


    public static Resource resource(Configuration cfg, int memory, int cores) {
        Resource resource = Records.newRecord(Resource.class);
        resource.setMemory(YarnUtils.minMemory(cfg, memory));
        setVirtualCores(cfg, resource, cores);
        return resource;
    }

View Full Code Here

        nmRpc.start();


        log.info(String.format("Allocating Elasticsearch cluster with %d nodes", appConfig.containersToAllocate()));


        // register requests
        Resource capability = YarnCompat.resource(cfg, appConfig.containerMem(), appConfig.containerVCores());
        Priority prio = Priority.newInstance(appConfig.amPriority());


        for (int i = 0; i < appConfig.containersToAllocate(); i++) {
            // TODO: Add allocation (host/rack rules) - and disable location constraints
            ContainerRequest req = new ContainerRequest(capability, null, null, prio);

View Full Code Here


      int numRequest = subQuery.getContext().getResourceAllocator().calculateNumRequestContainers(
          subQuery.getContext().getQueryMasterContext().getWorkerContext(), tasks.length
      );


      final Resource resource = Records.newRecord(Resource.class);


      resource.setMemory(2000);


      LOG.info("Request Container for " + subQuery.getId() + " containers=" + numRequest);


      Priority priority = Records.newRecord(Priority.class);
      priority.setPriority(subQuery.getPriority());

View Full Code Here

    TaskSplitMetaInfo splits = mock(TaskSplitMetaInfo.class);
    when(splits.getLocations()).thenReturn(new String[] {"127.0.0.1"});


    AppContext appCtx = mock(AppContext.class);
    ClusterInfo clusterInfo = mock(ClusterInfo.class);
    Resource resource = mock(Resource.class);
    when(appCtx.getClusterInfo()).thenReturn(clusterInfo);
    when(resource.getMemory()).thenReturn(1024);


    TaskAttemptImpl taImpl =
      new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
          splits, jobConf, taListener,
          new Token(), new Credentials(),

View Full Code Here

    TaskSplitMetaInfo splits = mock(TaskSplitMetaInfo.class);
    when(splits.getLocations()).thenReturn(new String[] {});


    AppContext appCtx = mock(AppContext.class);
    ClusterInfo clusterInfo = mock(ClusterInfo.class);
    Resource resource = mock(Resource.class);
    when(appCtx.getClusterInfo()).thenReturn(clusterInfo);
    when(resource.getMemory()).thenReturn(1024);


    TaskAttemptImpl taImpl =
      new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
          splits, jobConf, taListener,
          new Token(), new Credentials(),

View Full Code Here

    TaskSplitMetaInfo splits = mock(TaskSplitMetaInfo.class);
    when(splits.getLocations()).thenReturn(new String[] {"127.0.0.1"});


    AppContext appCtx = mock(AppContext.class);
    ClusterInfo clusterInfo = mock(ClusterInfo.class);
    Resource resource = mock(Resource.class);
    when(appCtx.getClusterInfo()).thenReturn(clusterInfo);
    when(resource.getMemory()).thenReturn(1024);


    TaskAttemptImpl taImpl =
      new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
          splits, jobConf, taListener,
          new Token(), new Credentials(),

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.yarn.api.records.Resource

com.alibaba.jstorm.yarn.JStormOnYarn

com.hortonworks.simpleyarnapp.ApplicationMaster

com.hortonworks.simpleyarnapp.Client

eu.stratosphere.yarn.Client

org.apache.flink.yarn.Client

org.apache.giraph.yarn.GiraphApplicationMaster

org.apache.hadoop.hive.ql.exec.tez.DagUtils

org.apache.hadoop.mapred.YARNRunner

org.apache.hadoop.mapreduce.TestTypeConverter

org.apache.hadoop.mapreduce.v2.app.job.impl.TestTaskAttempt

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.