Examples of org.apache.hadoop.yarn.api.records.ResourceRequest$ResourceRequestComparator

Package org.apache.hadoop.yarn.api.records

Examples of org.apache.hadoop.yarn.api.records.ResourceRequest$ResourceRequestComparator

org.apache.hadoop.yarn.api.records.ResourceRequest
ResourceRequest represents the request made by an application to the ResourceManager to obtain various Container allocations.

It includes:
- {@link Priority} of the request.
- The name of the machine or rack on which the allocation is desired. A special value of * signifies that any host/rack is acceptable to the application.
- {@link Resource} required for each request.
- Number of containers, of above specifications, which are required by the application.
- A boolean relaxLocality flag, defaulting to true, which tells the ResourceManager if the application wants locality to be loose (i.e. allows fall-through to rack or any) or strict (i.e. specify hard constraint on resource allocation).
@see Resource @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)

            if (!ask.remove(req)) {
              foundAll = false;
              // if ask already sent to RM, we can try and overwrite it if possible.
              // send a new ask to RM with numContainers
              // specified for the blacklisted host to be 0.
              ResourceRequest zeroedRequest =
                  ResourceRequest.newInstance(req.getPriority(),
                    req.getResourceName(), req.getCapability(),
                    req.getNumContainers(), req.getRelaxLocality());


              zeroedRequest.setNumContainers(0);
              // to be sent to RM on next heartbeat
              addResourceRequestToAsk(zeroedRequest);
            }
          }
          // if all requests were still in ask queue

View Full Code Here

    Map<Resource, ResourceRequest> reqMap = remoteRequests.get(resourceName);
    if (reqMap == null) {
      reqMap = new HashMap<Resource, ResourceRequest>();
      remoteRequests.put(resourceName, reqMap);
    }
    ResourceRequest remoteRequest = reqMap.get(capability);
    if (remoteRequest == null) {
      remoteRequest = recordFactory.newRecordInstance(ResourceRequest.class);
      remoteRequest.setPriority(priority);
      remoteRequest.setResourceName(resourceName);
      remoteRequest.setCapability(capability);
      remoteRequest.setNumContainers(0);
      reqMap.put(capability, remoteRequest);
    }
    remoteRequest.setNumContainers(remoteRequest.getNumContainers() + 1);


    // Note this down for next interaction with ResourceManager
    addResourceRequestToAsk(remoteRequest);
    if (LOG.isDebugEnabled()) {
      LOG.debug("addResourceRequest:" + " applicationId="
          + applicationId.getId() + " priority=" + priority.getPriority()
          + " resourceName=" + resourceName + " numContainers="
          + remoteRequest.getNumContainers() + " #asks=" + ask.size());
    }
  }

View Full Code Here

        LOG.debug("Not decrementing resource as " + resourceName
            + " is not present in request table");
      }
      return;
    }
    ResourceRequest remoteRequest = reqMap.get(capability);


    if (LOG.isDebugEnabled()) {
      LOG.debug("BEFORE decResourceRequest:" + " applicationId="
          + applicationId.getId() + " priority=" + priority.getPriority()
          + " resourceName=" + resourceName + " numContainers="
          + remoteRequest.getNumContainers() + " #asks=" + ask.size());
    }


    if(remoteRequest.getNumContainers() > 0) {
      // based on blacklisting comments above we can end up decrementing more 
      // than requested. so guard for that.
      remoteRequest.setNumContainers(remoteRequest.getNumContainers() -1);
    }
    
    if (remoteRequest.getNumContainers() == 0) {
      reqMap.remove(capability);
      if (reqMap.size() == 0) {
        remoteRequests.remove(resourceName);
      }
      if (remoteRequests.size() == 0) {
        remoteRequestsTable.remove(priority);
      }
    }


    // send the updated resource request to RM
    // send 0 container count requests also to cancel previous requests
    addResourceRequestToAsk(remoteRequest);


    if (LOG.isDebugEnabled()) {
      LOG.info("AFTER decResourceRequest:" + " applicationId="
          + applicationId.getId() + " priority=" + priority.getPriority()
          + " resourceName=" + resourceName + " numContainers="
          + remoteRequest.getNumContainers() + " #asks=" + ask.size());
    }
  }

View Full Code Here

    return attId;
  }


  private ResourceRequest createResourceRequest(int memory, String host,
      int priority, int numContainers) {
    ResourceRequest request = recordFactory
        .newRecordInstance(ResourceRequest.class);
    request.setCapability(Resources.createResource(memory));
    request.setResourceName(host);
    request.setNumContainers(numContainers);
    Priority prio = recordFactory.newRecordInstance(Priority.class);
    prio.setPriority(priority);
    request.setPriority(prio);
    return request;
  }

View Full Code Here

    int memory = 64;
    int nConts = 3;
    int priority = 20;


    List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
    ResourceRequest nodeLocal = createResourceRequest(memory,
        node0.getHostName(), priority, nConts);
    ResourceRequest rackLocal = createResourceRequest(memory,
        node0.getRackName(), priority, nConts);
    ResourceRequest any = createResourceRequest(memory, ResourceRequest.ANY, priority,
        nConts);
    ask.add(nodeLocal);
    ask.add(rackLocal);
    ask.add(any);
    scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null);

View Full Code Here


    int memory = 1024;
    int priority = 1;


    List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
    ResourceRequest nodeLocal = createResourceRequest(memory,
        node0.getHostName(), priority, 1);
    ResourceRequest rackLocal = createResourceRequest(memory,
        node0.getRackName(), priority, 1);
    ResourceRequest any = createResourceRequest(memory, ResourceRequest.ANY, priority,
        1);
    ask.add(nodeLocal);
    ask.add(rackLocal);
    ask.add(any);
    scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null);

View Full Code Here

    TestUtils.callMethod("internalInit", allocator);


    List<ResourceRequest> createRequests = TestUtils.callMethod("createRequests", allocator);
    assertThat(createRequests, notNullValue());
    assertThat(createRequests.size(), is(1));
    ResourceRequest req = createRequests.get(0);
    assertThat(req.getPriority().getPriority(), is(0));
    assertThat(req.getNumContainers(), is(0));
    assertThat(req.getRelaxLocality(), is(true));
  }

View Full Code Here


    List<ResourceRequest> createRequests = TestUtils.callMethod("createRequests", allocator);
    assertThat(createRequests, notNullValue());
    assertThat(createRequests.size(), is(1));


    ResourceRequest req0 = createRequests.get(0);
    assertThat(req0.getResourceName(), is("*"));
    assertThat(req0.getPriority().getPriority(), is(0));
    assertThat(req0.getNumContainers(), is(2));
    assertThat(req0.getRelaxLocality(), is(true));
  }

View Full Code Here


    List<ResourceRequest> createRequests = TestUtils.callMethod("createRequests", allocator);
    assertThat(createRequests, notNullValue());
    assertThat(createRequests.size(), is(3));


    ResourceRequest req0 = createRequests.get(0);
    assertThat(req0.getResourceName(), is("host1"));
    assertThat(req0.getPriority().getPriority(), is(0));
    assertThat(req0.getNumContainers(), is(1));
    assertThat(req0.getRelaxLocality(), is(true));


    ResourceRequest req1 = createRequests.get(1);
    assertThat(req1.getResourceName(), is("/default-rack"));
    assertThat(req1.getPriority().getPriority(), is(0));
    assertThat(req1.getNumContainers(), is(1));
    assertThat(req1.getRelaxLocality(), is(false));


    ResourceRequest req2 = createRequests.get(2);
    assertThat(req2.getResourceName(), is("*"));
    assertThat(req2.getPriority().getPriority(), is(0));
    assertThat(req2.getNumContainers(), is(2));
    assertThat(req2.getRelaxLocality(), is(false));
  }

View Full Code Here


    List<ResourceRequest> createRequests = TestUtils.callMethod("createRequests", allocator);
    assertThat(createRequests, notNullValue());
    assertThat(createRequests.size(), is(2));


    ResourceRequest req0 = createRequests.get(0);
    assertThat(req0.getResourceName(), is("/default-rack"));
    assertThat(req0.getPriority().getPriority(), is(0));
    assertThat(req0.getNumContainers(), is(1));
    assertThat(req0.getRelaxLocality(), is(true));


    ResourceRequest req1 = createRequests.get(1);
    assertThat(req1.getResourceName(), is("*"));
    assertThat(req1.getPriority().getPriority(), is(0));
    assertThat(req1.getNumContainers(), is(1));
    assertThat(req1.getRelaxLocality(), is(false));
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.yarn.api.records.ResourceRequest$ResourceRequestComparator

com.cloudera.iterativereduce.Utils

com.cloudera.iterativereduce.yarn.appmaster.ApplicationMaster

org.apache.giraph.yarn.GiraphApplicationMaster

org.apache.hadoop.mapreduce.v2.app.rm.RMContainerRequestor

org.apache.hadoop.mapreduce.v2.app.rm.TestRMContainerAllocator$MyFifoScheduler

org.apache.hadoop.mapreduce.v2.app.TestRMContainerAllocator$MyFifoScheduler

org.apache.hadoop.yarn.applications.distributedshell.ApplicationMaster

org.apache.hadoop.yarn.client.AMRMClientImpl

org.apache.hadoop.yarn.client.api.impl.TestAMRMClient

org.apache.hadoop.yarn.client.api.impl.TestAMRMClientContainerRequest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.