Examples of org.apache.hadoop.corona.ResourceRequest

org.apache.hadoop.corona.ResourceRequest

    return resourceRequestId.incrementAndGet();
  }


  public ResourceRequest newMapRequest(String[] splitLocations) {
    int requestId = nextRequestId();
    ResourceRequest req = new ResourceRequest(requestId,
        ResourceType.MAP);
    req.setSpecs(stdMapSpec());


    List<String> hosts = new ArrayList<String>();
    for (int j = 0; j < splitLocations.length; j++) {
      hosts.add(splitLocations[j]);
    }
    if (!hosts.isEmpty()) {
      req.setHosts(hosts);
    }
    return req;
  }

View Full Code Here

    return req;
  }


  public ResourceRequest newReduceRequest() {
    int requestId = nextRequestId();
    ResourceRequest req = new ResourceRequest(requestId,
        ResourceType.REDUCE);
    req.setSpecs(stdReduceSpec());
    return req;
  }

View Full Code Here

    return req;
  }


  public ResourceRequest newJobTrackerRequest() {
    int requestId = nextRequestId();
    ResourceRequest req = new ResourceRequest(requestId,
        ResourceType.JOBTRACKER);
    req.setSpecs(stdReduceSpec());
    return req;
  }

View Full Code Here

      ResourceTracker resourceTracker,
      SessionDriver sessionDriver,
      List<ResourceGrant> previousGrants)
    throws IOException, InterruptedException {
    LOG.info("Waiting for JT grant for " + attemptJobId);
    ResourceRequest req = resourceTracker.newJobTrackerRequest();
    for (ResourceGrant prev: previousGrants) {
      LOG.info("Adding " + prev.getNodeName() + " to excluded hosts");
      req.addToExcludeHosts(prev.getAddress().getHost());
    }
    resourceTracker.recordRequest(req);
    List<ResourceRequest> newRequests = resourceTracker.getWantedResources();
    sessionDriver.requestResources(newRequests);
    final List<ResourceGrant> grants = new ArrayList<ResourceGrant>();

View Full Code Here

        ResourceGrant resource = resourceTracker.getGrant(grant);
        String hostToExlcude = resource.getAddress().getHost();
        taskToContextMap.get(tip).excludedHosts.add(hostToExlcude);
        excludedHosts = taskToContextMap.get(tip).excludedHosts;
      }
      ResourceRequest newReq = resourceTracker.releaseAndRequestResource(grant,
          excludedHosts);
      requestToTipMap.put(newReq.getId(), tip);
      TaskContext context = taskToContextMap.get(tip);
      if (context == null) {
        context = new TaskContext(newReq);
      } else {
        context.resourceRequests.add(newReq);

View Full Code Here

  private void setupMapRequests(CoronaJobInProgress jip) {
    synchronized (lockObject) {
      TaskInProgress[] maps = jip.getTasks(TaskType.MAP);
      for (TaskInProgress map : maps) {
        if (!map.isComplete()) {
          ResourceRequest req = resourceTracker.newMapRequest(map.getSplitLocations());
          registerNewRequestForTip(map, req);
        }
      }
    }
  }

View Full Code Here

    synchronized (lockObject) {
      if (jip.scheduleReducesUnprotected() && !jip.initializeReducers()) {
        TaskInProgress[] reduces = jip.getTasks(TaskType.REDUCE);
        for (TaskInProgress reduce : reduces) {
          if (!reduce.isComplete()) {
            ResourceRequest req = resourceTracker.newReduceRequest();
            registerNewRequestForTip(reduce, req);
          }
        }


      }

View Full Code Here

        List<TaskInProgress> maps = job.getSpeculativeCandidates(TaskType.MAP);
        if (maps != null) {
          for (TaskInProgress tip : maps) {
            if (!speculatedMaps.contains(tip)) {
              // Speculate the tip
              ResourceRequest req =
                resourceTracker.newMapRequest(tip.getSplitLocations());
              excludeHostsUnprotected(req, tip);
              registerNewRequestForTip(tip, req);
            }
          }
          speculatedMaps.clear();
          speculatedMaps.addAll(maps);
        }
        List<TaskInProgress> reduces = job
            .getSpeculativeCandidates(TaskType.REDUCE);
        if (reduces != null) {
          for (TaskInProgress tip : reduces) {
            if (!speculatedReduces.contains(tip)) {
              // Speculate the tip
              ResourceRequest req = resourceTracker.newReduceRequest();
              excludeHostsUnprotected(req, tip);
              registerNewRequestForTip(tip, req);
            }
          }
          speculatedReduces.clear();

View Full Code Here

    
    TaskInProgress map0 = newMapTask(0);
    TaskInProgress map1 = newMapTask(1);
    TaskInProgress reduce0 = newReduceTask(0, 2);
    TaskInProgress reduce1 = newReduceTask(1, 2);
    ResourceRequest req = rt.newMapRequest(map0.getSplitLocations());
    rt.recordRequest(req);
    req = rt.newMapRequest(map1.getSplitLocations());
    rt.recordRequest(req);
    req = rt.newReduceRequest();
    rt.recordRequest(req);

View Full Code Here

    TaskInProgress map0 = newMapTask(0);
    TaskInProgress map1 = newMapTask(1);
    TaskInProgress reduce0 = newReduceTask(0, 2);
    TaskInProgress reduce1 = newReduceTask(1, 2);
    
    ResourceRequest req = rt.newMapRequest(map0.getSplitLocations());
    rt.recordRequest(req);
    req = rt.newMapRequest(map1.getSplitLocations());
    rt.recordRequest(req);
    req = rt.newReduceRequest();
    rt.recordRequest(req);

View Full Code Here

0 1 2

TOP

Related Classes of org.apache.hadoop.corona.ResourceRequest

org.apache.hadoop.mapred.CoronaJobTracker

org.apache.hadoop.mapred.CoronaJobTracker$ResourceUpdater

org.apache.hadoop.mapred.RemoteJTProxy

org.apache.hadoop.mapred.ResourceTracker

org.apache.hadoop.mapred.TestResourceTracker

java.util.BitSet

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.