Package org.apache.hadoop.corona

Examples of org.apache.hadoop.corona.ResourceRequest

379
380
381
382
383
384
385
386
387
388
    return req;
  }

  public ResourceRequest newJobTrackerRequest() {
    int requestId = nextRequestId();
    ResourceRequest req = new ResourceRequest(requestId,
        ResourceType.JOBTRACKER);
    req.setSpecs(stdReduceSpec());
    return req;
  }
View Full Code Here
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
        ResourceGrant resource = resourceTracker.getGrant(grant);
        String hostToExlcude = resource.getAddress().getHost();
        taskToContextMap.get(tip).excludedHosts.add(hostToExlcude);
        excludedHosts = taskToContextMap.get(tip).excludedHosts;
      }
      ResourceRequest newReq = resourceTracker.releaseAndRequestResource(grant,
          excludedHosts);
      requestToTipMap.put(newReq.getId(), tip);
      TaskContext context = taskToContextMap.get(tip);
      if (context == null) {
        context = new TaskContext(newReq);
      } else {
        context.resourceRequests.add(newReq);
View Full Code Here
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
  private void setupMapRequests(CoronaJobInProgress jip) {
    synchronized (lockObject) {
      TaskInProgress[] maps = jip.getTasks(TaskType.MAP);
      for (TaskInProgress map : maps) {
        ResourceRequest req =
          resourceTracker.newMapRequest(map.getSplitLocations());
        registerNewRequestForTip(map, req);
      }
    }
  }
View Full Code Here
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
    synchronized (lockObject) {
      if (jip.scheduleReducesUnprotected() && !jip.initializeReducers()) {

        TaskInProgress[] reduces = jip.getTasks(TaskType.REDUCE);
        for (TaskInProgress reduce : reduces) {
          ResourceRequest req = resourceTracker.newReduceRequest();
          registerNewRequestForTip(reduce, req);
        }
      }
    }
  }
View Full Code Here
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
        List<TaskInProgress> maps = job.getSpeculativeCandidates(TaskType.MAP);
        if (maps != null) {
          for (TaskInProgress tip : maps) {
            if (!speculatedMaps.contains(tip)) {
              // Speculate the tip
              ResourceRequest req =
                resourceTracker.newMapRequest(tip.getSplitLocations());
              excludeHostsUnprotected(req, tip);
              registerNewRequestForTip(tip, req);
            }
          }
          speculatedMaps.clear();
          speculatedMaps.addAll(maps);
        }
        List<TaskInProgress> reduces = job
            .getSpeculativeCandidates(TaskType.REDUCE);
        if (reduces != null) {
          for (TaskInProgress tip : reduces) {
            if (!speculatedReduces.contains(tip)) {
              // Speculate the tip
              ResourceRequest req = resourceTracker.newReduceRequest();
              excludeHostsUnprotected(req, tip);
              registerNewRequestForTip(tip, req);
            }
          }
          speculatedReduces.clear();
View Full Code Here
150
151
152
153
154
155
156
157
158
159
160
  public List<ResourceRequest> getWantedResources() {
    List<ResourceRequest> wanted = new ArrayList<ResourceRequest>();
    synchronized(lockObject) {
      for (Integer requestId:
        setDifference(requestMap.keySet(), requestedResources.keySet())) {
        ResourceRequest req = requestMap.get(requestId);
        LOG.info("Filing request for resource " + requestId);
        requestedResources.put(requestId, req);
        wanted.add(req);
      }
    }
View Full Code Here
170
171
172
173
174
175
176
177
178
179
180
181
182
183
    synchronized(lockObject) {
      for (Integer requestId:
        setDifference(requestedResources.keySet(), requestMap.keySet())) {
        // We update the data structures right away. This assumes that the
        // caller will be able to release the resources.
        ResourceRequest req = requestedResources.remove(requestId);
        if (req != null) {
          release.add(req);
          LOG.info("Filing release for requestId: " + req.getId());
        }
      }
    }
    return release;
  }
View Full Code Here
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
    return release;
  }

  public ResourceRequest releaseAndRequestResource(Integer grantIdToRelease, Set<String> excludedHosts) {
    synchronized(lockObject) {
      ResourceRequest requestToRelease = requestedResources.get(grantIdToRelease);
      if (requestToRelease != null) {
        removeRequestUnprotected(requestToRelease);
        ResourceRequest request = copyRequest(requestToRelease, excludedHosts);
        recordRequestUnprotected(request);
        LOG.info ("releaseAndRequest for grant: " + grantIdToRelease + " completed " +
            (excludedHosts != null ? "excluding resource" : "") +
            ". Generated new request #" + request.getId());
        return request;
      } else {
        LOG.info ("releaseAndRequest for grant: " + grantIdToRelease + " not found");
        return null;
      }
View Full Code Here
203
204
205
206
207
208
209
210
211
  /**
   * Release the resource that was requested
   */
  public void releaseResource(int resourceId) {
    synchronized (lockObject) {
      ResourceRequest req = requestedResources.get(resourceId);
      removeRequestUnprotected(req);
    }
  }
View Full Code Here
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
  }

  private ResourceRequest copyRequest(ResourceRequest requestToCopy,
      Set<String> excludedByTip) {
    int requestId = resourceRequestId.incrementAndGet();
    ResourceRequest req = new ResourceRequest(requestId, requestToCopy.getType());
    req.setSpecs(requestToCopy.getSpecs());

    Set<String> excluded = new HashSet<String>();
    if (excludedByTip != null || requestToCopy.getExcludeHosts() != null) {
      if (requestToCopy.getExcludeHosts() != null) {
        excluded.addAll(requestToCopy.getExcludeHosts());
      }
      if (excludedByTip != null) {
        excluded.addAll(excludedByTip);
      }
    }

    req.setExcludeHosts(new ArrayList<String>(excluded));
    if (requestToCopy.getHosts() != null) {
      List<String> hosts = new ArrayList<String>();
      for (String host : requestToCopy.getHosts()) {
        if (excluded == null || !excluded.contains(host)) {
          hosts.add(host);
        }
      }
      if (!hosts.isEmpty()) {
        req.setHosts(hosts);
      }
    }
    return req;
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.corona.ResourceRequest

  • org.apache.hadoop.mapred.CoronaJobTracker
  • org.apache.hadoop.mapred.CoronaJobTracker$ResourceUpdater
  • org.apache.hadoop.mapred.RemoteJTProxy
  • org.apache.hadoop.mapred.ResourceTracker
  • org.apache.hadoop.mapred.TestResourceTracker
  • java.util.BitSet
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.