Package org.apache.hadoop.corona

Examples of org.apache.hadoop.corona.ResourceRequest

204
205
206
207
208
209
210
211
212
213
214
215
216
217
      ResourceTracker resourceTracker,
      SessionDriver sessionDriver,
      List<ResourceGrant> previousGrants)
    throws IOException, InterruptedException {
    LOG.info("Waiting for JT grant for " + jobId);
    ResourceRequest req = resourceTracker.newJobTrackerRequest();
    for (ResourceGrant prev: previousGrants) {
      LOG.info("Adding " + prev.getNodeName() + " to excluded hosts");
      req.addToExcludeHosts(prev.getAddress().getHost());
    }
    resourceTracker.recordRequest(req);
    List<ResourceRequest> newRequests = resourceTracker.getWantedResources();
    sessionDriver.requestResources(newRequests);
    final List<ResourceGrant> grants = new ArrayList<ResourceGrant>();
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
   
    TaskInProgress map0 = newMapTask(0);
    TaskInProgress map1 = newMapTask(1);
    TaskInProgress reduce0 = newReduceTask(0, 2);
    TaskInProgress reduce1 = newReduceTask(1, 2);
    ResourceRequest req = rt.newMapRequest(map0.getSplitLocations());
    rt.recordRequest(req);
    req = rt.newMapRequest(map1.getSplitLocations());
    rt.recordRequest(req);
    req = rt.newReduceRequest();
    rt.recordRequest(req);
View Full Code Here
127
128
129
130
131
132
133
134
135
136
137
    TaskInProgress map0 = newMapTask(0);
    TaskInProgress map1 = newMapTask(1);
    TaskInProgress reduce0 = newReduceTask(0, 2);
    TaskInProgress reduce1 = newReduceTask(1, 2);
   
    ResourceRequest req = rt.newMapRequest(map0.getSplitLocations());
    rt.recordRequest(req);
    req = rt.newMapRequest(map1.getSplitLocations());
    rt.recordRequest(req);
    req = rt.newReduceRequest();
    rt.recordRequest(req);
View Full Code Here
125
126
127
128
129
130
131
132
133
134
135
  public List<ResourceRequest> getWantedResources() {
    List<ResourceRequest> wanted = new ArrayList<ResourceRequest>();
    synchronized(lockObject) {
      for (Integer requestId:
        setDifference(requestMap.keySet(), requestedResources.keySet())) {
        ResourceRequest req = requestMap.get(requestId);
        LOG.info("Filing request for resource " + requestId);
        requestedResources.put(requestId, req);
        wanted.add(req);
      }
    }
View Full Code Here
145
146
147
148
149
150
151
152
153
154
155
156
157
158
    synchronized(lockObject) {
      for (Integer requestId:
        setDifference(requestedResources.keySet(), requestMap.keySet())) {
        // We update the data structures right away. This assumes that the
        // caller will be able to release the resources.
        ResourceRequest req = requestedResources.remove(requestId);
        if (req != null) {
          release.add(req);
          LOG.info("Filing release for requestId: " + req.getId());
        }
      }
    }
    return release;
  }
View Full Code Here
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
    return release;
  }

  public ResourceRequest releaseAndRequestResource(Integer grantIdToRelease, Set<String> excludedHosts) {
    synchronized(lockObject) {
      ResourceRequest requestToRelease = requestedResources.get(grantIdToRelease);
      if (requestToRelease != null) {
        removeRequestUnprotected(requestToRelease);
        ResourceRequest request = copyRequest(requestToRelease, excludedHosts);
        recordRequestUnprotected(request);
        LOG.info ("releaseAndRequest for grant: " + grantIdToRelease + " completed " +
            (excludedHosts != null ? "excluding resource" : "") +
            ". Generated new request #" + request.getId());
        return request;
      } else {
        LOG.info ("releaseAndRequest for grant: " + grantIdToRelease + " not found");
        return null;
      }
View Full Code Here
178
179
180
181
182
183
184
185
186
  /**
   * Release the resource that was requested
   */
  public void releaseResource(int resourceId) {
    synchronized (lockObject) {
      ResourceRequest req = requestedResources.get(resourceId);
      removeRequestUnprotected(req);
    }
  }
View Full Code Here
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
  }

  private ResourceRequest copyRequest(ResourceRequest requestToCopy,
      Set<String> excludedByTip) {
    int requestId = resourceRequestId.incrementAndGet();
    ResourceRequest req = new ResourceRequest(requestId, requestToCopy.getType());
    req.setSpecs(requestToCopy.getSpecs());

    Set<String> excluded = new HashSet<String>();
    if (excludedByTip != null || requestToCopy.getExcludeHosts() != null) {
      if (requestToCopy.getExcludeHosts() != null) {
        excluded.addAll(requestToCopy.getExcludeHosts());
      }
      if (excludedByTip != null) {
        excluded.addAll(excludedByTip);
      }
    }

    req.setExcludeHosts(new ArrayList<String>(excluded));
    if (requestToCopy.getHosts() != null) {
      List<String> hosts = new ArrayList<String>();
      for (String host : requestToCopy.getHosts()) {
        if (excluded == null || !excluded.contains(host)) {
          hosts.add(host);
        }
      }
      if (!hosts.isEmpty()) {
        req.setHosts(hosts);
      }
    }
    return req;
  }
View Full Code Here
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
    return resourceRequestId.incrementAndGet();
  }

  public ResourceRequest newMapRequest(String[] splitLocations) {
    int requestId = nextRequestId();
    ResourceRequest req = new ResourceRequest(requestId,
        ResourceType.MAP);
    req.setSpecs(stdMapSpec());

    List<String> hosts = new ArrayList<String>();
    for (int j = 0; j < splitLocations.length; j++) {
      hosts.add(splitLocations[j]);
    }
    if (!hosts.isEmpty()) {
      req.setHosts(hosts);
    }
    return req;
  }
View Full Code Here
371
372
373
374
375
376
377
378
379
380
    return req;
  }

  public ResourceRequest newReduceRequest() {
    int requestId = nextRequestId();
    ResourceRequest req = new ResourceRequest(requestId,
        ResourceType.REDUCE);
    req.setSpecs(stdReduceSpec());
    return req;
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.corona.ResourceRequest

  • org.apache.hadoop.mapred.CoronaJobTracker
  • org.apache.hadoop.mapred.CoronaJobTracker$ResourceUpdater
  • org.apache.hadoop.mapred.RemoteJTProxy
  • org.apache.hadoop.mapred.ResourceTracker
  • org.apache.hadoop.mapred.TestResourceTracker
  • java.util.BitSet
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.