Package org.apache.hadoop.corona

Examples of org.apache.hadoop.corona.ComputeSpecs

227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
    int numCpu = resourceCalculatorPlugin.getNumProcessors();
    if (numCpu == ResourceCalculatorPlugin.UNAVAILABLE) {
      numCpu = 1;
    }
    int totalMemoryMB = (int) (resourceCalculatorPlugin.getPhysicalMemorySize() / 1024D / 1024);
    ComputeSpecs total = new ComputeSpecs((short)numCpu);
    total.setNetworkMBps((short)100);
    total.setMemoryMB(totalMemoryMB);
    total.setDiskGB(
       (int)(getDiskSpace(false) / 1024D / 1024 / 1024));
    String appInfo = getLocalHostname() + ":" + actionServerAddr.getPort();
    Map<ResourceType, String> resourceInfos =
        new EnumMap<ResourceType, String>(ResourceType.class);
    resourceInfos.put(ResourceType.MAP, appInfo);
    resourceInfos.put(ResourceType.REDUCE, appInfo);
    resourceInfos.put(ResourceType.JOBTRACKER, appInfo);

    while (running && !shuttingDown) {
      try {
        long now = System.currentTimeMillis();
        Thread.sleep(heartbeatCMInterval);

        float cpuUsage = resourceCalculatorPlugin.getCpuUsage();
        if (cpuUsage == ResourceCalculatorPlugin.UNAVAILABLE) {
          cpuUsage = 0;
        }
        ComputeSpecs free = new ComputeSpecs((short)(numCpu * cpuUsage / 100D));
        // TODO find free network.
        free.setNetworkMBps((short)100);
        int availableMemoryMB =
            (int)(resourceCalculatorPlugin.
                getAvailablePhysicalMemorySize() / 1024D / 1024);
        free.setMemoryMB(availableMemoryMB);
        free.setDiskGB((int)(getDiskSpace(true) / 1024D / 1024 / 1024));
        // TT puts it's MR specific host:port tuple here
        ClusterNodeInfo node = new ClusterNodeInfo
          (this.getName(), clusterManagerCallbackServerAddr, total);
        node.setFree(free);
        node.setResourceInfos(resourceInfos);
View Full Code Here
43
44
45
46
47
48
49
50
51
52
53
54
  AtomicInteger resourceRequestId = new AtomicInteger();

  static ComputeSpecs stdMapSpec() {
    short numCpus = 1;
    ComputeSpecs spec = new ComputeSpecs(numCpus);
    // Create Compute Spec.
    spec.setNetworkMBps((short) 10);
    spec.setMemoryMB(1024);
    spec.setDiskGB(10);
    return spec;
  }
View Full Code Here
53
54
55
56
57
58
59
60
61
62
63
    return spec;
  }

  static ComputeSpecs stdReduceSpec() {
    short numCpus = 1;
    ComputeSpecs spec = new ComputeSpecs(numCpus);
    spec.setNetworkMBps((short) 50);
    spec.setMemoryMB(1024);
    spec.setDiskGB(10);
    return spec;
  }
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
77
  /** Keeps last (possibly) assigned resource id */
  AtomicInteger resourceRequestId = new AtomicInteger(START_REQUEST_ID);

  static ComputeSpecs stdMapSpec() {
    short numCpus = 1;
    ComputeSpecs spec = new ComputeSpecs(numCpus);
    // Create Compute Spec.
    spec.setNetworkMBps((short) 10);
    spec.setMemoryMB(1024);
    spec.setDiskGB(10);
    return spec;
  }
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
    return spec;
  }

  static ComputeSpecs stdReduceSpec() {
    short numCpus = 1;
    ComputeSpecs spec = new ComputeSpecs(numCpus);
    spec.setNetworkMBps((short) 50);
    spec.setMemoryMB(1024);
    spec.setDiskGB(10);
    return spec;
  }
View Full Code Here
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
    if (numCpu == ResourceCalculatorPlugin.UNAVAILABLE) {
      numCpu = 1;
    }
    LOG.info("Will report " + numCpu + " CPUs");
    int totalMemoryMB = (int) (resourceCalculatorPlugin.getPhysicalMemorySize() / 1024D / 1024);
    ComputeSpecs total = new ComputeSpecs((short)numCpu);
    total.setNetworkMBps((short)100);
    total.setMemoryMB(totalMemoryMB);
    total.setDiskGB(
       (int)(getDiskSpace(false) / 1024D / 1024 / 1024));
    String appInfo = null;
    if (getLocalHostAddress() != null) {
      appInfo = getLocalHostAddress() + ":" + actionServerAddr.getPort();
    }
    else {
      appInfo = getLocalHostname() + ":" + actionServerAddr.getPort();
    }
    Map<ResourceType, String> resourceInfos =
        new EnumMap<ResourceType, String>(ResourceType.class);
    resourceInfos.put(ResourceType.MAP, appInfo);
    resourceInfos.put(ResourceType.REDUCE, appInfo);
    resourceInfos.put(ResourceType.JOBTRACKER, appInfo);

    while (running && !shuttingDown) {
      try {
        long now = System.currentTimeMillis();
        Thread.sleep(heartbeatCMInterval);

        float cpuUsage = resourceCalculatorPlugin.getCpuUsage();
        if (cpuUsage == ResourceCalculatorPlugin.UNAVAILABLE) {
          cpuUsage = 0;
        }
        ComputeSpecs free = new ComputeSpecs((short)(numCpu * cpuUsage / 100D));
        // TODO find free network.
        free.setNetworkMBps((short)100);
        int availableMemoryMB =
            (int)(resourceCalculatorPlugin.
                getAvailablePhysicalMemorySize() / 1024D / 1024);
        free.setMemoryMB(availableMemoryMB);
        long freeDiskSpace = getDiskSpace(true);
        long freeLogDiskSpace = getLogDiskFreeSpace();
        free.setDiskGB((int)(
          Math.min(freeDiskSpace, freeLogDiskSpace) / 1024D / 1024 / 1024));
        // TT puts it's MR specific host:port tuple here
        ClusterNodeInfo node = new ClusterNodeInfo
          (this.getName(), clusterManagerCallbackServerAddr, total);
        node.setFree(free);
View Full Code Here
TOP

Related Classes of org.apache.hadoop.corona.ComputeSpecs

  • org.apache.hadoop.mapred.CoronaTaskTracker
  • org.apache.hadoop.mapred.ResourceTracker
  • java.util.BitSet
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.