Package org.apache.hadoop.util

Examples of org.apache.hadoop.util.LinuxResourceCalculatorPlugin

311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
    if (!System.getProperty("os.name").startsWith("Linux")) {
      return;
    }

    JobConf conf = new JobConf();
    LinuxResourceCalculatorPlugin plugin = new LinuxResourceCalculatorPlugin();
    // In this case, we only check these four fields because they are static
    conf.setLong("totalVmemOnTT", plugin.getVirtualMemorySize());
    conf.setLong("totalPmemOnTT", plugin.getPhysicalMemorySize());
    conf.setLong("numProcessors", plugin.getNumProcessors());

    try {
      setUpCluster(conf);
      runSleepJob(miniMRCluster.createJobConf());
      verifyTestResults(true);
View Full Code Here
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
    if (!System.getProperty("os.name").startsWith("Linux")) {
      return;
    }

    JobConf conf = new JobConf();
    LinuxResourceCalculatorPlugin plugin = new LinuxResourceCalculatorPlugin();
    // In this case, we only check these three fields because they are static
    conf.setLong("totalVmemOnTT", plugin.getVirtualMemorySize());
    conf.setLong("totalPmemOnTT", plugin.getPhysicalMemorySize());
    conf.setLong("numProcessors", plugin.getNumProcessors());

    try {
      setUpCluster(conf);
      runSleepJob(miniMRCluster.createJobConf());
      verifyTestResults(true);
View Full Code Here
415
416
417
418
419
420
421
422
423
424
425
426
427
    // very small value, so that no task escapes to successful completion.
    fConf.set(TaskMemoryManagerThread.TT_MEMORY_MANAGER_MONITORING_INTERVAL,
        String.valueOf(300));
   
    // reserve all memory on TT so that the job will exceed memory limits
    LinuxResourceCalculatorPlugin memoryCalculatorPlugin =
            new LinuxResourceCalculatorPlugin();
    long totalPhysicalMemory = memoryCalculatorPlugin.getPhysicalMemorySize();
    long reservedPhysicalMemory = totalPhysicalMemory / (1024 * 1024) + 1;
    fConf.setLong(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY, 1024 * 1024L);
    fConf.setLong(JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY, 1024 * 1024L);
    fConf.setLong(TaskMemoryManagerThread.TT_RESERVED_PHYSICAL_MEMORY_MB,
                  reservedPhysicalMemory);
View Full Code Here
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
    if (!System.getProperty("os.name").startsWith("Linux")) {
      return;
    }

    JobConf conf = new JobConf();
    LinuxResourceCalculatorPlugin plugin = new LinuxResourceCalculatorPlugin();
    // In this case, we only check these three fields because they are static
    conf.setLong("totalVmemOnTT", plugin.getVirtualMemorySize());
    conf.setLong("totalPmemOnTT", plugin.getPhysicalMemorySize());
    conf.setLong("numProcessors", plugin.getNumProcessors());

    try {
      setUpCluster(conf);
      runSleepJob(miniMRCluster.createJobConf());
      verifyTestResults(true);
View Full Code Here
415
416
417
418
419
420
421
422
423
424
425
426
427
    // very small value, so that no task escapes to successful completion.
    fConf.set(TaskMemoryManagerThread.TT_MEMORY_MANAGER_MONITORING_INTERVAL,
        String.valueOf(300));
   
    // reserve all memory on TT so that the job will exceed memory limits
    LinuxResourceCalculatorPlugin memoryCalculatorPlugin =
            new LinuxResourceCalculatorPlugin();
    long totalPhysicalMemory = memoryCalculatorPlugin.getPhysicalMemorySize();
    long reservedPhysicalMemory = totalPhysicalMemory / (1024 * 1024) + 1;
    fConf.setLong(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY, 1024 * 1024L);
    fConf.setLong(JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY, 1024 * 1024L);
    fConf.setLong(TaskMemoryManagerThread.TT_RESERVED_PHYSICAL_MEMORY_MB,
                  reservedPhysicalMemory);
View Full Code Here
TOP

Related Classes of org.apache.hadoop.util.LinuxResourceCalculatorPlugin

  • org.apache.hadoop.mapred.TestTaskTrackerMemoryManager
  • org.apache.hadoop.mapred.TestTTResourceReporting
  • java.io.FileReader
  • java.io.BufferedReader
  • java.util.regex.Matcher
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.