Package org.apache.hadoop.mapred.FairScheduler

Examples of org.apache.hadoop.mapred.FairScheduler.JobInfo

1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
    scheduler.getPoolManager().reloadAllocs();
   
    // Submit jobs, advancing time in-between to make sure that they are
    // all submitted at distinct times.
    JobInProgress job1 = submitJob(JobStatus.RUNNING, 10, 10);
    JobInfo info1 = scheduler.infos.get(job1);
    JobInProgress job2 = submitJob(JobStatus.RUNNING, 10, 10, "poolA");
    JobInfo info2 = scheduler.infos.get(job2);
    JobInProgress job3 = submitJob(JobStatus.RUNNING, 10, 10, "poolB");
    JobInfo info3 = scheduler.infos.get(job3);
    advanceTime(10);
   
    assertEquals(1.14,  info1.mapSchedulable.getFairShare(), 0.01);
    assertEquals(1.14,  info1.reduceSchedulable.getFairShare(), 0.01);
    assertEquals(2.28,  info2.mapSchedulable.getFairShare(), 0.01);
    assertEquals(2.28,  info2.reduceSchedulable.getFairShare(), 0.01);
    assertEquals(0.57,  info3.mapSchedulable.getFairShare(), 0.01);
    assertEquals(0.57,  info3.reduceSchedulable.getFairShare(), 0.01);
   
    JobInProgress job4 = submitJob(JobStatus.RUNNING, 10, 10, "poolB");
    JobInfo info4 = scheduler.infos.get(job4);
    advanceTime(10);
   
    assertEquals(1.14,  info1.mapSchedulable.getFairShare(), 0.01);
    assertEquals(1.14,  info1.reduceSchedulable.getFairShare(), 0.01);
    assertEquals(2.28,  info2.mapSchedulable.getFairShare(), 0.01);
View Full Code Here
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
    scheduler.getPoolManager().reloadAllocs();
   
    // Submit jobs, advancing time in-between to make sure that they are
    // all submitted at distinct times.
    JobInProgress job1 = submitJob(JobStatus.RUNNING, 0, 10, "poolA");
    JobInfo info1 = scheduler.infos.get(job1);
    JobInProgress job2 = submitJob(JobStatus.RUNNING, 0, 10, "poolA");
    JobInfo info2 = scheduler.infos.get(job2);
    JobInProgress job3 = submitJob(JobStatus.RUNNING, 10, 10, "poolB");
    JobInfo info3 = scheduler.infos.get(job3);
    advanceTime(10);
   
    /*
    assertEquals(0,     info1.mapWeight, 0.01);
    assertEquals(1.0,   info1.reduceWeight, 0.01);
View Full Code Here
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
   
    JobInProgress job1 = submitJob(JobStatus.RUNNING, 1, 0, "pool1",
        new String[][] {
          {"rack2.node2"}
        }, true);
    JobInfo info1 = scheduler.infos.get(job1);
   
    // Advance time before submitting another job j2, to make j1 be ahead
    // of j2 in the queue deterministically.
    advanceTime(100);
    JobInProgress job2 = submitJob(JobStatus.RUNNING, 10, 0);
View Full Code Here
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
   
    JobInProgress job1 = submitJob(JobStatus.RUNNING, 4, 0, "pool1",
        new String[][] {
          {"rack2.node2"}, {"rack2.node2"}, {"rack2.node2"}, {"rack2.node2"}
        }, true);
    JobInfo info1 = scheduler.infos.get(job1);
   
    // Advance time before submitting another job j2, to make j1 be ahead
    // of j2 in the queue deterministically.
    advanceTime(100);
    JobInProgress job2 = submitJob(JobStatus.RUNNING, 20, 0);
View Full Code Here
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
    JobInProgress job1 = submitJob(JobStatus.RUNNING, 8, 0, "pool1",
        new String[][] {
          {"rack2.node2"}, {"rack2.node2"}, {"rack2.node2"}, {"rack2.node2"},
          {"rack2.node2"}, {"rack2.node2"}, {"rack2.node2"}, {"rack2.node2"},
        }, true);
    JobInfo info1 = scheduler.infos.get(job1);
    advanceTime(100);
   
    // Check that nothing is assigned on trackers 1-3
    assertNull(scheduler.assignTasks(tracker("tt1")));
    assertNull(scheduler.assignTasks(tracker("tt2")));
View Full Code Here
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
    Pool defaultPool = scheduler.getPoolManager().getPool("default");

    // Submit job 1 and assign all slots to it. Sleep a bit before assigning
    // tasks on tt1 and tt2 to ensure that the ones on tt2 get preempted first.
    JobInProgress job1 = submitJob(JobStatus.RUNNING, 10, 10, "pool1");
    JobInfo info1 = scheduler.infos.get(job1);
    checkAssignment("tt1", "attempt_test_0001_m_000000_0 on tt1");
    checkAssignment("tt1", "attempt_test_0001_r_000000_0 on tt1");
    checkAssignment("tt1", "attempt_test_0001_m_000001_0 on tt1");
    checkAssignment("tt1", "attempt_test_0001_r_000001_0 on tt1");
    advanceTime(100);
View Full Code Here
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
    }
    assertEquals(scheduler.infos.size() * 2, byJobIdAndType.size());
   
    for (Map.Entry<JobInProgress, JobInfo> entry :
            scheduler.infos.entrySet()) {
      JobInfo info = entry.getValue();
      String jobId = entry.getKey().getJobID().toString();
     
      OutputRecord mapMetrics = byJobIdAndType.get(jobId + "_MAP");
      assertNotNull("Job " + jobId + " should have map metrics", mapMetrics);
      verifySchedulableMetrics(info.mapSchedulable, mapMetrics);
View Full Code Here
98
99
100
101
102
103
104
105
106
      }
    }
  }

  protected boolean isRunnable() {
    JobInfo info = scheduler.getJobInfo(job);
    int runState = job.getStatus().getRunState();
    return (info != null && info.runnable && runState == JobStatus.RUNNING);
  }
View Full Code Here
57
58
59
60
61
62
63
64
65
   
    initMetrics();
  }

  public void addJob(JobInProgress job) {
    JobInfo info = scheduler.getJobInfo(job);
    jobScheds.add(taskType == TaskType.MAP ?
        info.mapSchedulable : info.reduceSchedulable);
  }
View Full Code Here
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
   * We check that all tasks are assigned, but job 1 gets them first because it
   * was submitted earlier.
   */
  public void testSmallJobs() throws IOException {
    JobInProgress job1 = submitJob(JobStatus.RUNNING, 2, 1);
    JobInfo info1 = scheduler.infos.get(job1);
   
    // Check scheduler variables
    assertEquals(0,    info1.mapSchedulable.getRunningTasks());
    assertEquals(0,    info1.reduceSchedulable.getRunningTasks());
    assertEquals(2,    info1.mapSchedulable.getDemand());
    assertEquals(1,    info1.reduceSchedulable.getDemand());
    assertEquals(2.0,  info1.mapSchedulable.getFairShare());
    assertEquals(1.0,  info1.reduceSchedulable.getFairShare());
   
    // Advance time before submitting another job j2, to make j1 run before j2
    // deterministically.
    advanceTime(100);
    JobInProgress job2 = submitJob(JobStatus.RUNNING, 1, 2);
    JobInfo info2 = scheduler.infos.get(job2);
   
    // Check scheduler variables
    assertEquals(0,    info1.mapSchedulable.getRunningTasks());
    assertEquals(0,    info1.reduceSchedulable.getRunningTasks());
    assertEquals(2,    info1.mapSchedulable.getDemand());
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapred.FairScheduler.JobInfo

  • org.apache.hadoop.mapred.FairSchedulerServlet
  • org.apache.hadoop.mapred.JobSchedulable
  • org.apache.hadoop.mapred.PoolSchedulable
  • org.apache.hadoop.mapred.TestFairScheduler
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.