Package org.apache.hadoop.mapred

Examples of org.apache.hadoop.mapred.TaskTrackerStatus$ResourceStatus

438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
  public List<Task> assignTasks(TaskTracker tt) {
    synchronized (jobTracker) {     
      try {
        Collection<JobInProgress> jobs = jobTracker.jobs.values();

        TaskTrackerStatus tts = tt.getStatus();
        String host = tts.getHost();
        LOG.info("In FrameworkScheduler.assignTasks for " + host);
       
        TaskTrackerInfo ttInfo = ttInfos.get(host);
        if (ttInfo == null) {
          LOG.error("No TaskTrackerInfo for " + host + "! This shouldn't happen.");
          return null;
        }
        ttInfo.maxMaps = tts.getMaxMapSlots();
        ttInfo.maxReduces = tts.getMaxReduceSlots();
       
        int clusterSize = jobTracker.getClusterStatus().getTaskTrackers();
        int numHosts = jobTracker.getNumberOfUniqueHosts();
       
        // Assigned tasks
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
67
    Assert.assertTrue("Interval cannot be zero.",interval != 0);
    UtilsForTests.waitFor(interval+2000);
    String defaultHealthScript = conf.get("mapred.healthChecker.script.path");   
    Assert.assertTrue("Task tracker is not healthy",
        nodeHealthStatus(client, true) == true);
    TaskTrackerStatus status = client.getStatus();
    JTClient jclient = cluster.getJTClient();
    Assert.assertTrue("Failed to move task tracker to healthy list",
        jclient.getProxy().isBlackListed(status.getTrackerName()) == false);       
    Assert.assertTrue("Health script was not set",defaultHealthScript != null);
   
  }
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
91
92
    Assert.assertTrue("Interval cannot be zero.",interval != 0);
    UtilsForTests.waitFor(interval+2000);
    //TaskTrackerStatus status = client.getStatus();
    Assert.assertTrue("Task tracker was never blacklisted ",
        nodeHealthStatus(client, false) == true);
    TaskTrackerStatus status = client.getStatus();
    Assert.assertTrue("The custom error message did not appear",
        status.getHealthStatus().getHealthReport().trim().
        equals(errorMessage));
    JTClient jClient = cluster.getJTClient();   
    Assert.assertTrue("Failed to move task tracker to blacklisted list",
        jClient.getProxy().isBlackListed(status.getTrackerName()) == true);   
  }
View Full Code Here
99
100
101
102
103
104
105
106
107
108
109
110
111
112
   * @return status of task tracker
   * @throws IOException failed to get the status of task tracker
   */
  public boolean nodeHealthStatus(TTClient client,boolean hStatus) throws IOException {
    int counter = 0;
    TaskTrackerStatus status = client.getStatus();
    while (counter < 60) {
      LOG.info("isNodeHealthy "+status.getHealthStatus().isNodeHealthy());
      if (status.getHealthStatus().isNodeHealthy() == hStatus) {
        break;
      } else {
        UtilsForTests.waitFor(3000);
        status = client.getStatus();
        Assert.assertNotNull("Task tracker status is null",status);
View Full Code Here
41
42
43
44
45
46
47
48
49
50
51
52
    Assert.assertTrue("Interval cannot be zero.",interval != 0);
    UtilsForTests.waitFor(interval+2000);
    String defaultHealthScript = conf.get("mapred.healthChecker.script.path");   
    Assert.assertTrue("Task tracker is not healthy",
        nodeHealthStatus(client, true) == true);
    TaskTrackerStatus status = client.getStatus();
    JTClient jclient = cluster.getJTClient();
    Assert.assertTrue("Failed to move task tracker to healthy list",
        jclient.getProxy().isBlackListed(status.getTrackerName()) == false);       
    Assert.assertTrue("Health script was not set",defaultHealthScript != null);
   
  }
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
76
77
    Assert.assertTrue("Interval cannot be zero.",interval != 0);
    UtilsForTests.waitFor(interval+2000);
    //TaskTrackerStatus status = client.getStatus();
    Assert.assertTrue("Task tracker was never blacklisted ",
        nodeHealthStatus(client, false) == true);
    TaskTrackerStatus status = client.getStatus();
    Assert.assertTrue("The custom error message did not appear",
        status.getHealthStatus().getHealthReport().trim().
        equals(errorMessage));
    JTClient jClient = cluster.getJTClient();   
    Assert.assertTrue("Failed to move task tracker to blacklisted list",
        jClient.getProxy().isBlackListed(status.getTrackerName()) == true);   
  }
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
95
96
97
   * @return status of task tracker
   * @throws IOException failed to get the status of task tracker
   */
  public boolean nodeHealthStatus(TTClient client,boolean hStatus) throws IOException {
    int counter = 0;
    TaskTrackerStatus status = client.getStatus();
    while (counter < 60) {
      LOG.info("isNodeHealthy "+status.getHealthStatus().isNodeHealthy());
      if (status.getHealthStatus().isNodeHealthy() == hStatus) {
        break;
      } else {
        UtilsForTests.waitFor(3000);
        status = client.getStatus();
        Assert.assertNotNull("Task tracker status is null",status);
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
    }
   
    @Override
    public List<Task> assignTasks(TaskTracker taskTracker)
        throws IOException {
      TaskTrackerStatus status = taskTracker.getStatus();
      long totalVirtualMemoryOnTT =
          getConf().getLong("totalVmemOnTT", -1);
      long totalPhysicalMemoryOnTT =
          getConf().getLong("totalPmemOnTT", -1);
      long mapSlotMemorySize =
          getConf().getLong("mapSlotMemorySize", -1);
      long reduceSlotMemorySize =
          getConf()
              .getLong("reduceSlotMemorySize", -1);
      long availableVirtualMemoryOnTT =
          getConf().getLong("availableVmemOnTT", -1);
      long availablePhysicalMemoryOnTT =
          getConf().getLong("availablePmemOnTT", -1);
      long cumulativeCpuTime =
          getConf().getLong("cumulativeCpuTime", -1);
      long cpuFrequency =
          getConf().getLong("cpuFrequency", -1);
      int numProcessors =
          getConf().getInt("numProcessors", -1);
      float cpuUsage =
          getConf().getFloat("cpuUsage", -1);

      long reportedTotalVirtualMemoryOnTT =
          status.getResourceStatus().getTotalVirtualMemory();
      long reportedTotalPhysicalMemoryOnTT =
          status.getResourceStatus().getTotalPhysicalMemory();
      long reportedMapSlotMemorySize =
          status.getResourceStatus().getMapSlotMemorySizeOnTT();
      long reportedReduceSlotMemorySize =
          status.getResourceStatus().getReduceSlotMemorySizeOnTT();
      long reportedAvailableVirtualMemoryOnTT =
          status.getResourceStatus().getAvailableVirtualMemory();
      long reportedAvailablePhysicalMemoryOnTT =
          status.getResourceStatus().getAvailablePhysicalMemory();
      long reportedCumulativeCpuTime =
          status.getResourceStatus().getCumulativeCpuTime();
      long reportedCpuFrequency = status.getResourceStatus().getCpuFrequency();
      int reportedNumProcessors = status.getResourceStatus().getNumProcessors();
      float reportedCpuUsage = status.getResourceStatus().getCpuUsage();

      message =
          "expected memory values : "
              + "(totalVirtualMemoryOnTT, totalPhysicalMemoryOnTT, "
              + "availableVirtualMemoryOnTT, availablePhysicalMemoryOnTT, "
              + "mapSlotMemSize, reduceSlotMemorySize, cumulativeCpuTime, "
              + "cpuFrequency, numProcessors, cpuUsage) = ("
              + totalVirtualMemoryOnTT + ", "
              + totalPhysicalMemoryOnTT + ","
              + availableVirtualMemoryOnTT + ", "
              + availablePhysicalMemoryOnTT + ","
              + mapSlotMemorySize + ","
              + reduceSlotMemorySize + ","
              + cumulativeCpuTime + ","
              + cpuFrequency + ","
              + numProcessors + ","
              + cpuUsage
              +")";
      message +=
          "\nreported memory values : "
              + "(totalVirtualMemoryOnTT, totalPhysicalMemoryOnTT, "
              + "availableVirtualMemoryOnTT, availablePhysicalMemoryOnTT, "
              + "reportedMapSlotMemorySize, reportedReduceSlotMemorySize, "
              + "reportedCumulativeCpuTime, reportedCpuFrequency, "
              + "reportedNumProcessors, cpuUsage) = ("
              + reportedTotalVirtualMemoryOnTT + ", "
              + reportedTotalPhysicalMemoryOnTT + ","
              + reportedAvailableVirtualMemoryOnTT + ", "
              + reportedAvailablePhysicalMemoryOnTT + ","
              + reportedMapSlotMemorySize + ","
              + reportedReduceSlotMemorySize + ","
              + reportedCumulativeCpuTime + ","
              + reportedCpuFrequency + ","
              + reportedNumProcessors + ","
              + reportedCpuUsage
               + ")";
      LOG.info(message);
      hasDynamicValuePassed = true;
      // Check task resource status in task reports
      for (TaskStatus taskStatus : status.getTaskReports()) {
        Counters counters = taskStatus.getCounters();
        // This should be zero because the initial CPU time is subtracted.
        long procCumulativeCpuTime = 0;
        long procVirtualMemorySize =
          getConf().getLong("procVirtualMemorySize", -1);
View Full Code Here
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
  public List<Task> assignTasks(TaskTracker tt) {
    synchronized (jobTracker) {
      try {
        Collection<JobInProgress> jobs = jobTracker.jobs.values();

        TaskTrackerStatus tts = tt.getStatus();
        String host = tts.getHost();

        TaskTrackerInfo ttInfo = ttInfos.get(host);
        if (ttInfo == null) {
          throw new RuntimeException(
              "Expecting TaskTrackerInfo for host " + host);
        }

        ttInfo.maxMaps = tts.getMaxMapSlots();
        ttInfo.maxReduces = tts.getMaxReduceSlots();

        int clusterSize = jobTracker.getClusterStatus().getTaskTrackers();
        int numHosts = jobTracker.getNumberOfUniqueHosts();

        // Assigned tasks
View Full Code Here
36
37
38
39
40
41
42
  private String taskTrackerName;
  private TaskTrackerStatus status;

  public TTInfoImpl() {
    taskTrackerName = "";
    status = new TaskTrackerStatus();
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapred.TaskTrackerStatus$ResourceStatus

  • org.apache.hadoop.mapred.FrameworkScheduler
  • org.apache.hadoop.mapred.HealthScriptHelper
  • org.apache.hadoop.mapred.TestTTResourceReporting$FakeTaskScheduler
  • org.apache.hadoop.mapred.TTInfoImpl
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.