* We try to use some light-weight mechanism to determine cluster load.
* @return Whether, from job client perspective, the cluster is overloaded.
*/
private boolean isOverloaded(long now) throws IOException {
try {
ClusterMetrics clusterMetrics = jobTracker.getClusterMetrics();
// If there are more jobs than number of task trackers, we assume the
// cluster is overloaded. This is to bound the memory usage of the
// simulator job tracker, in situations where we have jobs with small
// number of map tasks and large number of reduce tasks.
if (runningJobs.size() >= clusterMetrics.getTaskTrackerCount()) {
System.out.printf("%d Overloaded is %s: " +
"#runningJobs >= taskTrackerCount (%d >= %d)\n",
now, Boolean.TRUE.toString(),
runningJobs.size(), clusterMetrics.getTaskTrackerCount());
return true;
}
float incompleteMapTasks = 0; // include pending & running map tasks.
for (Map.Entry<JobID, JobSketchInfo> entry : runningJobs.entrySet()) {
org.apache.hadoop.mapreduce.JobStatus jobStatus = jobTracker
.getJobStatus(entry.getKey());
incompleteMapTasks += (1 - Math.min(jobStatus.getMapProgress(), 1.0))
* entry.getValue().numMaps;
}
boolean overloaded = incompleteMapTasks >
OVERLAOD_MAPTASK_MAPSLOT_RATIO * clusterMetrics.getMapSlotCapacity();
String relOp = (overloaded) ? ">" : "<=";
System.out.printf("%d Overloaded is %s: "
+ "incompleteMapTasks %s %.1f*mapSlotCapacity (%.1f %s %.1f*%d)\n",
now, Boolean.toString(overloaded), relOp, OVERLAOD_MAPTASK_MAPSLOT_RATIO,
incompleteMapTasks, relOp, OVERLAOD_MAPTASK_MAPSLOT_RATIO,
clusterMetrics.getMapSlotCapacity());
return overloaded;
} catch (InterruptedException e) {
throw new IOException("InterruptedException", e);
}
}