Examples of org.apache.hadoop.mapreduce.ClusterMetrics

org.apache.hadoop.mapreduce.ClusterMetrics
Status information on the current state of the Map-Reduce cluster.
ClusterMetrics provides clients with information such as:
1. Size of the cluster.
2. Number of blacklisted and decommissioned trackers.
3. Slot capacity of the cluster.
4. The number of currently occupied/reserved map & reduce slots.
5. The number of currently running map & reduce tasks.
6. The number of job submissions.

  public String getSummaryJson() {
    return getSummary().toJson();
  }


  InfoMap getSummary() {
    final ClusterMetrics metrics = getClusterMetrics();
    InfoMap map = new InfoMap();
    map.put("nodes", metrics.getTaskTrackerCount()
            + getBlacklistedTrackerCount());
    map.put("alive", metrics.getTaskTrackerCount());
    map.put("blacklisted", getBlacklistedTrackerCount());
    map.put("graylisted", getGraylistedTrackerCount());
    map.put("slots", new InfoMap() {{
      put("map_slots", metrics.getMapSlotCapacity());
      put("map_slots_used", metrics.getOccupiedMapSlots());
      put("reduce_slots", metrics.getReduceSlotCapacity());
      put("reduce_slots_used", metrics.getOccupiedReduceSlots());
    }});
    map.put("jobs", metrics.getTotalJobSubmissions());
    return map;
  }

View Full Code Here

    
    // create TaskTrackerStatus and send heartbeats
    sendHeartbeats(list);


    // assert ClusterMetrics
    ClusterMetrics metrics = jobTracker.getClusterMetrics();
    assertEquals("occupied map slots do not match", mapSlotsPerTask,
      metrics.getOccupiedMapSlots());
    assertEquals("occupied reduce slots do not match", reduceSlotsPerTask,
      metrics.getOccupiedReduceSlots());
    assertEquals("map slot capacities do not match",
      mapSlotsPerTracker * trackers.length,
      metrics.getMapSlotCapacity());
    assertEquals("reduce slot capacities do not match",
      reduceSlotsPerTracker * trackers.length,
      metrics.getReduceSlotCapacity());
    assertEquals("running map tasks do not match", 1,
      metrics.getRunningMaps());
    assertEquals("running reduce tasks do not match", 1,
      metrics.getRunningReduces());
    
    // assert the values in ClusterStatus also
    ClusterStatus stat = client.getClusterStatus();
    assertEquals("running map tasks do not match", 1,
      stat.getMapTasks());
    assertEquals("running reduce tasks do not match", 1,
      stat.getReduceTasks());
    assertEquals("map slot capacities do not match",
      mapSlotsPerTracker * trackers.length,
      stat.getMaxMapTasks());
    assertEquals("reduce slot capacities do not match",
      reduceSlotsPerTracker * trackers.length,
      stat.getMaxReduceTasks());
    
    // send a heartbeat finishing only a map and check
    // counts are updated.
    list.clear();
    addMapTaskAttemptToList(list, mapSlotsPerTask, TaskStatus.State.SUCCEEDED);
    addReduceTaskAttemptToList(list, 
        reduceSlotsPerTask, TaskStatus.State.RUNNING);
    sendHeartbeats(list);
    metrics = jobTracker.getClusterMetrics();
    assertEquals(0, metrics.getOccupiedMapSlots());
    assertEquals(reduceSlotsPerTask, metrics.getOccupiedReduceSlots());
    
    // send a heartbeat finishing the reduce task also.
    list.clear();
    addReduceTaskAttemptToList(list, 
        reduceSlotsPerTask, TaskStatus.State.SUCCEEDED);
    sendHeartbeats(list);
    metrics = jobTracker.getClusterMetrics();
    assertEquals(0, metrics.getOccupiedReduceSlots());
  }

View Full Code Here

                    jobTracker);
    
    sendHeartBeat(jobTracker, status1, false, true, trackers[0], responseId);
    sendHeartBeat(jobTracker, status2, false, true, trackers[1], responseId);
    responseId++; 
    ClusterMetrics metrics = jobTracker.getClusterMetrics();
    assertEquals("reserved map slots do not match", 
      2, metrics.getReservedMapSlots());
    assertEquals("reserved reduce slots do not match", 
      2, metrics.getReservedReduceSlots());


    // redo to test re-reservations.
    sendHeartBeat(jobTracker, status1, false, true, trackers[0], responseId);
    sendHeartBeat(jobTracker, status2, false, true, trackers[1], responseId);
    responseId++; 
    metrics = jobTracker.getClusterMetrics();
    assertEquals("reserved map slots do not match", 
        4, metrics.getReservedMapSlots());
    assertEquals("reserved reduce slots do not match", 
        4, metrics.getReservedReduceSlots());


    // undo reservations now.
    scheduler.setUnreserveSlots(true);
    sendHeartBeat(jobTracker, status1, false, true, trackers[0], responseId);
    sendHeartBeat(jobTracker, status2, false, true, trackers[1], responseId);
    responseId++;
    metrics = jobTracker.getClusterMetrics();
    assertEquals("map slots should have been unreserved",
        0, metrics.getReservedMapSlots());
    assertEquals("reduce slots should have been unreserved",
        0, metrics.getReservedReduceSlots());
  }

View Full Code Here

    return fs.getUri().toString();
  }
  
  public ClusterMetrics getClusterMetrics() {
    int numMapTasks = map_tasks.get();
    return new ClusterMetrics(numMapTasks, reduce_tasks, numMapTasks,
        reduce_tasks, 0, 0, 1, 1, jobs.size(), 1, 0, 0);
  }

View Full Code Here

   */
  public ClusterStatus getClusterStatus() throws IOException {
    try {
      return clientUgi.doAs(new PrivilegedExceptionAction<ClusterStatus>() {
        public ClusterStatus run()  throws IOException, InterruptedException {
          ClusterMetrics metrics = cluster.getClusterStatus();
          return new ClusterStatus(metrics.getTaskTrackerCount(),
              metrics.getBlackListedTaskTrackerCount(), cluster.getTaskTrackerExpiryInterval(),
              metrics.getOccupiedMapSlots(),
              metrics.getOccupiedReduceSlots(), metrics.getMapSlotCapacity(),
              metrics.getReduceSlotCapacity(),
              cluster.getJobTrackerStatus(),
              metrics.getDecommissionedTaskTrackerCount());
        }
      });
    }
      catch (InterruptedException ie) {
      throw new IOException(ie);

View Full Code Here

   */
  public ClusterStatus getClusterStatus(boolean detailed) throws IOException {
    try {
      return clientUgi.doAs(new PrivilegedExceptionAction<ClusterStatus>() {
        public ClusterStatus run() throws IOException, InterruptedException {
        ClusterMetrics metrics = cluster.getClusterStatus();
        return new ClusterStatus(arrayToStringList(cluster.getActiveTaskTrackers()),
          arrayToBlackListInfo(cluster.getBlackListedTaskTrackers()),
          cluster.getTaskTrackerExpiryInterval(), metrics.getOccupiedMapSlots(),
          metrics.getOccupiedReduceSlots(), metrics.getMapSlotCapacity(),
          metrics.getReduceSlotCapacity(), 
          cluster.getJobTrackerStatus());
        }
      });
    } catch (InterruptedException ie) {
      throw new IOException(ie);

View Full Code Here

  public ClusterMetrics getClusterMetrics() throws IOException,
      InterruptedException {
    GetClusterMetricsRequest request = recordFactory.newRecordInstance(GetClusterMetricsRequest.class);
    GetClusterMetricsResponse response = applicationsManager.getClusterMetrics(request);
    YarnClusterMetrics metrics = response.getClusterMetrics();
    ClusterMetrics oldMetrics = new ClusterMetrics(1, 1, 1, 1, 1, 1, 
        metrics.getNumNodeManagers() * 10, metrics.getNumNodeManagers() * 2, 1,
        metrics.getNumNodeManagers(), 0, 0);
    return oldMetrics;
  }

View Full Code Here

  }


  public ClusterMetrics getClusterMetrics() throws IOException,
      InterruptedException {
    YarnClusterMetrics metrics = super.getYarnClusterMetrics();
    ClusterMetrics oldMetrics = new ClusterMetrics(1, 1, 1, 1, 1, 1, 
        metrics.getNumNodeManagers() * 10, metrics.getNumNodeManagers() * 2, 1,
        metrics.getNumNodeManagers(), 0, 0);
    return oldMetrics;
  }

View Full Code Here

   */
  public ClusterStatus getClusterStatus() throws IOException {
    try {
      return clientUgi.doAs(new PrivilegedExceptionAction<ClusterStatus>() {
        public ClusterStatus run()  throws IOException, InterruptedException {
          ClusterMetrics metrics = cluster.getClusterStatus();
          return new ClusterStatus(metrics.getTaskTrackerCount(),
              metrics.getBlackListedTaskTrackerCount(), cluster.getTaskTrackerExpiryInterval(),
              metrics.getOccupiedMapSlots(),
              metrics.getOccupiedReduceSlots(), metrics.getMapSlotCapacity(),
              metrics.getReduceSlotCapacity(),
              cluster.getJobTrackerStatus(),
              metrics.getDecommissionedTaskTrackerCount());
        }
      });
    }
      catch (InterruptedException ie) {
      throw new IOException(ie);

View Full Code Here

   */
  public ClusterStatus getClusterStatus(boolean detailed) throws IOException {
    try {
      return clientUgi.doAs(new PrivilegedExceptionAction<ClusterStatus>() {
        public ClusterStatus run() throws IOException, InterruptedException {
        ClusterMetrics metrics = cluster.getClusterStatus();
        return new ClusterStatus(arrayToStringList(cluster.getActiveTaskTrackers()),
          arrayToBlackListInfo(cluster.getBlackListedTaskTrackers()),
          cluster.getTaskTrackerExpiryInterval(), metrics.getOccupiedMapSlots(),
          metrics.getOccupiedReduceSlots(), metrics.getMapSlotCapacity(),
          metrics.getReduceSlotCapacity(), 
          cluster.getJobTrackerStatus());
        }
      });
    } catch (InterruptedException ie) {
      throw new IOException(ie);

View Full Code Here

0 1 2 3 4 5 6 7 8

TOP

Related Classes of org.apache.hadoop.mapreduce.ClusterMetrics

org.apache.hadoop.examples.pi.DistSum$MixMachine

org.apache.hadoop.mapred.gridmix.Statistics$StatCollector

org.apache.hadoop.mapred.gridmix.StressJobFactory

org.apache.hadoop.mapred.JobClient

org.apache.hadoop.mapred.JobTracker

org.apache.hadoop.mapred.LocalJobRunner

org.apache.hadoop.mapred.LocalJobRunnerWithFix

org.apache.hadoop.mapred.ResourceMgrDelegate

org.apache.hadoop.mapred.SimulatorJobClient

org.apache.hadoop.mapred.TestClusterStatus

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.