Examples of org.apache.hadoop.mapreduce.ClusterMetrics

org.apache.hadoop.mapreduce.ClusterMetrics
Status information on the current state of the Map-Reduce cluster.
ClusterMetrics provides clients with information such as:
1. Size of the cluster.
2. Number of blacklisted and decommissioned trackers.
3. Slot capacity of the cluster.
4. The number of currently occupied/reserved map & reduce slots.
5. The number of currently running map & reduce tasks.
6. The number of job submissions.

   *         of {@link ClusterStatus}.
   * @throws IOException
   */
  public ClusterStatus getClusterStatus(boolean detailed) throws IOException {
    try {
      ClusterMetrics metrics = cluster.getClusterStatus();
      return new ClusterStatus(arrayToStringList(cluster.getActiveTaskTrackers()),
        arrayToBlackListInfo(cluster.getBlackListedTaskTrackers()),
        cluster.getTaskTrackerExpiryInterval(), metrics.getOccupiedMapSlots(),
        metrics.getOccupiedReduceSlots(), metrics.getMapSlotCapacity(),
        metrics.getReduceSlotCapacity(), 
        JobTracker.State.valueOf(cluster.getJobTrackerState().name()));
    } catch (InterruptedException ie) {
      throw new IOException(ie);
    }
  }

View Full Code Here

    return fs.getUri().toString();
  }
  
  public ClusterMetrics getClusterMetrics() {
    int numMapTasks = map_tasks.get();
    return new ClusterMetrics(numMapTasks, reduce_tasks, numMapTasks,
        reduce_tasks, 0, 0, 1, 1, jobs.size(), 1, 0, 0);
  }

View Full Code Here

    mrCluster.startTaskTracker(null, null, 0, 1);
    // Now wait for the job to fail.
    j.waitForCompletion(false);
    Assert.assertFalse("Job successfully completed.", j.isSuccessful());


    ClusterMetrics metrics = cluster.getClusterStatus();
    // validate number of slots in JobTracker
    Assert.assertEquals(0, metrics.getOccupiedMapSlots());
    Assert.assertEquals(0, metrics.getOccupiedReduceSlots());


    // validate number of slots in TaskTracker
    TaskTracker tt = mrCluster.getTaskTrackerRunner(0).getTaskTracker();
    Assert.assertEquals(metrics.getMapSlotCapacity(), tt.getFreeSlots(true));
    Assert.assertEquals(metrics.getReduceSlotCapacity(), tt.getFreeSlots(false));


  }

View Full Code Here

    
    // create TaskTrackerStatus and send heartbeats
    sendHeartbeats(list);
    
    // assert ClusterMetrics
    ClusterMetrics metrics = cluster.getClusterStatus();
    assertEquals("occupied map slots do not match", mapSlotsPerTask,
      metrics.getOccupiedMapSlots());
    assertEquals("occupied reduce slots do not match", reduceSlotsPerTask,
      metrics.getOccupiedReduceSlots());
    assertEquals("map slot capacities do not match",
      mapSlotsPerTracker * trackers.length,
      metrics.getMapSlotCapacity());
    assertEquals("reduce slot capacities do not match",
      reduceSlotsPerTracker * trackers.length,
      metrics.getReduceSlotCapacity());
    assertEquals("running map tasks do not match", 1,
      metrics.getRunningMaps());
    assertEquals("running reduce tasks do not match", 1,
      metrics.getRunningReduces());
    
    // assert the values in ClusterStatus also
    assertEquals("running map tasks do not match", 1,
      jobTracker.getClusterStatus().getMapTasks());
    assertEquals("running reduce tasks do not match", 1,
      jobTracker.getClusterStatus().getReduceTasks());
    assertEquals("map slot capacities do not match",
      mapSlotsPerTracker * trackers.length,
      jobTracker.getClusterStatus().getMaxMapTasks());
    assertEquals("reduce slot capacities do not match",
      reduceSlotsPerTracker * trackers.length,
      jobTracker.getClusterStatus().getMaxReduceTasks());
    
    // send a heartbeat finishing only a map and check
    // counts are updated.
    list.clear();
    addMapTaskAttemptToList(list, mapSlotsPerTask, TaskStatus.State.SUCCEEDED);
    addReduceTaskAttemptToList(list, 
        reduceSlotsPerTask, TaskStatus.State.RUNNING);
    sendHeartbeats(list);
    metrics = jobTracker.getClusterMetrics();
    assertEquals(0, metrics.getOccupiedMapSlots());
    assertEquals(reduceSlotsPerTask, metrics.getOccupiedReduceSlots());
    
    // send a heartbeat finishing the reduce task also.
    list.clear();
    addReduceTaskAttemptToList(list, 
        reduceSlotsPerTask, TaskStatus.State.SUCCEEDED);
    sendHeartbeats(list);
    metrics = jobTracker.getClusterMetrics();
    assertEquals(0, metrics.getOccupiedReduceSlots());
  }

View Full Code Here

    FakeObjectUtilities.sendHeartBeat(jobTracker, status1, false,
      true, trackers[0], responseId);
    FakeObjectUtilities.sendHeartBeat(jobTracker, status2, false,
      true, trackers[1], responseId);
    responseId++; 
    ClusterMetrics metrics = cluster.getClusterStatus();
    assertEquals("reserved map slots do not match", 
      2, metrics.getReservedMapSlots());
    assertEquals("reserved reduce slots do not match", 
      2, metrics.getReservedReduceSlots());


    // redo to test re-reservations.
    FakeObjectUtilities.sendHeartBeat(jobTracker, status1, false,
        true, trackers[0], responseId);
    FakeObjectUtilities.sendHeartBeat(jobTracker, status2, false,
        true, trackers[1], responseId);
    responseId++; 
    metrics = cluster.getClusterStatus();
    assertEquals("reserved map slots do not match", 
        4, metrics.getReservedMapSlots());
    assertEquals("reserved reduce slots do not match", 
        4, metrics.getReservedReduceSlots());


    TaskAttemptID mTid = fakeJob.findMapTask(trackers[1]);
    TaskAttemptID rTid = fakeJob.findReduceTask(trackers[1]);


    fakeJob.finishTask(mTid);
    fakeJob.finishTask(rTid);
    
    assertEquals("Job didnt complete successfully complete", 
      fakeJob.getStatus().getRunState(), JobStatus.SUCCEEDED);
    metrics = cluster.getClusterStatus();
    assertEquals("reserved map slots do not match", 
      0, metrics.getReservedMapSlots());
    assertEquals("reserved reduce slots do not match", 
      0, metrics.getReservedReduceSlots());
  }

View Full Code Here

      }
    }
  }
  
  public synchronized ClusterMetrics getClusterMetrics() {
    return new ClusterMetrics(totalMaps,
      totalReduces, occupiedMapSlots, occupiedReduceSlots,
      reservedMapSlots, reservedReduceSlots,
      totalMapTaskCapacity, totalReduceTaskCapacity,
      totalSubmissions,
      taskTrackers.size() - getBlacklistedTrackerCount(),

View Full Code Here

    
    assertEquals("Trackers not reserved for the job : maps", 
        2, fjob.getNumReservedTaskTrackersForMaps());
    assertEquals("Trackers not reserved for the job : reduces", 
        2, fjob.getNumReservedTaskTrackersForReduces());
    ClusterMetrics metrics = jobTracker.getClusterMetrics();
    assertEquals("reserved map slots do not match",
          4, metrics.getReservedMapSlots());
    assertEquals("reserved reduce slots do not match",
          4, metrics.getReservedReduceSlots());
    
    TaskAttemptID mTid = fjob.findMapTask(trackers[1]);
    TaskAttemptID rTid = fjob.findReduceTask(trackers[1]);


    fjob.finishTask(mTid);
    fjob.finishTask(rTid);
    
    assertEquals("Job didnt complete successfully complete", fjob.getStatus()
        .getRunState(), JobStatus.SUCCEEDED);
    
    assertEquals("Reservation for the job not released: Maps", 
        0, fjob.getNumReservedTaskTrackersForMaps());
    assertEquals("Reservation for the job not released : Reduces", 
        0, fjob.getNumReservedTaskTrackersForReduces());
    metrics = jobTracker.getClusterMetrics();
    assertEquals("reserved map slots do not match",
        0, metrics.getReservedMapSlots());
    assertEquals("reserved reduce slots do not match",
        0, metrics.getReservedReduceSlots());
  }

View Full Code Here

        .getRunState(), JobStatus.SUCCEEDED);
    assertEquals("Reservation for the job not released: Maps", 
        0, job.getNumReservedTaskTrackersForMaps());
    assertEquals("Reservation for the job not released : Reduces", 
        0, job.getNumReservedTaskTrackersForReduces());
    ClusterMetrics metrics = jobTracker.getClusterMetrics();
    assertEquals("reserved map slots do not match",
        0, metrics.getReservedMapSlots());
    assertEquals("reserved reduce slots do not match",
        0, metrics.getReservedReduceSlots());
  }

View Full Code Here

    
    assertEquals("Trackers not reserved for the job : maps", 
        2, job.getNumReservedTaskTrackersForMaps());
    assertEquals("Trackers not reserved for the job : reduces", 
        2, job.getNumReservedTaskTrackersForReduces());
    ClusterMetrics metrics = jobTracker.getClusterMetrics();
    assertEquals("reserved map slots do not match",
        4, metrics.getReservedMapSlots());
    assertEquals("reserved reduce slots do not match",
        4, metrics.getReservedReduceSlots());
  
    /*
     * FakeJobInProgress.findMapTask does not handle
     * task failures. So working around it by failing
     * reduce and blacklisting tracker.
     * Then finish the map task later. 
     */
    TaskAttemptID mTid = job.findMapTask(trackers[0]);
    TaskAttemptID rTid = job.findReduceTask(trackers[0]);
    //Task should blacklist the tasktracker.
    job.failTask(rTid);
    
    assertEquals("Tracker 0 not blacklisted for the job", 1, 
        job.getBlackListedTrackers().size());
    assertEquals("Extra Trackers reserved for the job : maps", 
        1, job.getNumReservedTaskTrackersForMaps());
    assertEquals("Extra Trackers reserved for the job : reduces", 
        1, job.getNumReservedTaskTrackersForReduces());
    metrics = jobTracker.getClusterMetrics();
    assertEquals("reserved map slots do not match",
        2, metrics.getReservedMapSlots());
    assertEquals("reserved reduce slots do not match",
        2, metrics.getReservedReduceSlots());


    //Finish the map task on the tracker 1. Finishing it here to work
    //around bug in the FakeJobInProgress object
    job.finishTask(mTid);
    mTid = job.findMapTask(trackers[1]);
    rTid = job.findReduceTask(trackers[1]);
    job.finishTask(mTid);
    job.finishTask(rTid);
    rTid = job.findReduceTask(trackers[1]);
    job.finishTask(rTid);
    assertEquals("Job didnt complete successfully complete", job.getStatus()
        .getRunState(), JobStatus.SUCCEEDED);
    assertEquals("Trackers not unreserved for the job : maps", 
        0, job.getNumReservedTaskTrackersForMaps());
    assertEquals("Trackers not unreserved for the job : reduces", 
        0, job.getNumReservedTaskTrackersForReduces());
    metrics = jobTracker.getClusterMetrics();
    assertEquals("reserved map slots do not match",
        0, metrics.getReservedMapSlots());
    assertEquals("reserved reduce slots do not match",
        0, metrics.getReservedReduceSlots());
  }

View Full Code Here


  public ClusterMetrics getClusterMetrics() throws IOException,
      InterruptedException {
    try {
      YarnClusterMetrics metrics = client.getYarnClusterMetrics();
      ClusterMetrics oldMetrics =
          new ClusterMetrics(1, 1, 1, 1, 1, 1,
              metrics.getNumNodeManagers() * 10,
              metrics.getNumNodeManagers() * 2, 1,
              metrics.getNumNodeManagers(), 0, 0);
      return oldMetrics;
    } catch (YarnException e) {

View Full Code Here

0 1 2 3 4 5 6 7 8

TOP

Related Classes of org.apache.hadoop.mapreduce.ClusterMetrics

org.apache.hadoop.examples.pi.DistSum$MixMachine

org.apache.hadoop.mapred.gridmix.Statistics$StatCollector

org.apache.hadoop.mapred.gridmix.StressJobFactory

org.apache.hadoop.mapred.JobClient

org.apache.hadoop.mapred.JobTracker

org.apache.hadoop.mapred.LocalJobRunner

org.apache.hadoop.mapred.LocalJobRunnerWithFix

org.apache.hadoop.mapred.ResourceMgrDelegate

org.apache.hadoop.mapred.SimulatorJobClient

org.apache.hadoop.mapred.TestClusterStatus

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.