Examples of org.apache.hadoop.mapreduce.v2.hs.HistoryStorage

Package org.apache.hadoop.mapreduce.v2.hs

Examples of org.apache.hadoop.mapreduce.v2.hs.HistoryStorage

org.apache.hadoop.mapreduce.v2.api.records.TaskType
Provides an API to query jobs that have finished. For those implementing this API be aware that there is no feedback when files are removed from HDFS. You may rely on HistoryFileManager to help you know when that has happened if you have not made a complete backup of the data stored on HDFS.

    YarnConfiguration conf = new YarnConfiguration();
    conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
    JobContext mockJobContext = mock(JobContext.class);
    ApplicationAttemptId attemptid = 
      ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
    JobId jobId =  TypeConverter.toYarn(
        TypeConverter.fromYarn(attemptid.getApplicationId()));
    
    WaitForItHandler waitForItHandler = new WaitForItHandler();
    
    when(mockContext.getApplicationID()).thenReturn(attemptid.getApplicationId());

View Full Code Here

    YarnConfiguration conf = new YarnConfiguration();
    conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
    JobContext mockJobContext = mock(JobContext.class);
    ApplicationAttemptId attemptid = 
      ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
    JobId jobId =  TypeConverter.toYarn(
        TypeConverter.fromYarn(attemptid.getApplicationId()));
    
    WaitForItHandler waitForItHandler = new WaitForItHandler();
    
    when(mockContext.getApplicationID()).thenReturn(attemptid.getApplicationId());

View Full Code Here

            appSubmitTime, startTime, finishTime, setupProgress, 0.0f, 0.0f,
            cleanupProgress, jobFile, amInfos, isUber, diagsb.toString());
      }


      computeProgress();
      JobReport report = MRBuilderUtils.newJobReport(jobId, jobName, username,
          state, appSubmitTime, startTime, finishTime, setupProgress,
          this.mapProgress, this.reduceProgress,
          cleanupProgress, jobFile, amInfos, isUber, diagsb.toString());
      return report;
    } finally {

View Full Code Here


  @Override
  public JobReport getReport() {
    readLock.lock();
    try {
      JobState state = getState();


      // jobFile can be null if the job is not yet inited.
      String jobFile =
          remoteJobConfFile == null ? "" : remoteJobConfFile.toString();

View Full Code Here


  @Override
  public JobState getState() {
    readLock.lock();
    try {
      JobState state = getExternalState(getInternalState());
      if (!appContext.hasSuccessfullyUnregistered()
          && (state == JobState.SUCCEEDED || state == JobState.FAILED
          || state == JobState.KILLED || state == JobState.ERROR)) {
        return lastNonFinalState;
      } else {

View Full Code Here

      writeLock.unlock();
    }
  }


  private void rememberLastNonFinalState(JobStateInternal stateInternal) {
    JobState state = getExternalState(stateInternal);
    // if state is not the final state, set lastNonFinalState
    if (state != JobState.SUCCEEDED && state != JobState.FAILED
        && state != JobState.KILLED && state != JobState.ERROR) {
      lastNonFinalState = state;
    }

View Full Code Here

    }
  }


  private static JobCounterUpdateEvent createJobCounterUpdateEventTASucceeded(
      TaskAttemptImpl taskAttempt) {
    TaskId taskId = taskAttempt.attemptId.getTaskId();
    JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskId.getJobId());
    updateMillisCounters(jce, taskAttempt);
    return jce;
  }

View Full Code Here

    }
  }
  
  private static void updateMillisCounters(JobCounterUpdateEvent jce,
      TaskAttemptImpl taskAttempt) {
    TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
    long duration = (taskAttempt.getFinishTime() - taskAttempt.getLaunchTime());
    int mbRequired =
        taskAttempt.getMemoryRequired(taskAttempt.conf, taskType);
    int vcoresRequired = taskAttempt.getCpuRequired(taskAttempt.conf, taskType);

View Full Code Here

    return jce;
  }
  
  private static JobCounterUpdateEvent createJobCounterUpdateEventTAFailed(
      TaskAttemptImpl taskAttempt, boolean taskAlreadyCompleted) {
    TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
    JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskAttempt.getID().getTaskId().getJobId());
    
    if (taskType == TaskType.MAP) {
      jce.addCounterUpdate(JobCounter.NUM_FAILED_MAPS, 1);
    } else {

View Full Code Here

    return jce;
  }
  
  private static JobCounterUpdateEvent createJobCounterUpdateEventTAKilled(
      TaskAttemptImpl taskAttempt, boolean taskAlreadyCompleted) {
    TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
    JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskAttempt.getID().getTaskId().getJobId());
    
    if (taskType == TaskType.MAP) {
      jce.addCounterUpdate(JobCounter.NUM_KILLED_MAPS, 1);
    } else {

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapreduce.v2.hs.HistoryStorage

org.apache.hadoop.conf.Configuration

org.apache.hadoop.fs.FileSystem

org.apache.hadoop.fs.FSDataInputStream

org.apache.hadoop.fs.Path

org.apache.hadoop.fs.permission.FsPermission

org.apache.hadoop.mapred.JobConf

org.apache.hadoop.mapred.MapTaskAttemptImpl

org.apache.hadoop.mapred.TaskAttemptListenerImpl

org.apache.hadoop.mapred.TestClientRedirect

org.apache.hadoop.mapred.TestClientRedirect$AMService

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.