Package org.apache.hadoop.mapreduce.v2.hs

Examples of org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager$JobListCache

235
236
237
238
239
240
241
242
243
244
245
    YarnConfiguration conf = new YarnConfiguration();
    conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
    JobContext mockJobContext = mock(JobContext.class);
    ApplicationAttemptId attemptid =
      ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
    JobId jobId =  TypeConverter.toYarn(
        TypeConverter.fromYarn(attemptid.getApplicationId()));
   
    WaitForItHandler waitForItHandler = new WaitForItHandler();
   
    when(mockContext.getApplicationID()).thenReturn(attemptid.getApplicationId());
View Full Code Here
283
284
285
286
287
288
289
290
291
292
293
    YarnConfiguration conf = new YarnConfiguration();
    conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
    JobContext mockJobContext = mock(JobContext.class);
    ApplicationAttemptId attemptid =
      ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
    JobId jobId =  TypeConverter.toYarn(
        TypeConverter.fromYarn(attemptid.getApplicationId()));
   
    WaitForItHandler waitForItHandler = new WaitForItHandler();
   
    when(mockContext.getApplicationID()).thenReturn(attemptid.getApplicationId());
View Full Code Here
865
866
867
868
869
870
871
872
873
874
875
            appSubmitTime, startTime, finishTime, setupProgress, 0.0f, 0.0f,
            cleanupProgress, jobFile, amInfos, isUber, diagsb.toString());
      }

      computeProgress();
      JobReport report = MRBuilderUtils.newJobReport(jobId, jobName, username,
          state, appSubmitTime, startTime, finishTime, setupProgress,
          this.mapProgress, this.reduceProgress,
          cleanupProgress, jobFile, amInfos, isUber, diagsb.toString());
      return report;
    } finally {
View Full Code Here
847
848
849
850
851
852
853
854
855
856
857
  @Override
  public JobReport getReport() {
    readLock.lock();
    try {
      JobState state = getState();

      // jobFile can be null if the job is not yet inited.
      String jobFile =
          remoteJobConfFile == null ? "" : remoteJobConfFile.toString();
View Full Code Here
945
946
947
948
949
950
951
952
953
954
955
  @Override
  public JobState getState() {
    readLock.lock();
    try {
      JobState state = getExternalState(getInternalState());
      if (!appContext.hasSuccessfullyUnregistered()
          && (state == JobState.SUCCEEDED || state == JobState.FAILED
          || state == JobState.KILLED || state == JobState.ERROR)) {
        return lastNonFinalState;
      } else {
View Full Code Here
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
      writeLock.unlock();
    }
  }

  private void rememberLastNonFinalState(JobStateInternal stateInternal) {
    JobState state = getExternalState(stateInternal);
    // if state is not the final state, set lastNonFinalState
    if (state != JobState.SUCCEEDED && state != JobState.FAILED
        && state != JobState.KILLED && state != JobState.ERROR) {
      lastNonFinalState = state;
    }
View Full Code Here
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
    }
  }

  private static JobCounterUpdateEvent createJobCounterUpdateEventTASucceeded(
      TaskAttemptImpl taskAttempt) {
    TaskId taskId = taskAttempt.attemptId.getTaskId();
    JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskId.getJobId());
    updateMillisCounters(jce, taskAttempt);
    return jce;
  }
View Full Code Here
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
    }
  }
 
  private static void updateMillisCounters(JobCounterUpdateEvent jce,
      TaskAttemptImpl taskAttempt) {
    TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
    long duration = (taskAttempt.getFinishTime() - taskAttempt.getLaunchTime());
    int mbRequired =
        taskAttempt.getMemoryRequired(taskAttempt.conf, taskType);
    int vcoresRequired = taskAttempt.getCpuRequired(taskAttempt.conf, taskType);
View Full Code Here
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
    return jce;
  }
 
  private static JobCounterUpdateEvent createJobCounterUpdateEventTAFailed(
      TaskAttemptImpl taskAttempt, boolean taskAlreadyCompleted) {
    TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
    JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskAttempt.getID().getTaskId().getJobId());
   
    if (taskType == TaskType.MAP) {
      jce.addCounterUpdate(JobCounter.NUM_FAILED_MAPS, 1);
    } else {
View Full Code Here
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
    return jce;
  }
 
  private static JobCounterUpdateEvent createJobCounterUpdateEventTAKilled(
      TaskAttemptImpl taskAttempt, boolean taskAlreadyCompleted) {
    TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
    JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskAttempt.getID().getTaskId().getJobId());
   
    if (taskType == TaskType.MAP) {
      jce.addCounterUpdate(JobCounter.NUM_KILLED_MAPS, 1);
    } else {
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager$JobListCache

  • com.google.common.util.concurrent.ThreadFactoryBuilder
  • org.apache.hadoop.conf.Configuration
  • org.apache.hadoop.fs.FileContext
  • org.apache.hadoop.fs.FileStatus
  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.FSDataInputStream
  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.fs.permission.FsPermission
  • org.apache.hadoop.mapred.JobACLsManager
  • org.apache.hadoop.mapred.JobConf
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.