Package org.apache.hadoop.mapreduce.v2.jobhistory

Examples of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo

51
52
53
54
55
56
57
58
59
60
61
62
63
    for(Map.Entry<JobId, Job> entry: mocked.entrySet()) {
      JobId id = entry.getKey();
      Job j = entry.getValue();
      ret.full.put(id, new MockCompletedJob(j));
      JobReport report = j.getReport();
      JobIndexInfo info = new JobIndexInfo(report.getStartTime(),
          report.getFinishTime(), j.getUserName(), j.getName(), id,
          j.getCompletedMaps(), j.getCompletedReduces(), String.valueOf(j.getState()));
      info.setQueueName(j.getQueueName());
      ret.partial.put(id, new PartialJob(info, id));
    }
    return ret;
  }
View Full Code Here
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
    Collections.sort(serialDirList);
    for (FileStatus serialDir : serialDirList) {
      List<FileStatus> historyFileList = scanDirectoryForHistoryFiles(
          serialDir.getPath(), doneDirFc);
      for (FileStatus historyFile : historyFileList) {
        JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(historyFile
            .getPath().getName());
        long effectiveTimestamp = getEffectiveTimestamp(
            jobIndexInfo.getFinishTime(), historyFile);
        if (effectiveTimestamp <= cutoff) {
          HistoryFileInfo fileInfo = this.jobListCache.get(jobIndexInfo
              .getJobId());
          if (fileInfo == null) {
            String confFileName = JobHistoryUtils
                .getIntermediateConfFileName(jobIndexInfo.getJobId());

            fileInfo = new HistoryFileInfo(historyFile.getPath(), new Path(
                historyFile.getPath().getParent(), confFileName), null,
                jobIndexInfo, true);
          }
View Full Code Here
351
352
353
354
355
356
357
358
359
360
361
    String jobhistoryDir = JobHistoryUtils
        .getHistoryIntermediateDoneDirForUser(conf);
    JobHistory jobHistory = new JobHistory();
    jobHistory.init(conf);

    JobIndexInfo jobIndexInfo = jobHistory.getJobFileInfo(jobId)
        .getJobIndexInfo();
    String jobhistoryFileName = FileNameIndexUtils
        .getDoneFileName(jobIndexInfo);

    Path historyFilePath = new Path(jobhistoryDir, jobhistoryFileName);
View Full Code Here
752
753
754
755
756
757
758
759
760
        String jobName, JobId jobId) {
      this.historyFile = historyFile;
      this.confFile = conf;
      this.writer = writer;
      this.jobIndexInfo =
          new JobIndexInfo(-1, -1, user, jobName, jobId, -1, -1, null);
      this.jobSummary = new JobSummary();
      this.flushTimer = new Timer("FlushTimer", true);
    }
View Full Code Here
801
802
803
804
805
806
807
808
809
        String jobName, JobId jobId) {
      this.historyFile = historyFile;
      this.confFile = conf;
      this.writer = writer;
      this.jobIndexInfo =
          new JobIndexInfo(-1, -1, user, jobName, jobId, -1, -1, null);
      this.jobSummary = new JobSummary();
      this.flushTimer = new Timer("FlushTimer", true);
    }
View Full Code Here
757
758
759
760
761
762
763
764
765
        String jobName, JobId jobId) {
      this.historyFile = historyFile;
      this.confFile = conf;
      this.writer = writer;
      this.jobIndexInfo =
          new JobIndexInfo(-1, -1, user, jobName, jobId, -1, -1, null);
      this.jobSummary = new JobSummary();
      this.flushTimer = new Timer("FlushTimer", true);
    }
View Full Code Here
826
827
828
829
830
831
832
833
834
835
836
        String queueName) {
      this.historyFile = historyFile;
      this.confFile = conf;
      this.writer = writer;
      this.jobIndexInfo =
          new JobIndexInfo(-1, -1, user, jobName, jobId, -1, -1, null,
                           queueName);
      this.jobSummary = new JobSummary();
      this.flushTimer = new Timer("FlushTimer", true);
      this.forcedJobStateOnShutDown = forcedJobStateOnShutDown;
    }
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
88
89
    for(Map.Entry<JobId, Job> entry: mocked.entrySet()) {
      JobId id = entry.getKey();
      Job j = entry.getValue();
      ret.full.put(id, new MockCompletedJob(j));
      JobReport report = j.getReport();
      JobIndexInfo info = new JobIndexInfo(report.getStartTime(),
          report.getFinishTime(), j.getUserName(), j.getName(), id,
          j.getCompletedMaps(), j.getCompletedReduces(), String.valueOf(j.getState()));
      info.setQueueName(j.getQueueName());
      ret.partial.put(id, new PartialJob(info, id));
    }
    return ret;
  }
View Full Code Here
107
108
109
110
111
112
113
114
115
116
117
  */
  @Test
  public void testPartialJob() throws Exception {
    JobId jobId = new JobIdPBImpl();
    jobId.setId(0);
    JobIndexInfo jii = new JobIndexInfo(0L, System.currentTimeMillis(), "user",
            "jobName", jobId, 3, 2, "JobStatus");
    PartialJob test = new PartialJob(jii, jobId);
    assertEquals(1.0f, test.getProgress(), 0.001);
    assertNull(test.getAllCounters());
    assertNull(test.getTasks());
View Full Code Here
392
393
394
395
396
397
398
399
400
401
402
    String jobhistoryDir = JobHistoryUtils
        .getHistoryIntermediateDoneDirForUser(conf);
    JobHistory jobHistory = new JobHistory();
    jobHistory.init(conf);

    JobIndexInfo jobIndexInfo = jobHistory.getJobFileInfo(jobId)
        .getJobIndexInfo();
    String jobhistoryFileName = FileNameIndexUtils
        .getDoneFileName(jobIndexInfo);

    Path historyFilePath = new Path(jobhistoryDir, jobhistoryFileName);
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo

  • org.apache.hadoop.conf.Configuration
  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.FSDataInputStream
  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.fs.permission.FsPermission
  • org.apache.hadoop.mapred.JobConf
  • org.apache.hadoop.mapred.TaskAttemptListenerImpl
  • org.apache.hadoop.mapred.TestClientRedirect
  • org.apache.hadoop.mapred.TestClientRedirect$AMService
  • org.apache.hadoop.mapred.TestClientRedirect$HistoryService
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.