Examples of JobIndexInfo

org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo
Maintains information which may be used by the jobHistory indexing system.

Examples of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo

        String jobName, JobId jobId) {
      this.historyFile = historyFile;
      this.confFile = conf;
      this.writer = writer;
      this.jobIndexInfo =
          new JobIndexInfo(-1, -1, user, jobName, jobId, -1, -1, null);
      this.jobSummary = new JobSummary();
      this.flushTimer = new Timer("FlushTimer", true);
    }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo

        doneDirFc);
    for (FileStatus fs : historyFileList) {
      if(LOG.isDebugEnabled()) {
        LOG.debug("Adding in history for "+fs.getPath());
      }
      JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath()
          .getName());
      String confFileName = JobHistoryUtils
          .getIntermediateConfFileName(jobIndexInfo.getJobId());
      String summaryFileName = JobHistoryUtils
          .getIntermediateSummaryFileName(jobIndexInfo.getJobId());
      MetaInfo metaInfo = new MetaInfo(fs.getPath(), new Path(fs.getPath()
          .getParent(), confFileName), new Path(fs.getPath().getParent(),
          summaryFileName), jobIndexInfo);
      addToJobListCache(jobIndexInfo.getJobId(), metaInfo);
    }
  }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo

  private void scanIntermediateDirectory(final Path absPath)
      throws IOException {
    List<FileStatus> fileStatusList = scanDirectoryForHistoryFiles(absPath,
        intermediateDoneDirFc);
    for (FileStatus fs : fileStatusList) {
      JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath()
          .getName());
      String confFileName = JobHistoryUtils
          .getIntermediateConfFileName(jobIndexInfo.getJobId());
      String summaryFileName = JobHistoryUtils
          .getIntermediateSummaryFileName(jobIndexInfo.getJobId());
      MetaInfo metaInfo = new MetaInfo(fs.getPath(), new Path(fs.getPath()
          .getParent(), confFileName), new Path(fs.getPath().getParent(),
          summaryFileName), jobIndexInfo);
      if (!intermediateListCache.containsKey(jobIndexInfo.getJobId())) {
        intermediateListCache.put(jobIndexInfo.getJobId(), metaInfo);
      }
    }
  }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo

   * @throws IOException
   */
  private MetaInfo getJobMetaInfo(List<FileStatus> fileStatusList, JobId jobId) 
  throws IOException {
    for (FileStatus fs : fileStatusList) {
      JobIndexInfo jobIndexInfo = 
        FileNameIndexUtils.getIndexInfo(fs.getPath().getName());
      if (jobIndexInfo.getJobId().equals(jobId)) {
        String confFileName = JobHistoryUtils
            .getIntermediateConfFileName(jobIndexInfo.getJobId());
        String summaryFileName = JobHistoryUtils
            .getIntermediateSummaryFileName(jobIndexInfo.getJobId());
        MetaInfo metaInfo = new MetaInfo(fs.getPath(), new Path(fs.getPath()
            .getParent(), confFileName), new Path(fs.getPath().getParent(),
            summaryFileName), jobIndexInfo);
        return metaInfo;
      }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo

        Collections.sort(serialDirList);
        for (FileStatus serialDir : serialDirList) {
          List<FileStatus> historyFileList = 
            scanDirectoryForHistoryFiles(serialDir.getPath(), doneDirFc);
          for (FileStatus historyFile : historyFileList) {
            JobIndexInfo jobIndexInfo = 
              FileNameIndexUtils.getIndexInfo(historyFile.getPath().getName());
            long effectiveTimestamp = 
              getEffectiveTimestamp(jobIndexInfo.getFinishTime(), historyFile);
            if (shouldDelete(effectiveTimestamp)) {
              String confFileName = 
                JobHistoryUtils.getIntermediateConfFileName(jobIndexInfo.getJobId());
              MetaInfo metaInfo = new MetaInfo(historyFile.getPath(),
                  new Path(historyFile.getPath().getParent(), confFileName), 
                  null, jobIndexInfo);
              delete(metaInfo);
            } else {

View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo

    String jobhistoryDir = JobHistoryUtils
        .getHistoryIntermediateDoneDirForUser(conf);
    JobHistory jobHistory = new JobHistory();
    jobHistory.init(conf);


    JobIndexInfo jobIndexInfo = jobHistory.getJobMetaInfo(jobId)
        .getJobIndexInfo();
    String jobhistoryFileName = FileNameIndexUtils
        .getDoneFileName(jobIndexInfo);


    Path historyFilePath = new Path(jobhistoryDir, jobhistoryFileName);

View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo

    String jobhistoryDir = JobHistoryUtils
        .getHistoryIntermediateDoneDirForUser(conf);
    JobHistory jobHistory = new JobHistory();
    jobHistory.init(conf);


    JobIndexInfo jobIndexInfo = jobHistory.getJobMetaInfo(jobId)
        .getJobIndexInfo();
    String jobhistoryFileName = FileNameIndexUtils
        .getDoneFileName(jobIndexInfo);


    Path historyFilePath = new Path(jobhistoryDir, jobhistoryFileName);

View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo

    for(Map.Entry<JobId, Job> entry: mocked.entrySet()) {
      JobId id = entry.getKey();
      Job j = entry.getValue();
      ret.full.put(id, new MockCompletedJob(j));
      JobReport report = j.getReport();
      JobIndexInfo info = new JobIndexInfo(report.getStartTime(), 
          report.getFinishTime(), j.getUserName(), j.getName(), id, 
          j.getCompletedMaps(), j.getCompletedReduces(), String.valueOf(j.getState()));
      info.setQueueName(j.getQueueName());
      ret.partial.put(id, new PartialJob(info, id));
    }
    return ret;
  }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo

        doneDirFc);
    for (FileStatus fs : historyFileList) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("Adding in history for " + fs.getPath());
      }
      JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath()
          .getName());
      String confFileName = JobHistoryUtils
          .getIntermediateConfFileName(jobIndexInfo.getJobId());
      String summaryFileName = JobHistoryUtils
          .getIntermediateSummaryFileName(jobIndexInfo.getJobId());
      HistoryFileInfo fileInfo = new HistoryFileInfo(fs.getPath(), new Path(fs
          .getPath().getParent(), confFileName), new Path(fs.getPath()
          .getParent(), summaryFileName), jobIndexInfo, true);
      jobListCache.addIfAbsent(fileInfo);
    }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo

   */
  private void scanIntermediateDirectory(final Path absPath) throws IOException {
    List<FileStatus> fileStatusList = scanDirectoryForHistoryFiles(absPath,
        intermediateDoneDirFc);
    for (FileStatus fs : fileStatusList) {
      JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath()
          .getName());
      String confFileName = JobHistoryUtils
          .getIntermediateConfFileName(jobIndexInfo.getJobId());
      String summaryFileName = JobHistoryUtils
          .getIntermediateSummaryFileName(jobIndexInfo.getJobId());
      HistoryFileInfo fileInfo = new HistoryFileInfo(fs.getPath(), new Path(fs
          .getPath().getParent(), confFileName), new Path(fs.getPath()
          .getParent(), summaryFileName), jobIndexInfo, false);


      final HistoryFileInfo old = jobListCache.addIfAbsent(fileInfo);

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.