Examples of org.apache.hadoop.mapreduce.jobhistory.JobHistory

Package org.apache.hadoop.mapreduce.jobhistory

Examples of org.apache.hadoop.mapreduce.jobhistory.JobHistory

org.apache.hadoop.mapreduce.jobhistory.JobHistory
JobHistory is the class that is responsible for creating and maintaining job history information.


    MiniMRCluster mr = null;
    mr = new MiniMRCluster(2, "file:///", 3, null, null, conf);


    JobTracker jt = mr.getJobTrackerRunner().getJobTracker();
    JobHistory jh = jt.getJobHistory();


    jh.init(jt, conf, "localhost", 1234);
    JobID jobId = JobID.forName("job_200809171136_0001");
    jh.setupEventWriter(jobId, conf);
    Map<JobACL, AccessControlList> jobACLs =
        new HashMap<JobACL, AccessControlList>();
    AccessControlList viewJobACL =
        new AccessControlList("user1,user2 group1,group2");
    AccessControlList modifyJobACL =
        new AccessControlList("user3,user4 group3, group4");
    jobACLs.put(JobACL.VIEW_JOB, viewJobACL);
    jobACLs.put(JobACL.MODIFY_JOB, modifyJobACL);
    JobSubmittedEvent jse =
        new JobSubmittedEvent(jobId, weirdJob, username, 12345, weirdPath,
            jobACLs);
    jh.logEvent(jse, jobId);


    JobFinishedEvent jfe =
      new JobFinishedEvent(jobId, 12346, 1, 1, 0, 0, new Counters(),
          new Counters(), new Counters());
    jh.logEvent(jfe, jobId);
    jh.closeWriter(jobId);


    // Try to write one more event now, should not fail
    TaskID tid = TaskID.forName("task_200809171136_0001_m_000002");
    TaskFinishedEvent tfe =
      new TaskFinishedEvent(tid, 0, TaskType.MAP, "", null);
    boolean caughtException = false;


    try {
      jh.logEvent(tfe, jobId);
    } catch (Exception e) {
      caughtException = true;
    }


    assertFalse("Writing an event after closing event writer is not handled",

View Full Code Here

   */
  public static void validateJobHistoryFileContent(MiniMRCluster mr,
                              RunningJob job, JobConf conf) throws IOException  {


    JobID id = job.getID();
    JobHistory jobHistory = 
      mr.getJobTrackerRunner().getJobTracker().getJobHistory();
    Path doneDir = jobHistory.getCompletedJobHistoryLocation();
    // Get the history file name
    String logFileName = getDoneFile(jobHistory, conf, id, doneDir);


    // Framework history log file location
    Path logFile = new Path(doneDir, logFileName);

View Full Code Here

          3, null, null, conf);


      assertEquals("Files in logDir did not move to DONE folder",
          0, logDirFs.listStatus(logDirPath).length);


      JobHistory jobHistory = 
        mr.getJobTrackerRunner().getJobTracker().getJobHistory();
      Path doneDir = jobHistory.getCompletedJobHistoryLocation();


      assertEquals("Files in DONE dir not correct",
          2, doneDir.getFileSystem(conf).listStatus(doneDir).length);


      // run the TCs

View Full Code Here

      // the validation of history file content
      conf.setInt("mapred.jobtracker.completeuserjobs.maximum", 10);


      // Run a job that will be succeeded and validate its history file
      RunningJob job = UtilsForTests.runJobSucceed(conf, inDir, outDir);
      JobHistory jobHistory = 
        mr.getJobTrackerRunner().getJobTracker().getJobHistory();
      Path doneDir = jobHistory.getCompletedJobHistoryLocation();
      assertEquals("History DONE folder not correct", 
          doneFolder, doneDir.toString());
      JobID id = job.getID();
      String logFileName = getDoneFile(jobHistory, conf, id, doneDir);

View Full Code Here


      // Run a job that will be succeeded and validate its job status
      // existing in history file
      RunningJob job = UtilsForTests.runJobSucceed(conf, inDir, outDir);
      
      JobHistory jobHistory = 
        mr.getJobTrackerRunner().getJobTracker().getJobHistory();
      validateJobHistoryJobStatus(jobHistory, job.getID(), conf, 
          JobStatus.getJobRunState(JobStatus.SUCCEEDED));
      
      // Run a job that will be failed and validate its job status

View Full Code Here

      }


      // Start the cluster, create a history file
      mr = new MiniMRCluster(0, "file:///", 3, null, null, conf);
      JobTracker jt = mr.getJobTrackerRunner().getJobTracker();
      JobHistory jh = jt.getJobHistory();
      final JobID jobId = JobID.forName("job_200809171136_0001");
      jh.setupEventWriter(jobId, conf);
      Map<JobACL, AccessControlList> jobACLs =
          new HashMap<JobACL, AccessControlList>();
      JobSubmittedEvent jse =
        new JobSubmittedEvent(jobId, "job", "user", 12345, "path", jobACLs);
      jh.logEvent(jse, jobId);
      jh.closeWriter(jobId);


      // Corrupt the history file. User RawLocalFileSystem so that we
      // do keep the original CRC file intact.
      String historyFileName = jobId.toString() + "_" + "user";
      Path historyFilePath = new Path (historyDir.toString(), historyFileName);

View Full Code Here

    this.startTime = clock.getTime();
    infoServer = new HttpServer("job", infoBindAddress, tmpInfoPort, 
        tmpInfoPort == 0, conf);
    infoServer.setAttribute("job.tracker", this);
    // initialize history parameters.
    jobHistory = new JobHistory();
    jobHistory.init(this, conf, this.localMachine, this.startTime);
    
    infoServer.addServlet("reducegraph", "/taskgraph", TaskGraphServlet.class);
    infoServer.start();

View Full Code Here

    infoServer.setAttribute("job.tracker", this);
    
    // initialize history parameters.
    FileSystem historyFS = null;


    jobHistory = new JobHistory();
    final JobTracker jtFinal = this;
    try {
      historyFS = mrOwner.doAs(new PrivilegedExceptionAction<FileSystem>() {
        public FileSystem run() throws IOException {
          jobHistory.init(jtFinal, conf, jtFinal.localMachine, jtFinal.startTime);

View Full Code Here

   */
  public static void validateJobHistoryFileContent(MiniMRCluster mr,
                              RunningJob job, JobConf conf) throws IOException  {


    JobID id = job.getID();
    JobHistory jobHistory = 
      mr.getJobTrackerRunner().getJobTracker().getJobHistory();
    Path doneDir = jobHistory.getCompletedJobHistoryLocation();
    // Get the history file name
    String logFileName = getDoneFile(jobHistory, conf, id, doneDir);


    // Framework history log file location
    Path logFile = new Path(doneDir, logFileName);

View Full Code Here

          3, null, null, conf);


      assertEquals("Files in logDir did not move to DONE folder",
          0, logDirFs.listStatus(logDirPath).length);


      JobHistory jobHistory = 
        mr.getJobTrackerRunner().getJobTracker().getJobHistory();
      Path doneDir = jobHistory.getCompletedJobHistoryLocation();


      assertEquals("Files in DONE dir not correct",
          2, doneDir.getFileSystem(conf).listStatus(doneDir).length);


      // run the TCs

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.hadoop.mapreduce.jobhistory.JobHistory

org.apache.hadoop.fs.FSDataOutputStream

org.apache.hadoop.fs.Path

org.apache.hadoop.fs.permission.FsPermission

org.apache.hadoop.mapred.JobTracker

org.apache.hadoop.mapred.TestJobHistory

org.apache.hadoop.mapred.TestJobHistoryParsing

java.io.File

java.util.concurrent.ThreadPoolExecutor

java.io.IOException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.