Package org.apache.hadoop.mapreduce.jobhistory

Examples of org.apache.hadoop.mapreduce.jobhistory.JobHistory

805
806
807
808
809
810
811
812
813
814
815
816
817
      // the validation of history file content
      conf.setInt("mapred.jobtracker.completeuserjobs.maximum", 10);

      // Run a job that will be succeeded and validate its history file
      RunningJob job = UtilsForTests.runJobSucceed(conf, inDir, outDir);
      JobHistory jobHistory =
        mr.getJobTrackerRunner().getJobTracker().getJobHistory();
      Path doneDir = jobHistory.getCompletedJobHistoryLocation();
      assertEquals("History DONE folder not correct",
          doneFolder, doneDir.toString());
      JobID id = job.getID();
      String logFileName = getDoneFile(jobHistory, conf, id, doneDir);
View Full Code Here
953
954
955
956
957
958
959
960
961
962
963
      // Run a job that will be succeeded and validate its job status
      // existing in history file
      RunningJob job = UtilsForTests.runJobSucceed(conf, inDir, outDir);
     
      JobHistory jobHistory =
        mr.getJobTrackerRunner().getJobTracker().getJobHistory();
      validateJobHistoryJobStatus(jobHistory, job.getID(), conf,
          JobStatus.getJobRunState(JobStatus.SUCCEEDED));
     
      // Run a job that will be failed and validate its job status
View Full Code Here
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
      }

      // Start the cluster, create a history file
      mr = new MiniMRCluster(0, "file:///", 3, null, null, conf);
      JobTracker jt = mr.getJobTrackerRunner().getJobTracker();
      JobHistory jh = jt.getJobHistory();
      final JobID jobId = JobID.forName("job_200809171136_0001");
      jh.setupEventWriter(jobId, conf);
      Map<JobACL, AccessControlList> jobACLs =
          new HashMap<JobACL, AccessControlList>();
      JobSubmittedEvent jse =
        new JobSubmittedEvent(jobId, "job", "user", 12345, "path", jobACLs,
        "default");
      jh.logEvent(jse, jobId);
      jh.closeWriter(jobId);

      // Corrupt the history file. User RawLocalFileSystem so that we
      // do keep the original CRC file intact.
      String historyFileName = jobId.toString() + "_" + "user";
      Path historyFilePath = new Path (historyDir.toString(), historyFileName);
View Full Code Here
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
    this.startTime = clock.getTime();
    infoServer = new HttpServer("job", infoBindAddress, tmpInfoPort,
        tmpInfoPort == 0, conf, aclsManager.getAdminsAcl());
    infoServer.setAttribute("job.tracker", this);
    // initialize history parameters.
    jobHistory = new JobHistory();
    jobHistory.init(this, conf, this.localMachine, this.startTime);
   
    infoServer.addServlet("reducegraph", "/taskgraph", TaskGraphServlet.class);
    infoServer.start();
   
View Full Code Here
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
    infoServer.setAttribute("job.tracker", this);
   
    // initialize history parameters.
    FileSystem historyFS = null;

    jobHistory = new JobHistory();
    final JobTracker jtFinal = this;
    try {
      historyFS = getMROwner().doAs(new PrivilegedExceptionAction<FileSystem>() {
        public FileSystem run() throws IOException {
          jobHistory.init(jtFinal, conf, jtFinal.localMachine, jtFinal.startTime);
View Full Code Here
563
564
565
566
567
568
569
570
571
572
573
574
575
   */
  public static void validateJobHistoryFileContent(MiniMRCluster mr,
                              RunningJob job, JobConf conf) throws IOException  {

    JobID id = job.getID();
    JobHistory jobHistory =
      mr.getJobTrackerRunner().getJobTracker().getJobHistory();
    Path doneDir = jobHistory.getCompletedJobHistoryLocation();
    // Get the history file name
    String logFileName = getDoneFile(jobHistory, conf, id, doneDir);

    // Framework history log file location
    Path logFile = new Path(doneDir, logFileName);
View Full Code Here
644
645
646
647
648
649
650
651
652
653
654
655
656
          3, null, null, conf);

      assertEquals("Files in logDir did not move to DONE folder",
          0, logDirFs.listStatus(logDirPath).length);

      JobHistory jobHistory =
        mr.getJobTrackerRunner().getJobTracker().getJobHistory();
      Path doneDir = jobHistory.getCompletedJobHistoryLocation();

      assertEquals("Files in DONE dir not correct",
          2, doneDir.getFileSystem(conf).listStatus(doneDir).length);

      // run the TCs
View Full Code Here
769
770
771
772
773
774
775
776
777
778
779
780
781
      // the validation of history file content
      conf.setInt("mapred.jobtracker.completeuserjobs.maximum", 10);

      // Run a job that will be succeeded and validate its history file
      RunningJob job = UtilsForTests.runJobSucceed(conf, inDir, outDir);
      JobHistory jobHistory =
        mr.getJobTrackerRunner().getJobTracker().getJobHistory();
      Path doneDir = jobHistory.getCompletedJobHistoryLocation();
      assertEquals("History DONE folder not correct",
          doneFolder, doneDir.toString());
      JobID id = job.getID();
      String logFileName = getDoneFile(jobHistory, conf, id, doneDir);
View Full Code Here
917
918
919
920
921
922
923
924
925
926
927
      // Run a job that will be succeeded and validate its job status
      // existing in history file
      RunningJob job = UtilsForTests.runJobSucceed(conf, inDir, outDir);
     
      JobHistory jobHistory =
        mr.getJobTrackerRunner().getJobTracker().getJobHistory();
      validateJobHistoryJobStatus(jobHistory, job.getID(), conf,
          JobStatus.getJobRunState(JobStatus.SUCCEEDED));
     
      // Run a job that will be failed and validate its job status
View Full Code Here
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
      }

      // Start the cluster, create a history file
      mr = new MiniMRCluster(0, "file:///", 3, null, null, conf);
      JobTracker jt = mr.getJobTrackerRunner().getJobTracker();
      JobHistory jh = jt.getJobHistory();
      final JobID jobId = JobID.forName("job_200809171136_0001");
      jh.setupEventWriter(jobId, conf);
      Map<JobACL, AccessControlList> jobACLs =
          new HashMap<JobACL, AccessControlList>();
      JobSubmittedEvent jse =
        new JobSubmittedEvent(jobId, "job", "user", 12345, "path", jobACLs,
        "default");
      jh.logEvent(jse, jobId);
      jh.closeWriter(jobId);

      // Corrupt the history file. User RawLocalFileSystem so that we
      // do keep the original CRC file intact.
      String historyFileName = jobId.toString() + "_" + "user";
      Path historyFilePath = new Path (historyDir.toString(), historyFileName);
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapreduce.jobhistory.JobHistory

  • org.apache.hadoop.fs.FSDataOutputStream
  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.fs.permission.FsPermission
  • org.apache.hadoop.mapred.JobTracker
  • org.apache.hadoop.mapred.TestJobHistory
  • org.apache.hadoop.mapred.TestJobHistoryParsing
  • java.io.File
  • java.util.concurrent.ThreadPoolExecutor
  • java.io.IOException
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.