Package org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager

Examples of org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo

478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
      app.waitForState(Service.STATE.STOPPED);

      JobHistory jobHistory = new JobHistory();
      jobHistory.init(conf);

      HistoryFileInfo fileInfo = jobHistory.getJobFileInfo(jobId);
     
      JobHistoryParser parser;
      JobInfo jobInfo;
      synchronized (fileInfo) {
        Path historyFilePath = fileInfo.getHistoryFile();
        FSDataInputStream in = null;
        FileContext fc = null;
        try {
          fc = FileContext.getFileContext(conf);
          in = fc.open(fc.makeQualified(historyFilePath));
View Full Code Here
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
      app.waitForState(Service.STATE.STOPPED);

      JobHistory jobHistory = new JobHistory();
      jobHistory.init(conf);

      HistoryFileInfo fileInfo = jobHistory.getJobFileInfo(jobId);

      JobHistoryParser parser;
      JobInfo jobInfo;
      synchronized (fileInfo) {
        Path historyFilePath = fileInfo.getHistoryFile();
        FSDataInputStream in = null;
        FileContext fc = null;
        try {
          fc = FileContext.getFileContext(conf);
          in = fc.open(fc.makeQualified(historyFilePath));
View Full Code Here
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
      // make sure all events are flushed
      app.waitForState(Service.STATE.STOPPED);

      HistoryFileManagerForTest hfm = new HistoryFileManagerForTest();
      hfm.init(conf);
      HistoryFileInfo fileInfo = hfm.getFileInfo(jobId);
      Assert.assertNotNull("Unable to locate job history", fileInfo);

      // force the manager to "forget" the job
      hfm.deleteJobFromJobListCache(fileInfo);
      final int msecPerSleep = 10;
      int msecToSleep = 10 * 1000;
      while (fileInfo.isMovePending() && msecToSleep > 0) {
        Assert.assertTrue(!fileInfo.didMoveFail());
        msecToSleep -= msecPerSleep;
        Thread.sleep(msecPerSleep);
      }
      Assert.assertTrue("Timeout waiting for history move", msecToSleep > 0);

View Full Code Here
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
      // make sure all events are flushed
      app.waitForState(Service.STATE.STOPPED);
      HistoryFileManager hfm = new HistoryFileManager();
      hfm.init(conf);
      HistoryFileInfo fileInfo = hfm.getFileInfo(jobId);
      hfm.initExisting();
      // wait for move files form the done_intermediate directory to the gone
      // directory
      while (fileInfo.isMovePending()) {
        Thread.sleep(300);
      }

      Assert.assertNotNull(hfm.jobListCache.values());

      // try to remove fileInfo
      hfm.clean();
      // check that fileInfo does not deleted
      Assert.assertFalse(fileInfo.isDeleted());
      // correct live time
      hfm.setMaxHistoryAge(-1);
      hfm.clean();
      hfm.stop();
      Assert.assertTrue("Thread pool shutdown",
          hfm.moveToDoneExecutor.isTerminated());
      // should be deleted !
      Assert.assertTrue("file should be deleted ", fileInfo.isDeleted());

    } finally {
      LOG.info("FINISHED testDeleteFileInfo");
    }
  }
View Full Code Here
34
35
36
37
38
39
40
41
42
43
44
45
  @Test (timeout = 1000)
  public void testAddExisting() {
    JobListCache cache = new JobListCache(2, 1000);

    JobId jobId = MRBuilderUtils.newJobId(1, 1, 1);
    HistoryFileInfo fileInfo = Mockito.mock(HistoryFileInfo.class);
    Mockito.when(fileInfo.getJobId()).thenReturn(jobId);

    cache.addIfAbsent(fileInfo);
    cache.addIfAbsent(fileInfo);
    assertEquals("Incorrect number of cache entries", 1,
        cache.values().size());
View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
  public void testEviction() throws InterruptedException {
    int maxSize = 2;
    JobListCache cache = new JobListCache(maxSize, 1000);

    JobId jobId1 = MRBuilderUtils.newJobId(1, 1, 1);
    HistoryFileInfo fileInfo1 = Mockito.mock(HistoryFileInfo.class);
    Mockito.when(fileInfo1.getJobId()).thenReturn(jobId1);

    JobId jobId2 = MRBuilderUtils.newJobId(2, 2, 2);
    HistoryFileInfo fileInfo2 = Mockito.mock(HistoryFileInfo.class);
    Mockito.when(fileInfo2.getJobId()).thenReturn(jobId2);

    JobId jobId3 = MRBuilderUtils.newJobId(3, 3, 3);
    HistoryFileInfo fileInfo3 = Mockito.mock(HistoryFileInfo.class);
    Mockito.when(fileInfo3.getJobId()).thenReturn(jobId3);

    cache.addIfAbsent(fileInfo1);
    cache.addIfAbsent(fileInfo2);
    cache.addIfAbsent(fileInfo3);
View Full Code Here
50
51
52
53
54
55
56
57
58
59
60
61
  @Test(timeout = 2000)
  public void testWithSingleElement() throws InterruptedException {
    JobIdHistoryFileInfoMap mapWithSize = new JobIdHistoryFileInfoMap();

    JobId jobId = MRBuilderUtils.newJobId(1, 1, 1);
    HistoryFileInfo fileInfo1 = Mockito.mock(HistoryFileInfo.class);
    Mockito.when(fileInfo1.getJobId()).thenReturn(jobId);

    // add it twice
    assertEquals("Incorrect return on putIfAbsent()",
        null, mapWithSize.putIfAbsent(jobId, fileInfo1));
    assertEquals("Incorrect return on putIfAbsent()",
View Full Code Here
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
      jobs[i] = mock(Job.class);
      jobIds[i] = mock(JobId.class);
      when(jobs[i].getID()).thenReturn(jobIds[i]);
    }

    HistoryFileInfo fileInfo = mock(HistoryFileInfo.class);
    when(historyManager.getFileInfo(any(JobId.class))).thenReturn(fileInfo);
    when(fileInfo.loadJob()).thenReturn(jobs[0]).thenReturn(jobs[1])
        .thenReturn(jobs[2]);

    // getFullJob will put the job in the cache if it isn't there
    for (int i = 0; i < 3; i++) {
      storage.getFullJob(jobs[i].getID());
    }

    Map<JobId, Job> jobCache = storage.getLoadedJobCache();
    // job0 should have been purged since cache size is 2
    assertFalse(jobCache.containsKey(jobs[0].getID()));
    assertTrue(jobCache.containsKey(jobs[1].getID())
        && jobCache.containsKey(jobs[2].getID()));

    // Setting cache size to 3
    conf.set(JHAdminConfig.MR_HISTORY_LOADED_JOB_CACHE_SIZE, "3");
    doReturn(conf).when(storage).createConf();

    when(fileInfo.loadJob()).thenReturn(jobs[0]).thenReturn(jobs[1])
        .thenReturn(jobs[2]);

    jobHistory.refreshLoadedJobCache();

    for (int i = 0; i < 3; i++) {
View Full Code Here
179
180
181
182
183
184
185
186
187
188
189
    doReturn(fileStatusList).when(historyManager)
        .getHistoryDirsForCleaning(Mockito.anyLong());
    doReturn(true).when(historyManager).deleteDir(any(FileStatus.class));

    JobListCache jobListCache = mock(JobListCache.class);
    HistoryFileInfo fileInfo = mock(HistoryFileInfo.class);
    doReturn(jobListCache).when(historyManager).createJobListCache();
    when(jobListCache.get(any(JobId.class))).thenReturn(fileInfo);

    doNothing().when(fileInfo).delete();
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
74
    Path fullConfPath =
        new Path(TestJobHistoryEntities.class.getClassLoader()
            .getResource(confFileName)
            .getFile());

    HistoryFileInfo info = mock(HistoryFileInfo.class);
    when(info.getConfFile()).thenReturn(fullConfPath);

    JobId jobId = MRBuilderUtils.newJobId(1329348432655l, 1, 1);
    CompletedJob completedJob =
        new CompletedJob(conf, jobId, fulleHistoryPath, true, "user",
            info, jobAclsMgr);
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo

  • org.apache.hadoop.conf.Configuration
  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.FSDataInputStream
  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.fs.permission.FsPermission
  • org.apache.hadoop.mapred.JobConf
  • org.apache.hadoop.mapred.TaskAttemptListenerImpl
  • org.apache.hadoop.mapred.TestClientRedirect
  • org.apache.hadoop.mapred.TestClientRedirect$AMService
  • org.apache.hadoop.mapred.TestClientRedirect$HistoryService
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.