Package org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager

Examples of org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.JobListCache

1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
            ((TaskAttemptKillEvent) event).getMessage());
      }

      //send the deallocate event to ContainerAllocator
      taskAttempt.eventHandler.handle(
          new ContainerAllocatorEvent(taskAttempt.attemptId,
          ContainerAllocator.EventType.CONTAINER_DEALLOCATE));

      // send event to speculator that we withdraw our container needs, if
      //  we're transitioning out of UNASSIGNED
      if (withdrawsContainerRequest) {
View Full Code Here
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
        taskAttempt.eventHandler.handle(
            ContainerRequestEvent.createContainerRequestEventForFailedContainer(
                taskAttempt.attemptId,
                taskAttempt.resourceCapability));
      } else {
        taskAttempt.eventHandler.handle(new ContainerRequestEvent(
            taskAttempt.attemptId, taskAttempt.resourceCapability,
            taskAttempt.dataLocalHosts.toArray(
                new String[taskAttempt.dataLocalHosts.size()]),
            taskAttempt.dataLocalRacks.toArray(
                new String[taskAttempt.dataLocalRacks.size()])));
View Full Code Here
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
    @Override
    public void transition(TaskAttemptImpl taskAttempt,
        TaskAttemptEvent event) {
      // Tell any speculator that we're requesting a container
      taskAttempt.eventHandler.handle
          (new SpeculatorEvent(taskAttempt.getID().getTaskId(), +1));
      //request for container
      if (rescheduled) {
        taskAttempt.eventHandler.handle(
            ContainerRequestEvent.createContainerRequestEventForFailedContainer(
                taskAttempt.attemptId,
View Full Code Here
1530
1531
1532
1533
1534
1535
1536
        .handle(new ContainerRemoteLaunchEvent(taskAttempt.attemptId,
          launchContext, container, taskAttempt.remoteTask));

      // send event to speculator that our container needs are satisfied
      taskAttempt.eventHandler.handle
          (new SpeculatorEvent(taskAttempt.getID().getTaskId(), -1));
    }
View Full Code Here
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
      // send event to speculator that we withdraw our container needs, if
      //  we're transitioning out of UNASSIGNED
      if (withdrawsContainerRequest) {
        taskAttempt.eventHandler.handle
            (new SpeculatorEvent(taskAttempt.getID().getTaskId(), -1));
      }

      switch(finalState) {
        case FAILED:
          taskAttempt.eventHandler.handle(new TaskTAttemptEvent(
View Full Code Here
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
          NetUtils.createSocketAddr(taskAttempt.container.getNodeHttpAddress());
      taskAttempt.trackerName = nodeHttpInetAddr.getHostName();
      taskAttempt.httpPort = nodeHttpInetAddr.getPort();
      taskAttempt.sendLaunchedEvents();
      taskAttempt.eventHandler.handle
          (new SpeculatorEvent
              (taskAttempt.attemptId, true, taskAttempt.clock.getTime()));
      //make remoteTask reference as null as it is no more needed
      //and free up the memory
      taskAttempt.remoteTask = null;
     
View Full Code Here
1679
1680
1681
1682
1683
1684
1685
1686
      taskAttempt.logAttemptFinishedEvent(TaskAttemptStateInternal.SUCCEEDED);
      taskAttempt.eventHandler.handle(new TaskTAttemptEvent(
          taskAttempt.attemptId,
          TaskEventType.T_ATTEMPT_SUCCEEDED));
      taskAttempt.eventHandler.handle
      (new SpeculatorEvent
          (taskAttempt.reportedStatus, taskAttempt.clock.getTime()));
   }
View Full Code Here
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
public class TestJobListCache {

  @Test (timeout = 1000)
  public void testAddExisting() {
    JobListCache cache = new JobListCache(2, 1000);

    JobId jobId = MRBuilderUtils.newJobId(1, 1, 1);
    HistoryFileInfo fileInfo = Mockito.mock(HistoryFileInfo.class);
    Mockito.when(fileInfo.getJobId()).thenReturn(jobId);

    cache.addIfAbsent(fileInfo);
    cache.addIfAbsent(fileInfo);
    assertEquals("Incorrect number of cache entries", 1,
        cache.values().size());
  }
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
  }

  @Test (timeout = 1000)
  public void testEviction() throws InterruptedException {
    int maxSize = 2;
    JobListCache cache = new JobListCache(maxSize, 1000);

    JobId jobId1 = MRBuilderUtils.newJobId(1, 1, 1);
    HistoryFileInfo fileInfo1 = Mockito.mock(HistoryFileInfo.class);
    Mockito.when(fileInfo1.getJobId()).thenReturn(jobId1);

    JobId jobId2 = MRBuilderUtils.newJobId(2, 2, 2);
    HistoryFileInfo fileInfo2 = Mockito.mock(HistoryFileInfo.class);
    Mockito.when(fileInfo2.getJobId()).thenReturn(jobId2);

    JobId jobId3 = MRBuilderUtils.newJobId(3, 3, 3);
    HistoryFileInfo fileInfo3 = Mockito.mock(HistoryFileInfo.class);
    Mockito.when(fileInfo3.getJobId()).thenReturn(jobId3);

    cache.addIfAbsent(fileInfo1);
    cache.addIfAbsent(fileInfo2);
    cache.addIfAbsent(fileInfo3);

    Collection <HistoryFileInfo> values;
    for (int i = 0; i < 9; i++) {
      values = cache.values();
      if (values.size() > maxSize) {
        Thread.sleep(100);
      } else {
        assertFalse("fileInfo1 should have been evicted",
          values.contains(fileInfo1));
View Full Code Here
178
179
180
181
182
183
184
185
186
187
188
189
190
191
    doReturn(fileStatusList).when(historyManager)
        .getHistoryDirsForCleaning(Mockito.anyLong());
    doReturn(true).when(historyManager).deleteDir(any(FileStatus.class));

    JobListCache jobListCache = mock(JobListCache.class);
    HistoryFileInfo fileInfo = mock(HistoryFileInfo.class);
    doReturn(jobListCache).when(historyManager).createJobListCache();
    when(jobListCache.get(any(JobId.class))).thenReturn(fileInfo);

    doNothing().when(fileInfo).delete();

    // Set job retention time to 24 hrs and cleaner interval to 2 secs
    Configuration conf = new Configuration();
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.JobListCache

  • org.apache.hadoop.conf.Configuration
  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.FSDataInputStream
  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.fs.permission.FsPermission
  • org.apache.hadoop.mapred.JobConf
  • org.apache.hadoop.mapred.TaskAttemptListenerImpl
  • org.apache.hadoop.mapred.TestClientRedirect
  • org.apache.hadoop.mapred.TestClientRedirect$AMService
  • org.apache.hadoop.mapred.TestClientRedirect$HistoryService
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.