Package org.apache.hadoop.mapreduce.v2.api.records

Examples of org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent

235
236
237
238
239
240
241
242
243
244
245
    YarnConfiguration conf = new YarnConfiguration();
    conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
    JobContext mockJobContext = mock(JobContext.class);
    ApplicationAttemptId attemptid =
      ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
    JobId jobId =  TypeConverter.toYarn(
        TypeConverter.fromYarn(attemptid.getApplicationId()));
   
    WaitForItHandler waitForItHandler = new WaitForItHandler();
   
    when(mockContext.getApplicationID()).thenReturn(attemptid.getApplicationId());
View Full Code Here
283
284
285
286
287
288
289
290
291
292
293
    YarnConfiguration conf = new YarnConfiguration();
    conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
    JobContext mockJobContext = mock(JobContext.class);
    ApplicationAttemptId attemptid =
      ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
    JobId jobId =  TypeConverter.toYarn(
        TypeConverter.fromYarn(attemptid.getApplicationId()));
   
    WaitForItHandler waitForItHandler = new WaitForItHandler();
   
    when(mockContext.getApplicationID()).thenReturn(attemptid.getApplicationId());
View Full Code Here
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
  private static class TaskAttemptCompletedEventTransition implements
      SingleArcTransition<JobImpl, JobEvent> {
    @Override
    public void transition(JobImpl job, JobEvent event) {
      TaskAttemptCompletionEvent tce =
        ((JobTaskAttemptCompletedEvent) event).getCompletionEvent();
      // Add the TaskAttemptCompletionEvent
      //eventId is equal to index in the arraylist
      tce.setEventId(job.taskAttemptCompletionEvents.size());
      job.taskAttemptCompletionEvents.add(tce);
      int mapEventIdx = -1;
      if (TaskType.MAP.equals(tce.getAttemptId().getTaskId().getTaskType())) {
        // we track map completions separately from task completions because
        // - getMapAttemptCompletionEvents uses index ranges specific to maps
        // - type converting the same events over and over is expensive
        mapEventIdx = job.mapAttemptCompletionEvents.size();
        job.mapAttemptCompletionEvents.add(TypeConverter.fromYarn(tce));
      }
      job.taskCompletionIdxToMapCompletionIdx.add(mapEventIdx);
     
      TaskAttemptId attemptId = tce.getAttemptId();
      TaskId taskId = attemptId.getTaskId();
      //make the previous completion event as obsolete if it exists
      Integer successEventNo =
          job.successAttemptCompletionEventNoMap.remove(taskId);
      if (successEventNo != null) {
        TaskAttemptCompletionEvent successEvent =
          job.taskAttemptCompletionEvents.get(successEventNo);
        successEvent.setStatus(TaskAttemptCompletionEventStatus.OBSOLETE);
        int mapCompletionIdx =
            job.taskCompletionIdxToMapCompletionIdx.get(successEventNo);
        if (mapCompletionIdx >= 0) {
          // update the corresponding TaskCompletionEvent for the map
          TaskCompletionEvent mapEvent =
View Full Code Here
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
  private static class TaskAttemptCompletedEventTransition implements
      SingleArcTransition<JobImpl, JobEvent> {
    @Override
    public void transition(JobImpl job, JobEvent event) {
      TaskAttemptCompletionEvent tce =
        ((JobTaskAttemptCompletedEvent) event).getCompletionEvent();
      // Add the TaskAttemptCompletionEvent
      //eventId is equal to index in the arraylist
      tce.setEventId(job.taskAttemptCompletionEvents.size());
      job.taskAttemptCompletionEvents.add(tce);
      int mapEventIdx = -1;
      if (TaskType.MAP.equals(tce.getAttemptId().getTaskId().getTaskType())) {
        // we track map completions separately from task completions because
        // - getMapAttemptCompletionEvents uses index ranges specific to maps
        // - type converting the same events over and over is expensive
        mapEventIdx = job.mapAttemptCompletionEvents.size();
        job.mapAttemptCompletionEvents.add(TypeConverter.fromYarn(tce));
      }
      job.taskCompletionIdxToMapCompletionIdx.add(mapEventIdx);
     
      TaskAttemptId attemptId = tce.getAttemptId();
      TaskId taskId = attemptId.getTaskId();
      //make the previous completion event as obsolete if it exists
      Integer successEventNo =
          job.successAttemptCompletionEventNoMap.remove(taskId);
      if (successEventNo != null) {
        TaskAttemptCompletionEvent successEvent =
          job.taskAttemptCompletionEvents.get(successEventNo);
        successEvent.setStatus(TaskAttemptCompletionEventStatus.OBSOLETE);
        int mapCompletionIdx =
            job.taskCompletionIdxToMapCompletionIdx.get(successEventNo);
        if (mapCompletionIdx >= 0) {
          // update the corresponding TaskCompletionEvent for the map
          TaskCompletionEvent mapEvent =
View Full Code Here
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
  private static class TaskAttemptCompletedEventTransition implements
      SingleArcTransition<JobImpl, JobEvent> {
    @Override
    public void transition(JobImpl job, JobEvent event) {
      TaskAttemptCompletionEvent tce =
        ((JobTaskAttemptCompletedEvent) event).getCompletionEvent();
      // Add the TaskAttemptCompletionEvent
      //eventId is equal to index in the arraylist
      tce.setEventId(job.taskAttemptCompletionEvents.size());
      job.taskAttemptCompletionEvents.add(tce);
     
      //make the previous completion event as obsolete if it exists
      Object successEventNo =
        job.successAttemptCompletionEventNoMap.remove(tce.getAttemptId().getTaskId());
      if (successEventNo != null) {
        TaskAttemptCompletionEvent successEvent =
          job.taskAttemptCompletionEvents.get((Integer) successEventNo);
        successEvent.setStatus(TaskAttemptCompletionEventStatus.OBSOLETE);
      }

      if (TaskAttemptCompletionEventStatus.SUCCEEDED.equals(tce.getStatus())) {
        job.successAttemptCompletionEventNoMap.put(tce.getAttemptId().getTaskId(),
            tce.getEventId());
View Full Code Here
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
    finishedAttempts++;
    TaskAttempt attempt = attempts.get(attemptId);
    //raise the completion event only if the container is assigned
    // to nextAttemptNumber
    if (attempt.getNodeHttpAddress() != null) {
      TaskAttemptCompletionEvent tce = recordFactory
          .newRecordInstance(TaskAttemptCompletionEvent.class);
      tce.setEventId(-1);
      tce.setMapOutputServerAddress("http://"
          + attempt.getNodeHttpAddress().split(":")[0] + ":"
          + attempt.getShufflePort());
      tce.setStatus(status);
      tce.setAttemptId(attempt.getID());
      int runTime = 0;
      if (attempt.getFinishTime() != 0 && attempt.getLaunchTime() !=0)
        runTime = (int)(attempt.getFinishTime() - attempt.getLaunchTime());
      tce.setAttemptRunTime(runTime);
     
      //raise the event to job so that it adds the completion event to its
      //data structures
      eventHandler.handle(new JobTaskAttemptCompletedEvent(tce));
    }
View Full Code Here
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
    });

    int eventId = 0;
    for (TaskAttempt taskAttempt : allTaskAttempts) {

      TaskAttemptCompletionEvent tace = RecordFactoryProvider.getRecordFactory(
          null).newRecordInstance(TaskAttemptCompletionEvent.class);

      int attemptRunTime = -1;
      if (taskAttempt.getLaunchTime() != 0 && taskAttempt.getFinishTime() != 0) {
        attemptRunTime =
            (int) (taskAttempt.getFinishTime() - taskAttempt.getLaunchTime());
      }
      // Default to KILLED
      TaskAttemptCompletionEventStatus taceStatus =
          TaskAttemptCompletionEventStatus.KILLED;
      String taStateString = taskAttempt.getState().toString();
      try {
        taceStatus = TaskAttemptCompletionEventStatus.valueOf(taStateString);
      } catch (Exception e) {
        LOG.warn("Cannot constuct TACEStatus from TaskAtemptState: ["
            + taStateString + "] for taskAttemptId: [" + taskAttempt.getID()
            + "]. Defaulting to KILLED");
      }

      tace.setAttemptId(taskAttempt.getID());
      tace.setAttemptRunTime(attemptRunTime);
      tace.setEventId(eventId++);
      tace.setMapOutputServerAddress(taskAttempt
          .getAssignedContainerMgrAddress());
      tace.setStatus(taceStatus);
      completionEvents.add(tace);
    }
  }
View Full Code Here
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
  private static class TaskAttemptCompletedEventTransition implements
      SingleArcTransition<JobImpl, JobEvent> {
    @Override
    public void transition(JobImpl job, JobEvent event) {
      TaskAttemptCompletionEvent tce =
        ((JobTaskAttemptCompletedEvent) event).getCompletionEvent();
      // Add the TaskAttemptCompletionEvent
      //eventId is equal to index in the arraylist
      tce.setEventId(job.taskAttemptCompletionEvents.size());
      job.taskAttemptCompletionEvents.add(tce);
      if (TaskType.MAP.equals(tce.getAttemptId().getTaskId().getTaskType())) {
        job.mapAttemptCompletionEvents.add(tce);
      }
     
      //make the previous completion event as obsolete if it exists
      Object successEventNo =
        job.successAttemptCompletionEventNoMap.remove(tce.getAttemptId().getTaskId());
      if (successEventNo != null) {
        TaskAttemptCompletionEvent successEvent =
          job.taskAttemptCompletionEvents.get((Integer) successEventNo);
        successEvent.setStatus(TaskAttemptCompletionEventStatus.OBSOLETE);
      }

      if (TaskAttemptCompletionEventStatus.SUCCEEDED.equals(tce.getStatus())) {
        job.successAttemptCompletionEventNoMap.put(tce.getAttemptId().getTaskId(),
            tce.getEventId());
View Full Code Here
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
      TaskAttemptCompletionEventStatus status) {
    TaskAttempt attempt = attempts.get(attemptId);
    //raise the completion event only if the container is assigned
    // to nextAttemptNumber
    if (attempt.getNodeHttpAddress() != null) {
      TaskAttemptCompletionEvent tce = recordFactory
          .newRecordInstance(TaskAttemptCompletionEvent.class);
      tce.setEventId(-1);
      tce.setMapOutputServerAddress(StringInterner.weakIntern("http://"
          + attempt.getNodeHttpAddress().split(":")[0] + ":"
          + attempt.getShufflePort()));
      tce.setStatus(status);
      tce.setAttemptId(attempt.getID());
      int runTime = 0;
      if (attempt.getFinishTime() != 0 && attempt.getLaunchTime() !=0)
        runTime = (int)(attempt.getFinishTime() - attempt.getLaunchTime());
      tce.setAttemptRunTime(runTime);
     
      //raise the event to job so that it adds the completion event to its
      //data structures
      eventHandler.handle(new JobTaskAttemptCompletedEvent(tce));
    }
View Full Code Here
191
192
193
194
195
196
197
198
199
200
201
202
    TaskId tid = MRBuilderUtils.newTaskId(jid, 0,
        isMap ? org.apache.hadoop.mapreduce.v2.api.records.TaskType.MAP
            : org.apache.hadoop.mapreduce.v2.api.records.TaskType.REDUCE);
    TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(tid, 0);
    RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
    TaskAttemptCompletionEvent tce = recordFactory
        .newRecordInstance(TaskAttemptCompletionEvent.class);
    tce.setEventId(eventId);
    tce.setAttemptId(attemptId);
    tce.setStatus(status);
    return tce;
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent

  • org.apache.hadoop.conf.Configuration
  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.FSDataInputStream
  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.fs.permission.FsPermission
  • org.apache.hadoop.mapred.JobConf
  • org.apache.hadoop.mapred.TaskAttemptListenerImpl
  • org.apache.hadoop.mapred.TestClientRedirect
  • org.apache.hadoop.mapred.TestClientRedirect$AMService
  • org.apache.hadoop.mapred.TestClientRedirect$HistoryService
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.