Package org.apache.hadoop.mapreduce

Examples of org.apache.hadoop.mapreduce.TaskAttemptID


    //Mock up the TaskAttempts
    Map<TaskAttemptID, TaskAttemptInfo> mockTaskAttempts =
        new HashMap<TaskAttemptID, TaskAttemptInfo>();

    TaskAttemptID taId1 = new TaskAttemptID(taskID, 2);
    TaskAttemptInfo mockTAinfo1 = getMockTaskAttemptInfo(taId1,
        TaskAttemptState.FAILED);
    mockTaskAttempts.put(taId1, mockTAinfo1);

    TaskAttemptID taId2 = new TaskAttemptID(taskID, 1);
    TaskAttemptInfo mockTAinfo2 = getMockTaskAttemptInfo(taId2,
        TaskAttemptState.FAILED);
    mockTaskAttempts.put(taId2, mockTAinfo2);

    OutputCommitter mockCommitter = mock (OutputCommitter.class);
    TaskInfo mockTaskInfo = mock(TaskInfo.class);
    when(mockTaskInfo.getTaskStatus()).thenReturn("SUCCEEDED");
    when(mockTaskInfo.getTaskId()).thenReturn(taskID);
    when(mockTaskInfo.getAllTaskAttempts()).thenReturn(mockTaskAttempts);

    recoverMapTask.handle(
        new TaskRecoverEvent(taskId, mockTaskInfo, mockCommitter, true));

    ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
    verify(mockEventHandler,atLeast(1)).handle(
        (org.apache.hadoop.yarn.event.Event) arg.capture());

    Map<TaskAttemptID, TaskAttemptState> finalAttemptStates =
        new HashMap<TaskAttemptID, TaskAttemptState>();
    finalAttemptStates.put(taId1, TaskAttemptState.FAILED);
    finalAttemptStates.put(taId2, TaskAttemptState.FAILED);
    // check for one new attempt launched since successful attempt not found
    TaskAttemptID taId3 = new TaskAttemptID(taskID, 2000);
    finalAttemptStates.put(taId3, TaskAttemptState.NEW);

    List<EventType> jobHistoryEvents = new ArrayList<EventType>();
    jobHistoryEvents.add(EventType.TASK_STARTED);
    jobHistoryEvents.add(EventType.MAP_ATTEMPT_STARTED);
View Full Code Here


    //Mock up the TaskAttempts
    Map<TaskAttemptID, TaskAttemptInfo> mockTaskAttempts =
        new HashMap<TaskAttemptID, TaskAttemptInfo>();

    TaskAttemptID taId1 = new TaskAttemptID(taskID, 2);
    TaskAttemptInfo mockTAinfo1 = getMockTaskAttemptInfo(taId1,
        TaskAttemptState.SUCCEEDED);
    mockTaskAttempts.put(taId1, mockTAinfo1);

    TaskAttemptID taId2 = new TaskAttemptID(taskID, 1);
    TaskAttemptInfo mockTAinfo2 = getMockTaskAttemptInfo(taId2,
        TaskAttemptState.SUCCEEDED);
    mockTaskAttempts.put(taId2, mockTAinfo2);

    OutputCommitter mockCommitter = mock (OutputCommitter.class);
View Full Code Here

        org.apache.hadoop.mapreduce.TaskType.MAP, taskId.getId());

    //Mock up the TaskAttempts
    Map<TaskAttemptID, TaskAttemptInfo> mockTaskAttempts =
        new HashMap<TaskAttemptID, TaskAttemptInfo>();
    TaskAttemptID taId1 = new TaskAttemptID(taskID, 2);
    TaskAttemptInfo mockTAinfo1 = getMockTaskAttemptInfo(taId1,
        TaskAttemptState.KILLED);
    mockTaskAttempts.put(taId1, mockTAinfo1);

    TaskAttemptID taId2 = new TaskAttemptID(taskID, 1);
    TaskAttemptInfo mockTAinfo2 = getMockTaskAttemptInfo(taId2,
        TaskAttemptState.KILLED);
    mockTaskAttempts.put(taId2, mockTAinfo2);

    OutputCommitter mockCommitter = mock (OutputCommitter.class);
View Full Code Here

  }
 
  @Test
  public void testInMemoryMerger() throws Throwable {
    JobID jobId = new JobID("a", 0);
    TaskAttemptID reduceId = new TaskAttemptID(
        new TaskID(jobId, TaskType.REDUCE, 0), 0);
    TaskAttemptID mapId1 = new TaskAttemptID(
        new TaskID(jobId, TaskType.MAP, 1), 0);
    TaskAttemptID mapId2 = new TaskAttemptID(
        new TaskID(jobId, TaskType.MAP, 2), 0);
   
    LocalDirAllocator lda = new LocalDirAllocator(MRConfig.LOCAL_DIR);
   
    MergeManagerImpl<Text, Text> mergeManager = new MergeManagerImpl<Text, Text>(
View Full Code Here

  @SuppressWarnings("unchecked")
  @Test(timeout=30000)
  public void testCopyFromHostConnectionTimeout() throws Exception {
    LOG.info("testCopyFromHostConnectionTimeout");
    JobConf job = new JobConf();
    TaskAttemptID id = TaskAttemptID.forName("attempt_0_1_r_1_1");
    ShuffleScheduler<Text, Text> ss = mock(ShuffleScheduler.class);
    MergeManager<Text, Text> mm = mock(MergeManager.class);
    Reporter r = mock(Reporter.class);
    ShuffleClientMetrics metrics = mock(ShuffleClientMetrics.class);
    ExceptionReporter except = mock(ExceptionReporter.class);
    SecretKey key = JobTokenSecretManager.createSecretKey(new byte[]{0,0,0,0});
    HttpURLConnection connection = mock(HttpURLConnection.class);
    when(connection.getInputStream()).thenThrow(
        new SocketTimeoutException("This is a fake timeout :)"));
   
    Counters.Counter allErrs = mock(Counters.Counter.class);
    when(r.getCounter(anyString(), anyString()))
      .thenReturn(allErrs);
   
    Fetcher<Text,Text> underTest = new FakeFetcher<Text,Text>(job, id, ss, mm,
        r, metrics, except, key, connection);

    MapHost host = new MapHost("localhost", "http://localhost:8080/");
   
    ArrayList<TaskAttemptID> maps = new ArrayList<TaskAttemptID>(1);
    TaskAttemptID map1ID = TaskAttemptID.forName("attempt_0_1_m_1_1");
    maps.add(map1ID);
    TaskAttemptID map2ID = TaskAttemptID.forName("attempt_0_1_m_2_1");
    maps.add(map2ID);
    when(ss.getMapsForHost(host)).thenReturn(maps);
   
    String encHash = "vFE234EIFCiBgYs2tCXY/SjT8Kg=";
   
View Full Code Here

  @SuppressWarnings("unchecked")
  @Test
  public void testCopyFromHostBogusHeader() throws Exception {
    LOG.info("testCopyFromHostBogusHeader");
    JobConf job = new JobConf();
    TaskAttemptID id = TaskAttemptID.forName("attempt_0_1_r_1_1");
    ShuffleScheduler<Text, Text> ss = mock(ShuffleScheduler.class);
    MergeManager<Text, Text> mm = mock(MergeManager.class);
    Reporter r = mock(Reporter.class);
    ShuffleClientMetrics metrics = mock(ShuffleClientMetrics.class);
    ExceptionReporter except = mock(ExceptionReporter.class);
    SecretKey key = JobTokenSecretManager.createSecretKey(new byte[]{0,0,0,0});
    HttpURLConnection connection = mock(HttpURLConnection.class);
   
    Counters.Counter allErrs = mock(Counters.Counter.class);
    when(r.getCounter(anyString(), anyString()))
      .thenReturn(allErrs);
   
    Fetcher<Text,Text> underTest = new FakeFetcher<Text,Text>(job, id, ss, mm,
        r, metrics, except, key, connection);
   

    MapHost host = new MapHost("localhost", "http://localhost:8080/");
   
    ArrayList<TaskAttemptID> maps = new ArrayList<TaskAttemptID>(1);
    TaskAttemptID map1ID = TaskAttemptID.forName("attempt_0_1_m_1_1");
    maps.add(map1ID);
    TaskAttemptID map2ID = TaskAttemptID.forName("attempt_0_1_m_2_1");
    maps.add(map2ID);
    when(ss.getMapsForHost(host)).thenReturn(maps);
   
    String encHash = "vFE234EIFCiBgYs2tCXY/SjT8Kg=";
    String replyHash = SecureShuffleUtils.generateHash(encHash.getBytes(), key);
View Full Code Here

  @SuppressWarnings("unchecked")
  @Test
  public void testCopyFromHostWait() throws Exception {
    LOG.info("testCopyFromHostWait");
    JobConf job = new JobConf();
    TaskAttemptID id = TaskAttemptID.forName("attempt_0_1_r_1_1");
    ShuffleScheduler<Text, Text> ss = mock(ShuffleScheduler.class);
    MergeManager<Text, Text> mm = mock(MergeManager.class);
    Reporter r = mock(Reporter.class);
    ShuffleClientMetrics metrics = mock(ShuffleClientMetrics.class);
    ExceptionReporter except = mock(ExceptionReporter.class);
    SecretKey key = JobTokenSecretManager.createSecretKey(new byte[]{0,0,0,0});
    HttpURLConnection connection = mock(HttpURLConnection.class);
   
    Counters.Counter allErrs = mock(Counters.Counter.class);
    when(r.getCounter(anyString(), anyString()))
      .thenReturn(allErrs);
   
    Fetcher<Text,Text> underTest = new FakeFetcher<Text,Text>(job, id, ss, mm,
        r, metrics, except, key, connection);
   

    MapHost host = new MapHost("localhost", "http://localhost:8080/");
   
    ArrayList<TaskAttemptID> maps = new ArrayList<TaskAttemptID>(1);
    TaskAttemptID map1ID = TaskAttemptID.forName("attempt_0_1_m_1_1");
    maps.add(map1ID);
    TaskAttemptID map2ID = TaskAttemptID.forName("attempt_0_1_m_2_1");
    maps.add(map2ID);
    when(ss.getMapsForHost(host)).thenReturn(maps);
   
    String encHash = "vFE234EIFCiBgYs2tCXY/SjT8Kg=";
    String replyHash = SecureShuffleUtils.generateHash(encHash.getBytes(), key);
View Full Code Here

  @SuppressWarnings("unchecked")
  @Test
  public void testCopyFromHostExtraBytes() throws Exception {
    LOG.info("testCopyFromHostWaitExtraBytes");
    JobConf job = new JobConf();
    TaskAttemptID id = TaskAttemptID.forName("attempt_0_1_r_1_1");
    ShuffleScheduler<Text, Text> ss = mock(ShuffleScheduler.class);
    MergeManager<Text, Text> mm = mock(MergeManager.class);
    Reporter r = mock(Reporter.class);
    ShuffleClientMetrics metrics = mock(ShuffleClientMetrics.class);
    ExceptionReporter except = mock(ExceptionReporter.class);
    SecretKey key = JobTokenSecretManager.createSecretKey(new byte[]{0,0,0,0});
    HttpURLConnection connection = mock(HttpURLConnection.class);

    Counters.Counter allErrs = mock(Counters.Counter.class);
    when(r.getCounter(anyString(), anyString()))
      .thenReturn(allErrs);

    Fetcher<Text,Text> underTest = new FakeFetcher<Text,Text>(job, id, ss, mm,
        r, metrics, except, key, connection);


    MapHost host = new MapHost("localhost", "http://localhost:8080/");

    ArrayList<TaskAttemptID> maps = new ArrayList<TaskAttemptID>(1);
    TaskAttemptID map1ID = TaskAttemptID.forName("attempt_0_1_m_1_1");
    maps.add(map1ID);
    TaskAttemptID map2ID = TaskAttemptID.forName("attempt_0_1_m_2_1");
    maps.add(map2ID);
    when(ss.getMapsForHost(host)).thenReturn(maps);

    String encHash = "vFE234EIFCiBgYs2tCXY/SjT8Kg=";
    String replyHash = SecureShuffleUtils.generateHash(encHash.getBytes(), key);

    when(connection.getResponseCode()).thenReturn(200);
    when(connection.getHeaderField(SecureShuffleUtils.HTTP_HEADER_REPLY_URL_HASH))
      .thenReturn(replyHash);
    ShuffleHeader header = new ShuffleHeader(map1ID.toString(), 14, 10, 1);

    ByteArrayOutputStream bout = new ByteArrayOutputStream();
    DataOutputStream dos = new DataOutputStream(bout);
    IFileOutputStream ios = new IFileOutputStream(dos);
    header.write(dos);
    ios.write("MAPDATA123".getBytes());
    ios.finish();

    ShuffleHeader header2 = new ShuffleHeader(map2ID.toString(), 14, 10, 1);
    IFileOutputStream ios2 = new IFileOutputStream(dos);
    header2.write(dos);
    ios2.write("MAPDATA456".getBytes());
    ios2.finish();
View Full Code Here

        exitCode = 0;
      } else if (displayTasks) {
        displayTasks(cluster.getJob(JobID.forName(jobid)), taskType, taskState);
        exitCode = 0;
      } else if(killTask) {
        TaskAttemptID taskID = TaskAttemptID.forName(taskid);
        Job job = cluster.getJob(taskID.getJobID());
        if (job == null) {
          System.out.println("Could not find job " + jobid);
        } else if (job.killTask(taskID)) {
          System.out.println("Killed task " + taskid);
          exitCode = 0;
        } else {
          System.out.println("Could not kill task " + taskid);
          exitCode = -1;
        }
      } else if(failTask) {
        TaskAttemptID taskID = TaskAttemptID.forName(taskid);
        Job job = cluster.getJob(taskID.getJobID());
        if (job == null) {
            System.out.println("Could not find job " + jobid);
        } else if(job.failTask(taskID)) {
          System.out.println("Killed task " + taskID + " by failing it");
          exitCode = 0;
        } else {
          System.out.println("Could not fail task " + taskid);
          exitCode = -1;
        }
      } else if (logs) {
        try {
        JobID jobID = JobID.forName(jobid);
        TaskAttemptID taskAttemptID = TaskAttemptID.forName(taskid);
        LogParams logParams = cluster.getLogParams(jobID, taskAttemptID);
        LogDumper logDumper = new LogDumper();
        logDumper.setConf(getConf());
        exitCode = logDumper.dumpAContainersLogs(logParams.getApplicationId(),
            logParams.getContainerId(), logParams.getNodeId(),
View Full Code Here

 
  private TaskAttemptID[] copyMapOutput(MapHost host,
                                DataInputStream input,
                                Set<TaskAttemptID> remaining) {
    MapOutput<K,V> mapOutput = null;
    TaskAttemptID mapId = null;
    long decompressedLength = -1;
    long compressedLength = -1;
   
    try {
      long startTime = System.currentTimeMillis();
View Full Code Here

TOP

Related Classes of org.apache.hadoop.mapreduce.TaskAttemptID

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.