Examples of org.apache.hadoop.mapreduce.v2.app.MRApp

Package org.apache.hadoop.mapreduce.v2.app

Examples of org.apache.hadoop.mapreduce.v2.app.MRApp

org.apache.hadoop.mapreduce.v2.app.MRApp
Mock MRAppMaster. Doesn't start RPC servers. No threads are started except of the event Dispatcher thread.

    }
  }


  @Test
  public void testMRAppHistoryForMap() throws Exception {
    MRApp app = new FailingAttemptsMRApp(1, 0);
    testMRAppHistory(app);
  }

View Full Code Here

    testMRAppHistory(app);
  }


  @Test
  public void testMRAppHistoryForReduce() throws Exception {
    MRApp app = new FailingAttemptsMRApp(0, 1);
    testMRAppHistory(app);
  }

View Full Code Here

  public void verifySlotMillis(int mapMemMb, int reduceMemMb,
      int minContainerSize) throws Exception {
    Clock actualClock = new SystemClock();
    ControlledClock clock = new ControlledClock(actualClock);
    clock.setTime(10);
    MRApp app =
        new MRApp(1, 1, false, "testSlotMillisCounterUpdate", true, clock);
    Configuration conf = new Configuration();
    conf.setInt(MRJobConfig.MAP_MEMORY_MB, mapMemMb);
    conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, reduceMemMb);
    app.setClusterInfo(new ClusterInfo(BuilderUtils
        .newResource(minContainerSize), BuilderUtils.newResource(10240)));


    Job job = app.submit(conf);
    app.waitForState(job, JobState.RUNNING);
    Map<TaskId, Task> tasks = job.getTasks();
    Assert.assertEquals("Num tasks is not correct", 2, tasks.size());
    Iterator<Task> taskIter = tasks.values().iterator();
    Task mTask = taskIter.next();
    app.waitForState(mTask, TaskState.RUNNING);
    Task rTask = taskIter.next();
    app.waitForState(rTask, TaskState.RUNNING);
    Map<TaskAttemptId, TaskAttempt> mAttempts = mTask.getAttempts();
    Assert.assertEquals("Num attempts is not correct", 1, mAttempts.size());
    Map<TaskAttemptId, TaskAttempt> rAttempts = rTask.getAttempts();
    Assert.assertEquals("Num attempts is not correct", 1, rAttempts.size());
    TaskAttempt mta = mAttempts.values().iterator().next();
    TaskAttempt rta = rAttempts.values().iterator().next();
    app.waitForState(mta, TaskAttemptState.RUNNING);
    app.waitForState(rta, TaskAttemptState.RUNNING);


    clock.setTime(11);
    app.getContext()
        .getEventHandler()
        .handle(new TaskAttemptEvent(mta.getID(), TaskAttemptEventType.TA_DONE));
    app.getContext()
        .getEventHandler()
        .handle(new TaskAttemptEvent(rta.getID(), TaskAttemptEventType.TA_DONE));
    app.waitForState(job, JobState.SUCCEEDED);
    Assert.assertEquals(mta.getFinishTime(), 11);
    Assert.assertEquals(mta.getLaunchTime(), 10);
    Assert.assertEquals(rta.getFinishTime(), 11);
    Assert.assertEquals(rta.getLaunchTime(), 10);
    Assert.assertEquals((int) Math.ceil((float) mapMemMb / minContainerSize),

View Full Code Here

    InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
    server = rpc.getServer(ContainerManager.class, new DummyContainerManager(),
        addr, conf, null, 1);
    server.start();


    MRApp app = new MRAppWithSlowNM();


    try {
    Job job = app.submit(conf);
    app.waitForState(job, JobState.RUNNING);


    Map<TaskId, Task> tasks = job.getTasks();
    Assert.assertEquals("Num tasks is not correct", 1, tasks.size());


    Task task = tasks.values().iterator().next();
    app.waitForState(task, TaskState.SCHEDULED);


    Map<TaskAttemptId, TaskAttempt> attempts = tasks.values().iterator()
        .next().getAttempts();
      Assert.assertEquals("Num attempts is not correct", maxAttempts,
          attempts.size());


    TaskAttempt attempt = attempts.values().iterator().next();
    app.waitForState(attempt, TaskAttemptState.ASSIGNED);


    app.waitForState(job, JobState.FAILED);


    String diagnostics = attempt.getDiagnostics().toString();
    LOG.info("attempt.getDiagnostics: " + diagnostics);


      Assert.assertTrue(diagnostics.contains("Container launch failed for "
          + "container_0_0000_01_000000 : "));
      Assert
          .assertTrue(diagnostics
              .contains("java.net.SocketTimeoutException: 3000 millis timeout while waiting for channel"));


    } finally {
      server.stop();
    app.stop();
  }
  }

View Full Code Here

            .setClass(
                    CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
                    MyResolver.class, DNSToSwitchMapping.class);


    RackResolver.init(config);
    MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
            true);
    app.submit(config);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    app.waitForState(job, JobState.SUCCEEDED);


    historyServer = new JobHistoryServer();


    historyServer.init(config);
    historyServer.start();

View Full Code Here

      Configuration conf = new Configuration();
      conf.setClass(
          CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
          MyResolver.class, DNSToSwitchMapping.class);
      RackResolver.init(conf);
      MRApp app = new MRAppWithHistoryWithFailedAttempt(2, 1, true, this
          .getClass().getName(), true);
      app.submit(conf);
      Job job = app.getContext().getAllJobs().values().iterator().next();
      JobId jobId = job.getID();
      app.waitForState(job, JobState.SUCCEEDED);


      // make sure all events are flushed
      app.waitForState(Service.STATE.STOPPED);


      String jobhistoryDir = JobHistoryUtils
          .getHistoryIntermediateDoneDirForUser(conf);
      JobHistory jobHistory = new JobHistory();
      jobHistory.init(conf);

View Full Code Here

      Configuration conf = new Configuration();
      conf.setClass(
          CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
          MyResolver.class, DNSToSwitchMapping.class);
      RackResolver.init(conf);
      MRApp app = new MRAppWithHistoryWithFailedTask(2, 1, true, this
          .getClass().getName(), true);
      app.submit(conf);
      Job job = app.getContext().getAllJobs().values().iterator().next();
      JobId jobId = job.getID();
      app.waitForState(job, JobState.FAILED);


      // make sure all events are flushed
      app.waitForState(Service.STATE.STOPPED);


      String jobhistoryDir = JobHistoryUtils
          .getHistoryIntermediateDoneDirForUser(conf);
      JobHistory jobHistory = new JobHistory();
      jobHistory.init(conf);

View Full Code Here

      Configuration conf = new Configuration();
      conf.setClass(
          CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
          MyResolver.class, DNSToSwitchMapping.class);
      RackResolver.init(conf);
      MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
          true);
      app.submit(conf);
      Job job = app.getContext().getAllJobs().values().iterator().next();
      JobId jobId = job.getID();
      LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
      app.waitForState(job, JobState.SUCCEEDED);


      // make sure all events are flushed
      app.waitForState(Service.STATE.STOPPED);


      HistoryFileManagerForTest hfm = new HistoryFileManagerForTest();
      hfm.init(conf);
      HistoryFileInfo fileInfo = hfm.getFileInfo(jobId);
      Assert.assertNotNull("Unable to locate job history", fileInfo);

View Full Code Here

      conf.setClass(
          CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
          MyResolver.class, DNSToSwitchMapping.class);


      RackResolver.init(conf);
      MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
          true);
      app.submit(conf);
      Job job = app.getContext().getAllJobs().values().iterator().next();
      JobId jobId = job.getID();


      app.waitForState(job, JobState.SUCCEEDED);


      // make sure all events are flushed
      app.waitForState(Service.STATE.STOPPED);
      HistoryFileManager hfm = new HistoryFileManager();
      hfm.init(conf);
      HistoryFileInfo fileInfo = hfm.getFileInfo(jobId);
      hfm.initExisting();
      // wait for move files form the done_intermediate directory to the gone

View Full Code Here

          .setClass(
              CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
              MyResolver.class, DNSToSwitchMapping.class);


      RackResolver.init(configuration);
      MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
          true);
      app.submit(configuration);
      Job job = app.getContext().getAllJobs().values().iterator().next();
      app.waitForState(job, JobState.SUCCEEDED);


      JobHistory jobHistory = new JobHistory();
      jobHistory.init(configuration);
      // Method getAllJobs
      Assert.assertEquals(1, jobHistory.getAllJobs().size());
      // and with ApplicationId
      Assert.assertEquals(1, jobHistory.getAllJobs(app.getAppID()).size());


      JobsInfo jobsinfo = jobHistory.getPartialJobs(0L, 10L, null, "default",
          0L, System.currentTimeMillis() + 1, 0L,
          System.currentTimeMillis() + 1, JobState.SUCCEEDED);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapreduce.v2.app.MRApp

org.apache.hadoop.conf.Configuration

org.apache.hadoop.fs.FileSystem

org.apache.hadoop.fs.FSDataInputStream

org.apache.hadoop.fs.Path

org.apache.hadoop.fs.permission.FsPermission

org.apache.hadoop.io.Text

org.apache.hadoop.mapred.JobConf

org.apache.hadoop.mapred.TaskAttemptListenerImpl

org.apache.hadoop.mapred.TestClientRedirect

org.apache.hadoop.mapred.TestClientRedirect$AMService

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.