Examples of MRApp

org.apache.hadoop.mapreduce.v2.app.MRApp
Mock MRAppMaster. Doesn't start RPC servers. No threads are started except of the event Dispatcher thread.

Examples of org.apache.hadoop.mapreduce.v2.app.MRApp

      conf.setClass(
          NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
          MyResolver.class, DNSToSwitchMapping.class);


      RackResolver.init(conf);
      MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
          true);
      app.submit(conf);
      Job job = app.getContext().getAllJobs().values().iterator().next();
      JobId jobId = job.getID();


      app.waitForState(job, JobState.SUCCEEDED);


      // make sure all events are flushed
      app.waitForState(Service.STATE.STOPPED);
      HistoryFileManager hfm = new HistoryFileManager();
      hfm.init(conf);
      HistoryFileInfo fileInfo = hfm.getFileInfo(jobId);
      hfm.initExisting();
      // wait for move files form the done_intermediate directory to the gone

View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.app.MRApp

          .setClass(
              NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
              MyResolver.class, DNSToSwitchMapping.class);


      RackResolver.init(configuration);
      MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
          true);
      app.submit(configuration);
      Job job = app.getContext().getAllJobs().values().iterator().next();
      app.waitForState(job, JobState.SUCCEEDED);


      JobHistory jobHistory = new JobHistory();
      jobHistory.init(configuration);
      // Method getAllJobs
      Assert.assertEquals(1, jobHistory.getAllJobs().size());
      // and with ApplicationId
      Assert.assertEquals(1, jobHistory.getAllJobs(app.getAppID()).size());


      JobsInfo jobsinfo = jobHistory.getPartialJobs(0L, 10L, null, "default",
          0L, System.currentTimeMillis() + 1, 0L,
          System.currentTimeMillis() + 1, JobState.SUCCEEDED);

View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.app.MRApp

                         appContext, params);
  }


  @Test
  public void testMRWebAppSSLDisabled() throws Exception {
    MRApp app = new MRApp(2, 2, true, this.getClass().getName(), true) {
      @Override
      protected ClientService createClientService(AppContext context) {
        return new MRClientService(context);
      }
    };
    Configuration conf = new Configuration();
    // MR is explicitly disabling SSL, even though setting as HTTPS_ONLY
    conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, Policy.HTTPS_ONLY.name());
    Job job = app.submit(conf);


    String hostPort =
        NetUtils.getHostPortString(((MRClientService) app.getClientService())
          .getWebApp().getListenerAddress());
    // http:// should be accessible
    URL httpUrl = new URL("http://" + hostPort);
    HttpURLConnection conn = (HttpURLConnection) httpUrl.openConnection();
    InputStream in = conn.getInputStream();
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    IOUtils.copyBytes(in, out, 1024);
    Assert.assertTrue(out.toString().contains("MapReduce Application"));


    // https:// is not accessible.
    URL httpsUrl = new URL("https://" + hostPort);
    try {
      HttpURLConnection httpsConn =
          (HttpURLConnection) httpsUrl.openConnection();
      httpsConn.getInputStream();
      Assert.fail("https:// is not accessible, expected to fail");
    } catch (Exception e) {
      Assert.assertTrue(e instanceof SSLException);
    }


    app.waitForState(job, JobState.SUCCEEDED);
    app.verifyCompleted();
  }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.app.MRApp

  public void testMRWebAppRedirection() throws Exception {


    String[] schemePrefix =
        { WebAppUtils.HTTP_PREFIX, WebAppUtils.HTTPS_PREFIX };
    for (String scheme : schemePrefix) {
      MRApp app = new MRApp(2, 2, true, this.getClass().getName(), true) {
        @Override
        protected ClientService createClientService(AppContext context) {
          return new MRClientService(context);
        }
      };
      Configuration conf = new Configuration();
      conf.set(YarnConfiguration.PROXY_ADDRESS, "9.9.9.9");
      conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, scheme
        .equals(WebAppUtils.HTTPS_PREFIX) ? Policy.HTTPS_ONLY.name()
          : Policy.HTTP_ONLY.name());
      webProxyBase = "/proxy/" + app.getAppID();
      conf.set("hadoop.http.filter.initializers",
        TestAMFilterInitializer.class.getName());
      Job job = app.submit(conf);
      String hostPort =
          NetUtils.getHostPortString(((MRClientService) app.getClientService())
            .getWebApp().getListenerAddress());
      URL httpUrl = new URL("http://" + hostPort + "/mapreduce");


      HttpURLConnection conn = (HttpURLConnection) httpUrl.openConnection();
      conn.setInstanceFollowRedirects(false);
      conn.connect();
      String expectedURL =
          scheme + conf.get(YarnConfiguration.PROXY_ADDRESS)
              + ProxyUriUtils.getPath(app.getAppID(), "/mapreduce");
      Assert.assertEquals(expectedURL,
        conn.getHeaderField(HttpHeaders.LOCATION));
      Assert.assertEquals(HttpStatus.SC_MOVED_TEMPORARILY,
        conn.getResponseCode());
      app.waitForState(job, JobState.SUCCEEDED);
      app.verifyCompleted();
    }
  }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.app.MRApp

  private static final Log LOG = LogFactory.getLog(TestJobHistoryEvents.class);


  @Test
  public void testHistoryEvents() throws Exception {
    Configuration conf = new Configuration();
    MRApp app = new MRAppWithHistory(2, 1, true, this.getClass().getName(), true);
    app.submit(conf);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    JobId jobId = job.getID();
    LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
    app.waitForState(job, JobState.SUCCEEDED);
    
    //make sure all events are flushed 
    app.waitForState(Service.STATE.STOPPED);
    /*
     * Use HistoryContext to read logged events and verify the number of 
     * completed maps 
    */
    HistoryContext context = new JobHistory();

View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.app.MRApp

   */
  @Test
  public void testEventsFlushOnStop() throws Exception {


    Configuration conf = new Configuration();
    MRApp app = new MRAppWithSpecialHistoryHandler(1, 0, true, this
        .getClass().getName(), true);
    app.submit(conf);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    JobId jobId = job.getID();
    LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
    app.waitForState(job, JobState.SUCCEEDED);


    // make sure all events are flushed
    app.waitForState(Service.STATE.STOPPED);
    /*
     * Use HistoryContext to read logged events and verify the number of
     * completed maps
     */
    HistoryContext context = new JobHistory();

View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.app.MRApp

        parsedJob.getState());
  }


  @Test
  public void testJobHistoryEventHandlerIsFirstServiceToStop() {
    MRApp app = new MRAppWithSpecialHistoryHandler(1, 0, true, this
        .getClass().getName(), true);
    Configuration conf = new Configuration();
    app.init(conf);
    Service[] services = app.getServices().toArray(new Service[0]);
    // Verifying that it is the last to be added is same as verifying that it is
    // the first to be stopped. CompositeService related tests already validate
    // this.
    Assert.assertEquals("JobHistoryEventHandler",
        services[services.length - 1].getName());

View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.app.MRApp

  }
  
  @Test
  public void testAssignedQueue() throws Exception {
    Configuration conf = new Configuration();
    MRApp app = new MRAppWithHistory(2, 1, true, this.getClass().getName(),
        true, "assignedQueue");
    app.submit(conf);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    JobId jobId = job.getID();
    LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
    app.waitForState(job, JobState.SUCCEEDED);
    
    //make sure all events are flushed 
    app.waitForState(Service.STATE.STOPPED);
    /*
     * Use HistoryContext to read logged events and verify the number of 
     * completed maps 
    */
    HistoryContext context = new JobHistory();

View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.app.MRApp

@SuppressWarnings("unchecked")
public class TestTaskAttempt{


  @Test
  public void testMRAppHistoryForMap() throws Exception {
    MRApp app = new FailingAttemptsMRApp(1, 0);
    testMRAppHistory(app);
  }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.v2.app.MRApp

    testMRAppHistory(app);
  }


  @Test
  public void testMRAppHistoryForReduce() throws Exception {
    MRApp app = new FailingAttemptsMRApp(0, 1);
    testMRAppHistory(app);
  }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.