Package org.apache.hadoop.mapreduce.v2.app

Examples of org.apache.hadoop.mapreduce.v2.app.MRApp

134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
            .setClass(
                    CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
                    MyResolver.class, DNSToSwitchMapping.class);

    RackResolver.init(config);
    MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
            true);
    app.submit(config);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    app.waitForState(job, JobState.SUCCEEDED);

    historyServer = new JobHistoryServer();

    historyServer.init(config);
    historyServer.start();
View Full Code Here
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
    long amStartTimeEst = System.currentTimeMillis();
    conf.setClass(
        CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
        MyResolver.class, DNSToSwitchMapping.class);
    RackResolver.init(conf);
    MRApp app =
        new MRAppWithHistory(numMaps, numReduces, true,
            this.getClass().getName(), true);
    app.submit(conf);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    JobId jobId = job.getID();
    LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
    app.waitForState(job, JobState.SUCCEEDED);

    // make sure all events are flushed
    app.waitForState(Service.STATE.STOPPED);

    String jobhistoryDir = JobHistoryUtils
        .getHistoryIntermediateDoneDirForUser(conf);
   
    FileContext fc = null;
View Full Code Here
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
    conf
        .setClass(
            CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
            MyResolver.class, DNSToSwitchMapping.class);
    RackResolver.init(conf);
    MRApp app = new MRAppWithHistoryWithFailedAttempt(2, 1, true, this.getClass().getName(),
        true);
    app.submit(conf);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    JobId jobId = job.getID();
    app.waitForState(job, JobState.SUCCEEDED);
   
    // make sure all events are flushed
    app.waitForState(Service.STATE.STOPPED);

    String jobhistoryDir = JobHistoryUtils
        .getHistoryIntermediateDoneDirForUser(conf);
    JobHistory jobHistory = new JobHistory();
    jobHistory.init(conf);
View Full Code Here
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
    conf
        .setClass(
            CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
            MyResolver.class, DNSToSwitchMapping.class);
    RackResolver.init(conf);
    MRApp app = new MRAppWithHistoryWithFailedTask(2, 1, true,
        this.getClass().getName(), true);
    app.submit(conf);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    JobId jobId = job.getID();
    app.waitForState(job, JobState.FAILED);

    // make sure all events are flushed
    app.waitForState(Service.STATE.STOPPED);

    String jobhistoryDir = JobHistoryUtils
        .getHistoryIntermediateDoneDirForUser(conf);
    JobHistory jobHistory = new JobHistory();
    jobHistory.init(conf);
View Full Code Here
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
    conf
        .setClass(
            CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
            MyResolver.class, DNSToSwitchMapping.class);
    RackResolver.init(conf);
    MRApp app =
        new MRAppWithHistory(1, 1, true,
            this.getClass().getName(), true);
    app.submit(conf);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    JobId jobId = job.getID();
    LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
    app.waitForState(job, JobState.SUCCEEDED);

    // make sure all events are flushed
    app.waitForState(Service.STATE.STOPPED);

    HistoryFileManagerForTest hfm = new HistoryFileManagerForTest();
    hfm.init(conf);
    HistoryFileInfo fileInfo = hfm.getFileInfo(jobId);
    Assert.assertNotNull("Unable to locate job history", fileInfo);
View Full Code Here
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
      conf.setClass(
          CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
          MyResolver.class, DNSToSwitchMapping.class);

      RackResolver.init(conf);
      MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
          true);
      app.submit(conf);
      Job job = app.getContext().getAllJobs().values().iterator().next();
      JobId jobId = job.getID();

      app.waitForState(job, JobState.SUCCEEDED);

      // make sure all events are flushed
      app.waitForState(Service.STATE.STOPPED);
      HistoryFileManager hfm = new HistoryFileManager();
      hfm.init(conf);
      HistoryFileInfo fileInfo = hfm.getFileInfo(jobId);
      hfm.initExisting();
      // wait for move files form the done_intermediate directory to the gone
View Full Code Here
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
      configuration.setClass(
              CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
              MyResolver.class, DNSToSwitchMapping.class);

      RackResolver.init(configuration);
      MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
          true);
      app.submit(configuration);
      Job job = app.getContext().getAllJobs().values().iterator().next();
      JobId jobId = job.getID();
      LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
      app.waitForState(job, JobState.SUCCEEDED);

      JobHistory jobHistory = new JobHistory();
      jobHistory.init(configuration);
      // Method getAllJobs
      Assert.assertEquals(1, jobHistory.getAllJobs().size());
      // and with ApplicationId
      Assert.assertEquals(1, jobHistory.getAllJobs(app.getAppID()).size());

      JobsInfo jobsinfo = jobHistory.getPartialJobs(0L, 10L, null, "default",
          0L, System.currentTimeMillis() + 1, 0L,
          System.currentTimeMillis() + 1, JobState.SUCCEEDED);
     
View Full Code Here
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
  private static final Log LOG = LogFactory.getLog(TestJobHistoryEvents.class);

  @Test
  public void testHistoryEvents() throws Exception {
    Configuration conf = new Configuration();
    MRApp app = new MRAppWithHistory(2, 1, true, this.getClass().getName(), true);
    app.submit(conf);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    JobId jobId = job.getID();
    LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
    app.waitForState(job, JobState.SUCCEEDED);
   
    //make sure all events are flushed
    app.waitForState(Service.STATE.STOPPED);
    /*
     * Use HistoryContext to read logged events and verify the number of
     * completed maps
    */
    HistoryContext context = new JobHistory();
View Full Code Here
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
   */
  @Test
  public void testEventsFlushOnStop() throws Exception {

    Configuration conf = new Configuration();
    MRApp app = new MRAppWithSpecialHistoryHandler(1, 0, true, this
        .getClass().getName(), true);
    app.submit(conf);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    JobId jobId = job.getID();
    LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
    app.waitForState(job, JobState.SUCCEEDED);

    // make sure all events are flushed
    app.waitForState(Service.STATE.STOPPED);
    /*
     * Use HistoryContext to read logged events and verify the number of
     * completed maps
     */
    HistoryContext context = new JobHistory();
View Full Code Here
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
        parsedJob.getState());
  }

  @Test
  public void testJobHistoryEventHandlerIsFirstServiceToStop() {
    MRApp app = new MRAppWithSpecialHistoryHandler(1, 0, true, this
        .getClass().getName(), true);
    Configuration conf = new Configuration();
    app.init(conf);
    Service[] services = app.getServices().toArray(new Service[0]);
    // Verifying that it is the last to be added is same as verifying that it is
    // the first to be stopped. CompositeService related tests already validate
    // this.
    Assert.assertEquals("JobHistoryEventHandler",
        services[services.length - 1].getName());
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapreduce.v2.app.MRApp

  • org.apache.hadoop.conf.Configuration
  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.FSDataInputStream
  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.fs.permission.FsPermission
  • org.apache.hadoop.io.Text
  • org.apache.hadoop.mapred.JobConf
  • org.apache.hadoop.mapred.TaskAttemptListenerImpl
  • org.apache.hadoop.mapred.TestClientRedirect
  • org.apache.hadoop.mapred.TestClientRedirect$AMService
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.