Package org.apache.hadoop

Examples of org.apache.hadoop.SleepJob$SleepInputFormat

442
443
444
445
446
447
448
449
450
451
452
453
454
      LOG.info("Token is " + str.encodeToUrlString());
    }
    user.doAs(new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception { 
        SleepJob sleepJob = new SleepJob();
        sleepJob.setConf(mrCluster.getConfig());
        Job job = sleepJob.createJob(3, 0, 10000, 1, 0, 0);
        // //Job with reduces
        // Job job = sleepJob.createJob(3, 2, 10000, 1, 10000, 1);
        job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
        job.submit();
        String trackingUrl = job.getTrackingURL();
View Full Code Here
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
      LOG.info("MRAppJar " + MiniMRYarnCluster.APPJAR
          + " not found. Not running test.");
      return;
    }

    final SleepJob sleepJob = new SleepJob();
    final JobConf sleepConf = new JobConf(mrCluster.getConfig());
    sleepConf.set(MRJobConfig.MAP_LOG_LEVEL, Level.ALL.toString());
    final long userLogKb = 4;
    sleepConf.setLong(MRJobConfig.TASK_USERLOG_LIMIT, userLogKb);
    sleepConf.setInt(MRJobConfig.TASK_LOG_BACKUPS, 3);
    sleepConf.set(MRJobConfig.MR_AM_LOG_LEVEL, Level.ALL.toString());
    final long amLogKb = 7;
    sleepConf.setLong(MRJobConfig.MR_AM_LOG_KB, amLogKb);
    sleepConf.setInt(MRJobConfig.MR_AM_LOG_BACKUPS, 7);
    sleepJob.setConf(sleepConf);

    final Job job = sleepJob.createJob(1, 0, 1L, 100, 0L, 0);
    job.setJarByClass(SleepJob.class);
    job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
    job.waitForCompletion(true);
    final JobId jobId = TypeConverter.toYarn(job.getJobID());
    final ApplicationId appID = jobId.getAppId();
View Full Code Here
117
118
119
120
121
122
123
124
125
126
127
128
129
130
      return;
    }


   
    SleepJob sleepJob = new SleepJob();
    sleepJob.setConf(mrCluster.getConfig());
    // Job with 3 maps and 2 reduces
    Job job = sleepJob.createJob(3, 2, 1000, 1, 500, 1);
    job.setJarByClass(SleepJob.class);
    job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
    job.waitForCompletion(true);
    Counters counterMR = job.getCounters();
    JobId jobId = TypeConverter.toYarn(job.getJobID());
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
103
104
    JobConf jobConf = new JobConf(mrCluster.getConfig());
    jobConf.setInt("mapreduce.map.memory.mb", 700);
    jobConf.setInt("mapred.reduce.memory.mb", 1500);

    SleepJob sleepJob = new SleepJob();
    sleepJob.setConf(jobConf);
    Job job = sleepJob.createJob(3, 2, 1000, 1, 500, 1);
    job.setJarByClass(SleepJob.class);
    job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
    job.submit();
    boolean completed = job.waitForCompletion(true);
    Assert.assertTrue("Job should be completed", completed);
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
131
132
133
      return;
    }


   
    SleepJob sleepJob = new SleepJob();
    sleepJob.setConf(mrCluster.getConfig());
    // Job with 3 maps and 2 reduces
    Job job = sleepJob.createJob(3, 2, 1000, 1, 500, 1);
    job.setJarByClass(SleepJob.class);
    job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
    job.waitForCompletion(true);
    Counters counterMR = job.getCounters();
    JobId jobId = TypeConverter.toYarn(job.getJobID());
View Full Code Here
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
    Configuration sleepConf = new Configuration(mrCluster.getConfig());
    // set master address to local to test that local mode applied iff framework == local
    sleepConf.set(MRConfig.MASTER_ADDRESS, "local"); 
   
    SleepJob sleepJob = new SleepJob();
    sleepJob.setConf(sleepConf);

    int numReduces = sleepConf.getInt("TestMRJobs.testSleepJob.reduces", 2); // or sleepConf.getConfig().getInt(MRJobConfig.NUM_REDUCES, 2);
  
    // job with 3 maps (10s) and numReduces reduces (5s), 1 "record" each:
    Job job = sleepJob.createJob(3, numReduces, 10000, 1, 5000, 1);

    job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
    job.setJarByClass(SleepJob.class);
    job.setMaxMapAttempts(1); // speed up failures
    job.waitForCompletion(true);
View Full Code Here
352
353
354
355
356
357
358
359
360
361
362
363
364
      LOG.info("Token is " + str.encodeToUrlString());
    }
    user.doAs(new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception { 
        SleepJob sleepJob = new SleepJob();
        sleepJob.setConf(mrCluster.getConfig());
        Job job = sleepJob.createJob(3, 0, 10000, 1, 0, 0);
        // //Job with reduces
        // Job job = sleepJob.createJob(3, 2, 10000, 1, 10000, 1);
        job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
        job.waitForCompletion(true);
        Assert.assertEquals(JobStatus.State.SUCCEEDED, job.getJobState());
View Full Code Here
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
    Configuration sleepConf = new Configuration(mrCluster.getConfig());
    // set master address to local to test that local mode applied iff framework == local
    sleepConf.set(MRConfig.MASTER_ADDRESS, "local"); 
   
    SleepJob sleepJob = new SleepJob();
    sleepJob.setConf(sleepConf);

    int numReduces = sleepConf.getInt("TestMRJobs.testSleepJob.reduces", 2); // or sleepConf.getConfig().getInt(MRJobConfig.NUM_REDUCES, 2);
  
    // job with 3 maps (10s) and numReduces reduces (5s), 1 "record" each:
    Job job = sleepJob.createJob(3, numReduces, 10000, 1, 5000, 1);

    job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
    job.setJarByClass(SleepJob.class);
    job.setMaxMapAttempts(1); // speed up failures
    job.submit();
View Full Code Here
372
373
374
375
376
377
378
379
380
381
382
383
384
      LOG.info("Token is " + str.encodeToUrlString());
    }
    user.doAs(new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception { 
        SleepJob sleepJob = new SleepJob();
        sleepJob.setConf(mrCluster.getConfig());
        Job job = sleepJob.createJob(3, 0, 10000, 1, 0, 0);
        // //Job with reduces
        // Job job = sleepJob.createJob(3, 2, 10000, 1, 10000, 1);
        job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
        job.submit();
        String trackingUrl = job.getTrackingURL();
View Full Code Here
TOP

Related Classes of org.apache.hadoop.SleepJob$SleepInputFormat

  • org.apache.hadoop.conf.Configuration
  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.io.IntWritable
  • org.apache.hadoop.io.LongWritable
  • org.apache.hadoop.mapreduce.Job
  • org.apache.hadoop.mapreduce.v2.TestMRAMWithNonNormalizedCapabilities
  • org.apache.hadoop.mapreduce.v2.TestMRJobs
  • org.apache.hadoop.mapreduce.v2.TestMRJobsWithHistoryService
  • org.apache.hadoop.mapreduce.v2.TestMRJobsWithProfiler
  • java.io.IOException
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.