Examples of org.apache.hadoop.mapreduce.SleepJob$SleepJobPartitioner

org.apache.hadoop.mapreduce.SleepJob
Dummy class for testing MR framefork. Sleeps for a defined period of time in mapper and reducer. Generates fake input for map / reduce jobs. Note that generated number of input pairs is in the order of numMappers * mapSleepTime / 100, so the job uses some disk space.

      IOException {
    String[] args = { "-m", "0", "-r", "0", "-mt", "0", "-rt", "0" };
    boolean throwsException = false;
    String msg = null;
    try {
      ToolRunner.run(jobConf, new SleepJob(), args);
    } catch (RemoteException re) {
      throwsException = true;
      msg = re.unwrapRemoteException().getMessage();
    }
    assertTrue(throwsException);

View Full Code Here

      IOException {
    String[] args = { "-m", "0", "-r", "0", "-mt", "0", "-rt", "0" };
    boolean throwsException = false;
    String msg = null;
    try {
      ToolRunner.run(jobConf, new SleepJob(), args);
    } catch (RemoteException re) {
      throwsException = true;
      msg = re.unwrapRemoteException().getMessage();
    }
    assertTrue(throwsException);

View Full Code Here

  @Test
  public void testFailedTaskJobStatus()
      throws IOException, InterruptedException, ClassNotFoundException {
    Configuration conf = new Configuration(cluster.getConf());
    TaskInfo taskInfo = null;
    SleepJob job = new SleepJob();
    job.setConf(conf);
    Job slpJob = job.createJob(3, 1, 4000, 4000, 100, 100);
    JobConf jobConf = new JobConf(conf);
    jobConf.setMaxMapAttempts(20);
    jobConf.setMaxReduceAttempts(20);
    slpJob.submit();
    RunningJob runJob =

View Full Code Here

  public void testAllTaskAttemptKill() throws Exception {
    Configuration conf = new Configuration(cluster.getConf());


    JobStatus[] jobStatus = null;


    SleepJob job = new SleepJob();
    job.setConf(conf);
    Job slpJob = job.createJob(3, 1, 40000, 1000, 100, 100);
    JobConf jconf = new JobConf(conf);


    // Submitting the job
    slpJob.submit();
    RunningJob rJob =

View Full Code Here

  }


  @Test
  public void testJobSubmission() throws Exception {
    Configuration conf = new Configuration(cluster.getConf());
    SleepJob job = new SleepJob();
    job.setConf(conf);
    Job rJob = job.createJob(1, 1, 100, 100, 100, 100);
    rJob = cluster.getJTClient().submitAndVerifyJob(rJob);
    cluster.getJTClient().verifyJobHistory(rJob.getJobID());
  }

View Full Code Here

  @Test
  public void testTaskDetails() throws Exception {
    Configuration conf = new Configuration(cluster.getConf());
    JTProtocol wovenClient = cluster.getJTClient().getProxy();
    FinishTaskControlAction.configureControlActionForJob(conf);
    SleepJob job = new SleepJob();
    job.setConf(conf);


    Job rJob = job.createJob(1, 1, 100, 100, 100, 100);
    JobClient client = cluster.getJTClient().getClient();
    rJob.submit();
    RunningJob rJob1 =
        client.getJob(org.apache.hadoop.mapred.JobID.downgrade(rJob.getJobID()));
    JobID id = rJob.getJobID();

View Full Code Here

  }
  
  private void runSleepJob(JobConf conf) throws Exception {
    String[] args = { "-m", "1", "-r", "1",
                      "-mt", "10", "-rt", "10" };
    ToolRunner.run(conf, new SleepJob(), args);
  }

View Full Code Here

      ugi = UserGroupInformation.getCurrentUser();
    }
    if (queueName != null) {
      clientConf.set(JobContext.QUEUE_NAME, queueName);
    }
    final SleepJob sleep = new SleepJob();
    sleep.setConf(clientConf);
    
    Job job = ugi.doAs(new PrivilegedExceptionAction<Job>() {
        public Job run() throws IOException {
          return sleep.createJob(numMappers, numReducers, mapSleepTime,
              (int) mapSleepTime, reduceSleepTime, (int) reduceSleepTime);
      }});
    if (shouldComplete) {
      job.waitForCompletion(false);
    } else {

View Full Code Here

        getHttpStatusCode(url, modifyColleague, method));
  }


  // starts a sleep job with 1 map task that runs for a long time
  private Job startSleepJobAsUser(String user, JobConf conf) throws Exception {
    final SleepJob sleepJob = new SleepJob();
    sleepJob.setConf(conf);
    UserGroupInformation jobSubmitterUGI = 
    UserGroupInformation.createRemoteUser(user);
    Job job = jobSubmitterUGI.doAs(new PrivilegedExceptionAction<Job>() {
      public Job run() throws Exception {
        // Very large sleep job.
        Job job = sleepJob.createJob(1, 0, 900000, 1, 0, 0);
        job.submit();
        return job;
      }
    });
    return job;

View Full Code Here


    // Let us add group1 and group3 to modify-job-acl. So modifyColleague and
    // viewAndModifyColleague will be able to modify the job
    conf.set(MRJobConfig.JOB_ACL_MODIFY_JOB, " group1,group3");


    final SleepJob sleepJob = new SleepJob();
    sleepJob.setConf(conf);
    UserGroupInformation jobSubmitterUGI =
        UserGroupInformation.createRemoteUser(jobSubmitter);
    Job job = jobSubmitterUGI.doAs(new PrivilegedExceptionAction<Job>() {
      public Job run() throws Exception {
        // Very large sleep job.
        Job job = sleepJob.createJob(1, 0, 1000, 1, 0, 0);
        job.waitForCompletion(true);
        return job;
      }
    });

View Full Code Here

0 1 2 3 4

TOP

Related Classes of org.apache.hadoop.mapreduce.SleepJob$SleepJobPartitioner

org.apache.hadoop.conf.Configuration

org.apache.hadoop.fs.Path

org.apache.hadoop.io.IntWritable

org.apache.hadoop.mapred.QueueManagerTestUtils

org.apache.hadoop.mapred.TestCapacitySchedulerWithJobTracker

org.apache.hadoop.mapred.TestCluster

org.apache.hadoop.mapred.TestControlledJob

org.apache.hadoop.mapred.TestDistributedCacheModifiedFile

org.apache.hadoop.mapred.TestDistributedCachePrivateFile

org.apache.hadoop.mapred.TestDistributedCacheUnModifiedFile

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.