Examples of org.apache.hadoop.mapreduce.v2.api.records.JobId

Package org.apache.hadoop.mapreduce.v2.api.records

Examples of org.apache.hadoop.mapreduce.v2.api.records.JobId

org.apache.hadoop.mapreduce.v2.api.records.JobId

JobId represents the globally unique identifier for a MapReduce job.

The globally unique nature of the identifier is achieved by using the cluster timestamp from the associated ApplicationId. i.e. start-time of the ResourceManager along with a monotonically increasing counter for the jobId.

    Assert.assertTrue(isUber);
  }


  private boolean testUberDecision(Configuration conf) {
    JobID jobID = JobID.forName("job_1234567890000_0001");
    JobId jobId = TypeConverter.toYarn(jobID);
    MRAppMetrics mrAppMetrics = MRAppMetrics.create();
    JobImpl job =
        new JobImpl(jobId, ApplicationAttemptId.newInstance(
          ApplicationId.newInstance(0, 0), 0), conf, mock(EventHandler.class),
          null, new JobTokenSecretManager(), new Credentials(), null, null,

View Full Code Here

    commitHandler.start();


    AppContext mockContext = mock(AppContext.class);
    when(mockContext.hasSuccessfullyUnregistered()).thenReturn(false);
    JobImpl job = createStubbedJob(conf, dispatcher, 2, mockContext);
    JobId jobId = job.getID();
    job.handle(new JobEvent(jobId, JobEventType.JOB_INIT));
    assertJobState(job, JobStateInternal.INITED);
    job.handle(new JobStartEvent(jobId));
    assertJobState(job, JobStateInternal.FAILED);

View Full Code Here

  static final String EXCEPTIONMSG = "Splits max exceeded";
  @Test
  public void testMetaInfoSizeOverMax() throws Exception {
    Configuration conf = new Configuration();
    JobID jobID = JobID.forName("job_1234567890000_0001");
    JobId jobId = TypeConverter.toYarn(jobID);
    MRAppMetrics mrAppMetrics = MRAppMetrics.create();
    JobImpl job =
        new JobImpl(jobId, ApplicationAttemptId.newInstance(
          ApplicationId.newInstance(0, 0), 0), conf, mock(EventHandler.class),
          null, new JobTokenSecretManager(), new Credentials(), null, null,

View Full Code Here

  }


  private static StubbedJob createStubbedJob(Configuration conf,
      Dispatcher dispatcher, int numSplits, AppContext appContext) {
    JobID jobID = JobID.forName("job_1234567890000_0001");
    JobId jobId = TypeConverter.toYarn(jobID);
    if (appContext == null) {
      appContext = mock(AppContext.class);
      when(appContext.hasSuccessfullyUnregistered()).thenReturn(true);
    }
    StubbedJob job = new StubbedJob(jobId,

View Full Code Here

        Token containerToken = newContainerToken(nodeId, "password".getBytes(),
              containerTokenIdentifier);
        Container container = Container.newInstance(cId, nodeId,
            NM_HOST + ":" + NM_HTTP_PORT, resource, null, containerToken);
        JobID id = TypeConverter.fromYarn(applicationId);
        JobId jobId = TypeConverter.toYarn(id);
        getContext().getEventHandler().handle(new JobHistoryEvent(jobId, 
            new NormalizedResourceEvent(
                org.apache.hadoop.mapreduce.TaskType.REDUCE,
            100)));
        getContext().getEventHandler().handle(new JobHistoryEvent(jobId,

View Full Code Here

      public Dispatcher createDispatcher() {
        return dispatcher;
      }
    };
    Job job = app.submit(new Configuration());
    JobId jobId = app.getJobId();
    app.waitForState(job, JobState.RUNNING);
    Assert.assertEquals("Num tasks not correct", 2, job.getTasks().size());
    Iterator<Task> it = job.getTasks().values().iterator();
    Task mapTask = it.next();
    Task reduceTask = it.next();

View Full Code Here

      public Dispatcher createDispatcher() {
        return dispatcher;
      }
    };
    Job job = app.submit(new Configuration());
    JobId jobId = app.getJobId();
    app.waitForState(job, JobState.RUNNING);
    Assert.assertEquals("Num tasks not correct", 2, job.getTasks().size());
    Iterator<Task> it = job.getTasks().values().iterator();
    Task mapTask = it.next();
    Task reduceTask = it.next();

View Full Code Here

     String user = UserGroupInformation.getCurrentUser().getShortUserName();
     Path stagingDir = MRApps.getStagingAreaDir(conf, user);
     when(fs.exists(stagingDir)).thenReturn(true);
     ApplicationId appId = ApplicationId.newInstance(0, 1);
     ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
     JobId jobid = recordFactory.newRecordInstance(JobId.class);
     jobid.setAppId(appId);
     TestMRApp appMaster = new TestMRApp(attemptId, null,
         JobStateInternal.RUNNING, maxAttempts);
     appMaster.crushUnregistration = true;
     appMaster.init(conf);
     appMaster.start();

View Full Code Here

     Path stagingDir = MRApps.getStagingAreaDir(conf, user);
     when(fs.exists(stagingDir)).thenReturn(true);
     ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(),
        0);
     ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
     JobId jobid = recordFactory.newRecordInstance(JobId.class);
     jobid.setAppId(appId);
     ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
     Assert.assertTrue(MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS > 1);
     MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc,
         JobStateInternal.RUNNING, MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS);
     appMaster.init(conf);

View Full Code Here

     Path stagingDir = MRApps.getStagingAreaDir(conf, user);
     when(fs.exists(stagingDir)).thenReturn(true);
     ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(),
         0);
     ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 0);
     JobId jobid = recordFactory.newRecordInstance(JobId.class);
     jobid.setAppId(appId);
     ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
     MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc, 4);
     appMaster.init(conf);
     //simulate the process being killed
     MRAppMaster.MRAppMasterShutdownHook hook =

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapreduce.v2.api.records.JobId

org.apache.hadoop.conf.Configuration

org.apache.hadoop.fs.FileSystem

org.apache.hadoop.fs.FSDataInputStream

org.apache.hadoop.fs.Path

org.apache.hadoop.fs.permission.FsPermission

org.apache.hadoop.mapred.JobConf

org.apache.hadoop.mapred.TaskAttemptListenerImpl

org.apache.hadoop.mapred.TestClientRedirect

org.apache.hadoop.mapred.TestClientRedirect$AMService

org.apache.hadoop.mapred.TestLocalContainerLauncher

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.