Package org.apache.hadoop.mapreduce

Examples of org.apache.hadoop.mapreduce.JobID


  @Override
  public void start() {
    scheduler= createSchedulerProxy();
    register();
    startAllocatorThread();
    JobID id = TypeConverter.fromYarn(this.applicationId);
    JobId jobId = TypeConverter.toYarn(id);
    job = context.getJob(jobId);
    super.start();
  }
View Full Code Here


    String user2 = user1 + "1234";
    UserGroupInformation ugi1 = UserGroupInformation.createRemoteUser(user1);
    UserGroupInformation ugi2 = UserGroupInformation.createRemoteUser(user2);

    // Create the job
    JobID jobID = JobID.forName("job_1234567890000_0001");
    JobId jobId = TypeConverter.toYarn(jobID);

    // Setup configuration access only to user1 (owner)
    Configuration conf1 = new Configuration();
    conf1.setBoolean(MRConfig.MR_ACLS_ENABLED, true);
View Full Code Here

        cId.setApplicationAttemptId(getContext().getApplicationAttemptId());
        cId.setId(containerCount++);
        NodeId nodeId = BuilderUtils.newNodeId(NM_HOST, NM_PORT);
        Container container = BuilderUtils.newContainer(cId, nodeId,
            NM_HOST + ":" + NM_HTTP_PORT, null, null, null);
        JobID id = TypeConverter.fromYarn(applicationId);
        JobId jobId = TypeConverter.toYarn(id);
        getContext().getEventHandler().handle(new JobHistoryEvent(jobId,
            new NormalizedResourceEvent(
                org.apache.hadoop.mapreduce.TaskType.REDUCE,
            100)));
View Full Code Here

  @Test public void testGetJobFileWithUser() {
    Configuration conf = new Configuration();
    conf.set(MRJobConfig.MR_AM_STAGING_DIR, "/my/path/to/staging");
    String jobFile = MRApps.getJobFile(conf, "dummy-user",
        new JobID("dummy-job", 12345));
    assertNotNull("getJobFile results in null.", jobFile);
    assertEquals("jobFile with specified user is not as expected.",
        "/my/path/to/staging/dummy-user/.staging/job_dummy-job_12345/job.xml", jobFile);
  }
View Full Code Here

        cId.setApplicationAttemptId(getContext().getApplicationAttemptId());
        cId.setId(containerCount++);
        NodeId nodeId = BuilderUtils.newNodeId(NM_HOST, NM_PORT);
        Container container = BuilderUtils.newContainer(cId, nodeId,
            NM_HOST + ":" + NM_HTTP_PORT, null, null, null);
        JobID id = TypeConverter.fromYarn(applicationId);
        JobId jobId = TypeConverter.toYarn(id);
        getContext().getEventHandler().handle(new JobHistoryEvent(jobId,
            new NormalizedResourceEvent(
                org.apache.hadoop.mapreduce.TaskType.REDUCE,
            100)));
View Full Code Here

    } else {
      sb.append(address.getHostName());
    }
    sb.append(":").append(address.getPort());
    sb.append("/jobhistory/job/");
    JobID jobId = TypeConverter.fromYarn(appId);
    sb.append(jobId.toString());
    return sb.toString();
  }
View Full Code Here

        throws IOException  {
    if (loc.exists()) {
        long now = clock.getTime();
        for(String logDir: loc.list()) {
          // add all the log dirs to taskLogsMnonitor.
          JobID jobid = null;
          try {
            jobid = JobID.forName(logDir);
          } catch (IllegalArgumentException ie) {
            deleteLogPath(logDir);
            continue;
View Full Code Here

    ts.addToken(new Text(nn2), token2);
    ts.addToken(new Text(nn3), token3);
   
    // register the tokens for renewal
    DelegationTokenRenewal.registerDelegationTokensForRenewal(
        new JobID("job1", 1), ts, conf);
    // first 3 initial renewals + 1 real
    int numberOfExpectedRenewals = 3+1;
   
    int attempts = 10;
    while(attempts-- > 0) {
      try {
        Thread.sleep(3*1000); // sleep 3 seconds, so it has time to renew
      } catch (InterruptedException e) {}
     
      // since we cannot guarantee timely execution - let's give few chances
      if(dfs.getCounter()==numberOfExpectedRenewals)
        break;
    }
   
    System.out.println("dfs=" + dfs.hashCode() +
        ";Counter = " + dfs.getCounter() + ";t="+  dfs.getToken());
    assertEquals("renew wasn't called as many times as expected(4):",
        numberOfExpectedRenewals, dfs.getCounter());
    assertEquals("most recently renewed token mismatch", dfs.getToken(),
        token1);
   
    // Test 2.
    // add another token ( that expires in 2 secs). Then remove it, before
    // time is up.
    // Wait for 3 secs , and make sure no renews were called
    ts = new Credentials();
    MyToken token4 = dfs.getDelegationToken(new Text("user4"));
   
    //to cause this one to be set for renew in 2 secs
    dfs.setTokenToRenewIn2Sec(token4);
    System.out.println("token="+token4+" should be renewed for 2 secs");
   
    String nn4 = DelegationTokenRenewal.SCHEME + "://host4:0";
    ts.addToken(new Text(nn4), token4);
   

    JobID jid2 = new JobID("job2",1);
    DelegationTokenRenewal.registerDelegationTokensForRenewal(jid2, ts, conf);
    DelegationTokenRenewal.removeDelegationTokenRenewalForJob(jid2);
    numberOfExpectedRenewals = dfs.getCounter(); // number of renewals so far
    try {
      Thread.sleep(6*1000); // sleep 6 seconds, so it has time to renew
View Full Code Here

   * @throws IOException
   *           Signals that an I/O exception has occurred.
   */
  @Test
  public void testMaxVersions() throws IOException {
    JobContext job = new JobContext(new Configuration(), new JobID());
    AccumuloInputFormat.setMaxVersions(job.getConfiguration(), 1);
    int version = AccumuloInputFormat.getMaxVersions(job.getConfiguration());
    assertEquals(1, version);
  }
View Full Code Here

   * @throws IOException
   *           Signals that an I/O exception has occurred.
   */
  @Test(expected = IOException.class)
  public void testMaxVersionsLessThan1() throws IOException {
    JobContext job = new JobContext(new Configuration(), new JobID());
    AccumuloInputFormat.setMaxVersions(job.getConfiguration(), 0);
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.mapreduce.JobID

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.