Examples of org.apache.hadoop.mapreduce.JobID

org.apache.hadoop.mapreduce.JobID
JobID represents the immutable and unique identifier for the job. JobID consists of two parts. First part represents the jobtracker identifier, so that jobID to jobtracker map is defined. For cluster setup this string is the jobtracker start time, for local setting, it is "local" and a random number. Second part of the JobID is the job number.
An example JobID is : job_200707121733_0003 , which represents the third job running at the jobtracker started at 200707121733.
Applications should never construct or parse JobID strings, but rather use appropriate constructors or {@link #forName(String)} method. @see TaskID @see TaskAttemptID

    return createJobContext(new Configuration());
  }
  
  public static JobContext createJobContext(Configuration conf) {
    try {
      return (JobContext) JOB_CONTEXT_CONSTRUCTOR.newInstance(conf, new JobID("local", 0));
    } catch (InstantiationException e) {
      throw new IllegalArgumentException("Can't create object", e);
    } catch (IllegalAccessException e) {
      throw new IllegalArgumentException("Can't create object", e);
    } catch (InvocationTargetException e) {

View Full Code Here

    String fileName = jhFileName.substring(0, jhFileName.indexOf(JobHistoryUtils.JOB_HISTORY_FILE_EXTENSION));
    JobIndexInfo indexInfo = new JobIndexInfo();
    
    String[] jobDetails = fileName.split(DELIMITER);
    
    JobID oldJobId = JobID.forName(decodeJobHistoryFileName(jobDetails[JOB_ID_INDEX]));
    JobId jobId = TypeConverter.toYarn(oldJobId);
    indexInfo.setJobId(jobId);


    // Do not fail if there are some minor parse errors
    try {

View Full Code Here

  @Test (timeout = 120000)
  public void testGetJobFileWithUser() {
    Configuration conf = new Configuration();
    conf.set(MRJobConfig.MR_AM_STAGING_DIR, "/my/path/to/staging");
    String jobFile = MRApps.getJobFile(conf, "dummy-user", 
        new JobID("dummy-job", 12345));
    assertNotNull("getJobFile results in null.", jobFile);
    assertEquals("jobFile with specified user is not as expected.",
        "/my/path/to/staging/dummy-user/.staging/job_dummy-job_12345/job.xml", jobFile);
  }

View Full Code Here

  private static final String JOB_START_TIME = "1317928742060";


  @Test
  public void testEncodingDecodingEquivalence() throws IOException {
    JobIndexInfo info = new JobIndexInfo();
    JobID oldJobId = JobID.forName(JOB_ID);
    JobId jobId = TypeConverter.toYarn(oldJobId);
    info.setJobId(jobId);
    info.setSubmitTime(Long.parseLong(SUBMIT_TIME));
    info.setUser(USER_NAME);
    info.setJobName(JOB_NAME);

View Full Code Here

  }


  @Test
  public void testUserNamePercentEncoding() throws IOException {
    JobIndexInfo info = new JobIndexInfo();
    JobID oldJobId = JobID.forName(JOB_ID);
    JobId jobId = TypeConverter.toYarn(oldJobId);
    info.setJobId(jobId);
    info.setSubmitTime(Long.parseLong(SUBMIT_TIME));
    info.setUser(USER_NAME_WITH_DELIMITER);
    info.setJobName(JOB_NAME);

View Full Code Here

  }


  @Test
  public void testJobNamePercentEncoding() throws IOException {
    JobIndexInfo info = new JobIndexInfo();
    JobID oldJobId = JobID.forName(JOB_ID);
    JobId jobId = TypeConverter.toYarn(oldJobId);
    info.setJobId(jobId);
    info.setSubmitTime(Long.parseLong(SUBMIT_TIME));
    info.setUser(USER_NAME);
    info.setJobName(JOB_NAME_WITH_DELIMITER);

View Full Code Here

  }


  @Test
  public void testQueueNamePercentEncoding() throws IOException {
    JobIndexInfo info = new JobIndexInfo();
    JobID oldJobId = JobID.forName(JOB_ID);
    JobId jobId = TypeConverter.toYarn(oldJobId);
    info.setJobId(jobId);
    info.setSubmitTime(Long.parseLong(SUBMIT_TIME));
    info.setUser(USER_NAME);
    info.setJobName(JOB_NAME);

View Full Code Here

    Assert.assertEquals(info.getJobStartTime(), info.getSubmitTime());
  }


  @Test
  public void testJobHistoryFileNameBackwardsCompatible() throws IOException {
    JobID oldJobId = JobID.forName(JOB_ID);
    JobId jobId = TypeConverter.toYarn(oldJobId);


    long submitTime = Long.parseLong(SUBMIT_TIME);
    long finishTime = Long.parseLong(FINISH_TIME);
    int numMaps = Integer.parseInt(NUM_MAPS);

View Full Code Here

  public void testConvertValueToTuple() throws IOException,InterruptedException{
    BytesRefArrayWritable[] bytesArr = initTestEnvironment();


    HCatSchema schema = buildHiveSchema();
    RCFileInputDriver sd = new RCFileInputDriver();
    JobContext jc = new JobContext(conf, new JobID());
    sd.setInputPath(jc, file.toString());
    InputFormat<?,?> iF = sd.getInputFormat(null);
    InputSplit split = iF.getSplits(jc).get(0);
    sd.setOriginalSchema(jc, schema);
    sd.setOutputSchema(jc, schema);

View Full Code Here


  public void testPruning() throws IOException,InterruptedException{
    BytesRefArrayWritable[] bytesArr = initTestEnvironment();


    RCFileInputDriver sd = new RCFileInputDriver();
    JobContext jc = new JobContext(conf, new JobID());
    sd.setInputPath(jc, file.toString());
    InputFormat<?,?> iF = sd.getInputFormat(null);
    InputSplit split = iF.getSplits(jc).get(0);
    sd.setOriginalSchema(jc, buildHiveSchema());
    sd.setOutputSchema(jc, buildPrunedSchema());

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapreduce.JobID

com.datasalt.pangool.tuplemr.mapred.lib.output.PangoolMultipleOutputs

com.datasalt.pangool.tuplemr.mapred.lib.output.ProxyOutputFormat$ProxyOutputCommitter

com.facebook.hiveio.benchmark.InputBenchmark

com.facebook.hiveio.input.HiveInput

com.facebook.hiveio.tailer.TailerCmd

com.twitter.elephantbird.mapreduce.input.TestIntegerListInputFormat

org.apache.accumulo.core.client.mapreduce.AccumuloInputFormatTest

org.apache.accumulo.core.client.mapreduce.lib.partition.RangePartitionerTest

org.apache.accumulo.core.util.ContextFactory

org.apache.accumulo.examples.simple.filedata.ChunkInputFormatTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.