Examples of org.apache.hadoop.mapreduce.TaskAttemptID

org.apache.hadoop.mapreduce.TaskAttemptID
TaskAttemptID represents the immutable and unique identifier for a task attempt. Each task attempt is one particular instance of a Map or Reduce Task identified by its TaskID. TaskAttemptID consists of 2 parts. First part is the {@link TaskID}, that this TaskAttemptID belongs to. Second part is the task attempt number.
An example TaskAttemptID is : attempt_200707121733_0003_m_000005_0 , which represents the zeroth task attempt for the fifth map task in the third job running at the jobtracker started at 200707121733.
Applications should never construct or parse TaskAttemptID strings , but rather use appropriate constructors or {@link #forName(String)} method. @see JobID @see TaskID

            LOG.debug(MessageFormat.format(
                    "Progressable object is found (jobId={0}, object={1})",
                    jobContext.getJobID(),
                    progressable));
        }
        TaskAttemptID id = JobCompatibility.newTaskAttemptId(JobCompatibility.newTaskId(jobContext.getJobID()));
        return JobCompatibility.newTaskAttemptContext(jobContext.getConfiguration(), id, progressable);
    }

View Full Code Here

    conf.setInt(org.apache.hadoop.mapreduce.lib.input.
        LineRecordReader.MAX_LINE_LENGTH, Integer.MAX_VALUE);
    assertTrue("unexpected test data at " + testFile,
        testFileSize > firstSplitLength);


    TaskAttemptContext context = new TaskAttemptContextImpl(conf, new TaskAttemptID());


    // read the data without splitting to count the records
    FileSplit split = new FileSplit(testFilePath, 0, testFileSize,
        (String[])null);
    LineRecordReader reader = new LineRecordReader();

View Full Code Here

    File testFile = new File(testFileUrl.getFile());
    long testFileSize = testFile.length();
    Path testFilePath = new Path(testFile.getAbsolutePath());
    Configuration conf = new Configuration();
    conf.setInt("io.file.buffer.size", 1);
    TaskAttemptContext context = new TaskAttemptContextImpl(conf, new TaskAttemptID());


    // Gather the records returned by the record reader
    ArrayList<String> records = new ArrayList<String>();


    long offset = 0;

View Full Code Here

    long testFileSize = testFile.length();
    Configuration conf = new Configuration();
    conf.setInt(org.apache.hadoop.mapreduce.lib.input.
        LineRecordReader.MAX_LINE_LENGTH, Integer.MAX_VALUE);


    TaskAttemptContext context = new TaskAttemptContextImpl(conf, new TaskAttemptID());


    // read the data and check whether BOM is skipped
    FileSplit split = new FileSplit(testFilePath, 0, testFileSize,
        (String[])null);
    LineRecordReader reader = new LineRecordReader();

View Full Code Here

    //
    // Old Hadoop API
    //
    public org.apache.hadoop.mapred.InputSplit[] getSplits(JobConf jobConf, int numSplits) throws IOException
    {
        TaskAttemptContext tac = new TaskAttemptContext(jobConf, new TaskAttemptID());
        List<org.apache.hadoop.mapreduce.InputSplit> newInputSplits = this.getSplits(tac);
        org.apache.hadoop.mapred.InputSplit[] oldInputSplits = new org.apache.hadoop.mapred.InputSplit[newInputSplits.size()];
        for (int i = 0; i < newInputSplits.size(); i++)
            oldInputSplits[i] = (ColumnFamilySplit)newInputSplits.get(i);
        return oldInputSplits;

View Full Code Here

      }
    } catch (Exception e) {
      throw new IOException("Failed getting constructor", e);
    }
    try {
      return (TaskAttemptContext)c.newInstance(job.getConfiguration(), new TaskAttemptID());
    } catch (Exception e) {
      throw new IOException("Failed creating instance", e);
    }
  }

View Full Code Here


  @Override
  public RecordWriter<Text, BlurMutate> getRecordWriter(TaskAttemptContext context) throws IOException,
      InterruptedException {
    int id = context.getTaskAttemptID().getTaskID().getId();
    TaskAttemptID taskAttemptID = context.getTaskAttemptID();
    return new BlurRecordWriter(context.getConfiguration(), id, taskAttemptID.toString() + ".tmp");
  }

View Full Code Here

        POStore store = new POStore(new OperatorKey());
        store.setSFile(new FileSpec(file, storeFuncSpec));
        PigOutputFormat.setLocation(jc, store);
        OutputCommitter oc;
        // create a simulated TaskAttemptContext
        TaskAttemptContext tac = new TaskAttemptContext(conf, new TaskAttemptID());
        PigOutputFormat.setLocation(tac, store);
        RecordWriter<?,?> rw ;
        try {
            of.checkOutputSpecs(jc);
            oc = of.getOutputCommitter(tac);

View Full Code Here

        if(reader != null){
            reader.close();
        }
        InputSplit curSplit = inpSplits.get(curSplitIndex);
        TaskAttemptContext tAContext = new TaskAttemptContext(conf, 
                new TaskAttemptID());
        reader = inputFormat.createRecordReader(curSplit, tAContext);
        reader.initialize(curSplit, tAContext);
        // create a dummy pigsplit - other than the actual split, the other
        // params are really not needed here where we are just reading the
        // input completely

View Full Code Here


  private TaskAttemptContext getTestTaskAttemptContext(final Job job)
  throws IOException, Exception {
    TaskAttemptContext context;
    if (isPost020MapReduce()) {
      TaskAttemptID id =
        TaskAttemptID.forName("attempt_200707121733_0001_m_000000_0");
      Class<?> clazz =
        Class.forName("org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl");
      Constructor<?> c = clazz.
          getConstructor(Configuration.class, TaskAttemptID.class);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapreduce.TaskAttemptID

com.digitalpebble.behemoth.io.sequencefile.SequenceFileConverterMapperTest

com.facebook.giraph.hive.input.benchmark.InputBenchmark

com.facebook.giraph.hive.tailer.Tailer

com.facebook.hiveio.benchmark.InputBenchmark

com.facebook.hiveio.input.HiveInput$RecordIterator

com.hadoop.mapreduce.TestLzoTextInputFormat

com.thinkaurelius.faunus.formats.graphson.GraphSONRecordReaderTest

com.tinkerpop.gremlin.giraph.hdfs.GiraphElementIterator

com.twitter.elephantbird.mapred.input.DeprecatedInputFormatWrapper$RecordReaderWrapper

com.twitter.elephantbird.pig.load.TestRCFileProtobufStorage

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.