Examples of org.apache.hadoop.mapreduce.TaskID

org.apache.hadoop.mapreduce.TaskID
TaskID represents the immutable and unique identifier for a Map or Reduce Task. Each TaskID encompasses multiple attempts made to execute the Map or Reduce Task, each of which are uniquely indentified by their TaskAttemptID. TaskID consists of 3 parts. First part is the {@link JobID}, that this TaskInProgress belongs to. Second part of the TaskID is either 'm' or 'r' representing whether the task is a map task or a reduce task. And the third part is the task number.
An example TaskID is : task_200707121733_0003_m_000005 , which represents the fifth map task in the third job running at the jobtracker started at 200707121733.
Applications should never construct or parse TaskID strings , but rather use appropriate constructors or {@link #forName(String)} method. @see JobID @see TaskAttemptID

  protected void setup(Context context) throws java.io.IOException,
      InterruptedException {
    super.setup(context);
    debugInfo=0;
    doccount=0;
    TaskID taskId = context.getTaskAttemptID().getTaskID();
    this.Index = taskId.getId();


    context.getCounter("higo", "dumpcount").increment(0);


    Configuration conf = context.getConfiguration();
    isNotFdtMode=conf.get("mdrill.table.mode","").indexOf("@hdfs@")<0;

View Full Code Here


      if (!fs.exists(new Path(indexHdfsPath))) {
        fs.rename(new Path(tmpath), new Path(indexHdfsPath));
      }
      if (shardWriter.getNumDocs() > 0 && lastkey != null) {
        TaskID taskId = context.getTaskAttemptID().getTaskID();
        int partition = taskId.getId();
        System.out.println("###########>>>>"+partition);
        context.write(new IntWritable(partition),new Text(indexHdfsPath));
      }
      FileSystem lfs = FileSystem.getLocal(conf);
      if(lfs.exists(new Path(localtmpath)))

View Full Code Here

  protected void setup(Context context) throws IOException, InterruptedException {
    super.setup(context);
    Configuration conf = context.getConfiguration();
    this.res= new PriorityQueue<Clusterlist>(limit,Collections.reverseOrder(cmp));


  TaskID taskId = context.getTaskAttemptID().getTaskID();
  this.Index = taskId.getId()*10000;
  }

View Full Code Here

     */
    public static TaskID newTaskId(JobID jobId) {
        if (jobId == null) {
            throw new IllegalArgumentException("jobId must not be null"); //$NON-NLS-1$
        }
        return new TaskID(jobId, true, 0);
    }

View Full Code Here

          return store;
        }
          
        private DataOutputStream createOutputStream(String fieldValue) throws IOException {
          Configuration conf = ctx.getConfiguration();
          TaskID taskId = ctx.getTaskAttemptID().getTaskID();
          Path path = new Path(fieldValue, fieldValue + '-' 
                  + NumberFormat.getInstance().format(taskId.getId()));
          Path workOutputPath = ((FileOutputCommitter)getOutputCommitter(ctx)).getWorkPath();
          Path file = new Path(workOutputPath, path);
          FileSystem fs = file.getFileSystem(conf);                
          FSDataOutputStream fileOut = fs.create(file, false);
          return fileOut;

View Full Code Here


  public void initialize(DoFn<?, ?> fn, Integer tid) {
    if (context == null || !Objects.equal(lastTID, tid)) {
      TaskAttemptID attemptID;
      if (tid != null) {
        TaskID taskId = new TaskID(new JobID(jobName, 0), false, tid);
        attemptID = new TaskAttemptID(taskId, 0);
        lastTID = tid;
      } else {
        attemptID = new TaskAttemptID();
        lastTID = null;

View Full Code Here

   * @return masked {@link TaskID} with empty {@link JobID}.
   */
  private TaskID maskTaskID(TaskID taskId) {
    JobID jobId = new JobID();
    TaskType taskType = taskId.getTaskType();
    return new TaskID(jobId, taskType, taskId.getId());
  }

View Full Code Here

   * @return masked {@link TaskAttemptID} with empty {@link JobID}.
   */
  private TaskAttemptID maskAttemptID(TaskAttemptID attemptId) {
    JobID jobId = new JobID();
    TaskType taskType = attemptId.getTaskType();
    TaskID taskId = attemptId.getTaskID();
    return new TaskAttemptID(jobId.getJtIdentifier(), jobId.getId(), taskType,
        taskId.getId(), attemptId.getId());
  }

View Full Code Here

    return taskInfo;
  }


  private TaskAttemptID makeTaskAttemptID(TaskType taskType, int taskNumber,
      int taskAttemptNumber) {
    return new TaskAttemptID(new TaskID(JobID.forName(job.getJobID()),
        taskType, taskNumber), taskAttemptNumber);
  }

View Full Code Here

          : State.FAILED;
    }
  }


  private TaskID getMaskedTaskID(TaskType taskType, int taskNumber) {
    return new TaskID(new JobID(), taskType, taskNumber);
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapreduce.TaskID

co.nubetech.hiho.mapreduce.lib.output.AppendSequenceFileOutputFormat

co.nubetech.hiho.mapreduce.lib.output.AppendTextOutputFormat

com.alimama.mdrill.index.IndexMapper

com.alimama.mdrill.index.IndexReducer

com.alimama.mdrill.index.utils.JobIndexPublic

com.alimama.quanjingmonitor.kmeans.KMeansGroupReducer

com.asakusafw.runtime.compatibility.JobCompatibility

com.datasalt.pangool.solr.SolrRecordWriter

com.datasalt.pangool.tuplemr.mapred.lib.output.TestTupleInputOutputFormat

com.mozilla.pig.storage.SeqFileMultiStorage$MultiStorageSequenceOutputFormat

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.