Examples of TaskAttemptID


Examples of org.apache.hadoop.mapred.TaskAttemptID

      // In case of DefaultTaskController, set permissions here.
      FileUtil.chmod(executable, "u+x");
    }
    cmd.add(executable);
    // wrap the command in a stdout/stderr capture
    TaskAttemptID taskid = TaskAttemptID.forName(conf.get("mapred.task.id"));
    // we are starting map/reduce task of the pipes job. this is not a cleanup
    // attempt.
    File stdout = TaskLog.getTaskLogFile(taskid, false, TaskLog.LogName.STDOUT);
    File stderr = TaskLog.getTaskLogFile(taskid, false, TaskLog.LogName.STDERR);
    long logLength = TaskLog.getTaskLogLength(conf);
View Full Code Here

Examples of org.apache.hadoop.mapred.TaskAttemptID

      Path path = fileStatus.getPath();
      String name = path.getName();
      boolean taskComplete = name.endsWith(".task_complete");
      if (fileStatus.isDir()) {
        String taskAttemptName = getTaskAttemptName(name);
        TaskAttemptID taskAttemptID = TaskAttemptID.forName(taskAttemptName);
        if (taskAttemptID.getJobID().equals(jobContext.getJobID())) {
          if (commit) {
            if (taskComplete) {
              fileSystem.rename(path, new Path(shardPath, taskAttemptName + ".commit"));
              LOG.info("Committing [{0}] in path [{1}]", taskAttemptID, path);
            } else {
View Full Code Here

Examples of org.apache.hadoop.mapreduce.TaskAttemptID

  public void map(Text key, Text val, Context context)
      throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();

    UUID randomUuid = UUID.randomUUID();
    TaskAttemptID taskAttemptId = context.getTaskAttemptID();
    Long mostSignificant = (long) taskAttemptId.hashCode();
    Long leastSignificant = randomUuid.getLeastSignificantBits();
    UUID uniquieId = new UUID(mostSignificant, leastSignificant);

    String delimeter = conf.get(DELIMITER_CONF);
    String value = val.toString();
View Full Code Here

Examples of org.apache.hadoop.mapreduce.TaskAttemptID

     */
    public MockReduceContext(final Configuration configuration,
        final List<Pair<KEYIN, List<VALUEIN>>> in,
        final Counters counters) throws IOException, InterruptedException {
      super(configuration,
            new TaskAttemptID("mrunit-jt", 0, false, 0, 0),
            new MockRawKeyValueIterator(), null, null, null,
            new MockOutputCommitter(), new MockReporter(counters), null,
            (Class) Text.class, (Class) Text.class);
      this.inputIter = in.iterator();
      this.output = new MockOutputCollector<KEYOUT, VALUEOUT>();
View Full Code Here

Examples of org.apache.hadoop.mapreduce.TaskAttemptID

    public MockMapContext(final Configuration configuration,
        final List<Pair<KEYIN, VALUEIN>> in,
        final Counters counters) throws IOException, InterruptedException {

      super(configuration,
            new TaskAttemptID("mrunit-jt", 0, true, 0, 0),
            null, null, new MockOutputCommitter(), new MockReporter(counters), null);
      this.inputIter = in.iterator();
      this.output = new MockOutputCollector<KEYOUT, VALUEOUT>();
    }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.TaskAttemptID

   * @return masked {@link TaskAttemptID} with empty {@link JobID}.
   */
  private TaskAttemptID maskAttemptID(TaskAttemptID attemptId) {
    JobID jobId = new JobID();
    TaskID taskId = attemptId.getTaskID();
    return new TaskAttemptID(jobId.getJtIdentifier(), jobId.getId(),
        attemptId.isMap(), taskId.getId(), attemptId.getId());
  }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.TaskAttemptID

          loggedTaskMap.put(maskTaskID(TaskID.forName(map.taskID)), map);

          for (LoggedTaskAttempt mapAttempt : map.getAttempts()) {
            mapAttempt = sanitizeLoggedTaskAttempt(mapAttempt);
            if (mapAttempt != null) {
              TaskAttemptID id = TaskAttemptID.forName(mapAttempt
                  .getAttemptID());
              loggedTaskAttemptMap.put(maskAttemptID(id), mapAttempt);
            }
          }
        }
      }
      for (LoggedTask reduce : job.getReduceTasks()) {
        reduce = sanitizeLoggedTask(reduce);
        if (reduce != null) {
          loggedTaskMap.put(maskTaskID(TaskID.forName(reduce.taskID)), reduce);

          for (LoggedTaskAttempt reduceAttempt : reduce.getAttempts()) {
            reduceAttempt = sanitizeLoggedTaskAttempt(reduceAttempt);
            if (reduceAttempt != null) {
              TaskAttemptID id = TaskAttemptID.forName(reduceAttempt
                  .getAttemptID());
              loggedTaskAttemptMap.put(maskAttemptID(id), reduceAttempt);
            }
          }
        }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.TaskAttemptID

    return taskInfo;
  }

  private TaskAttemptID makeTaskAttemptID(TaskType taskType, int taskNumber,
      int taskAttemptNumber) {
    return new TaskAttemptID(new TaskID(JobID.forName(job.getJobID()),
        TaskType.MAP == taskType, taskNumber), taskAttemptNumber);
  }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.TaskAttemptID

  }

  private LoggedTaskAttempt getLoggedTaskAttempt(TaskType taskType,
      int taskNumber, int taskAttemptNumber) {
    buildMaps();
    TaskAttemptID id =
        new TaskAttemptID(getMaskedTaskID(taskType, taskNumber),
            taskAttemptNumber);
    return loggedTaskAttemptMap.get(id);
  }
View Full Code Here

Examples of org.apache.hadoop.mapreduce.TaskAttemptID

  public void testRecordReaderInit() throws InterruptedException, IOException {
    // Test that we properly initialize the child recordreader when
    // CombineFileInputFormat and CombineFileRecordReader are used.

    TaskAttemptID taskId = new TaskAttemptID("jt", 0, true, 0, 0);
    Configuration conf1 = new Configuration();
    conf1.set(DUMMY_KEY, "STATE1");
    TaskAttemptContext context1 = new TaskAttemptContext(conf1, taskId);

    // This will create a CombineFileRecordReader that itself contains a
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.