List of needsTaskCommit() Examples

Examples of needsTaskCommit()

org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()
Check whether task needs a commit. This is called from each individual task's process that will output to HDFS, and it is called just for that task. @param taskContext @return true/false @throws IOException
org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter.needsTaskCommit()
Did this task write any files in the work directory? @param context the task's context

Examples of org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()

          String dynKey = outputJobInfoEntry.getKey();
          OutputJobInfo outputJobInfo = outputJobInfoEntry.getValue();
          LOG.info("Committing task-attempt for " + outputJobInfo.getLocation());
          TaskAttemptContext dynContext = dynamicContexts.get(dynKey);
          OutputCommitter dynCommitter = baseDynamicCommitters.get(dynKey);
          if (dynCommitter.needsTaskCommit(dynContext)) {
            dynCommitter.commitTask(dynContext);
          }
          else {
            LOG.info("Skipping commitTask() for " + outputJobInfo.getLocation());
          }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()

      }


      @Override
      public boolean needsTaskCommit(
          TaskAttemptContext context) throws IOException {
        return outputCommitter.needsTaskCommit(
            HadoopUtils.makeTaskAttemptContext(getConf(), context));
      }


      @Override
      public void commitTask(TaskAttemptContext context) throws IOException {

View Full Code Here

Examples of org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()

      }


      @Override
      public boolean needsTaskCommit(
          TaskAttemptContext context) throws IOException {
        return outputCommitter.needsTaskCommit(
            HadoopUtils.makeTaskAttemptContext(getConf(), context));
      }


      @Override
      public void commitTask(TaskAttemptContext context) throws IOException {

View Full Code Here

Examples of org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()

    if (getConfiguration().isPureYarnJob() &&
      getConfiguration().getVertexOutputFormatClass() != null) {
      try {
        OutputCommitter outputCommitter =
          vertexOutputFormat.getOutputCommitter(getContext());
        if (outputCommitter.needsTaskCommit(getContext())) {
          LoggerUtils.setStatusAndLog(getContext(), LOG, Level.INFO,
            "OutputCommitter: committing task output.");
          // transfer from temp dirs to "task commit" dirs to prep for
          // the master's OutputCommitter#commitJob(context) call to finish.
          outputCommitter.commitTask(getContext());

View Full Code Here

Examples of org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()

    if (conf.isPureYarnJob() &&
      conf.getVertexOutputFormatClass() != null) {
      try {
        OutputCommitter outputCommitter =
          edgeOutputFormat.getOutputCommitter(getContext());
        if (outputCommitter.needsTaskCommit(getContext())) {
          LoggerUtils.setStatusAndLog(getContext(), LOG, Level.INFO,
            "OutputCommitter: committing task output.");
          // transfer from temp dirs to "task commit" dirs to prep for
          // the master's OutputCommitter#commitJob(context) call to finish.
          outputCommitter.commitTask(getContext());

View Full Code Here

Examples of org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()

        bwriter.close(reporter);
      }
      for (Map.Entry<String, org.apache.hadoop.mapred.OutputCommitter> entry : baseDynamicCommitters.entrySet()) {
        org.apache.hadoop.mapred.TaskAttemptContext currContext = dynamicContexts.get(entry.getKey());
        OutputCommitter baseOutputCommitter = entry.getValue();
        if (baseOutputCommitter.needsTaskCommit(currContext)) {
          baseOutputCommitter.commitTask(currContext);
        }
      }
    } else {
      getBaseRecordWriter().close(reporter);

View Full Code Here

Examples of org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()

        bwriter.close(reporter);
      }
      for (Map.Entry<String, org.apache.hadoop.mapred.OutputCommitter> entry : baseDynamicCommitters.entrySet()) {
        org.apache.hadoop.mapred.TaskAttemptContext currContext = dynamicContexts.get(entry.getKey());
        OutputCommitter baseOutputCommitter = entry.getValue();
        if (baseOutputCommitter.needsTaskCommit(currContext)) {
          baseOutputCommitter.commitTask(currContext);
        }
      }
    } else {
      getBaseRecordWriter().close(reporter);

View Full Code Here

Examples of org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()

      while (recordItr.hasNext()) {
        HCatRecord rec = recordItr.next();
        writer.write(null, rec);
      }
      writer.close(cntxt);
      if (committer.needsTaskCommit(cntxt)) {
        committer.commitTask(cntxt);
      }
    } catch (IOException e) {
      if (null != committer) {
        try {

View Full Code Here

Examples of org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()

        public void commitTask(TaskAttemptContext taskContext) throws IOException {
            for (String alias : outputCommitters.keySet()) {
                BaseOutputCommitterContainer outputContainer = outputCommitters.get(alias);
                OutputCommitter baseCommitter = outputContainer.getBaseCommitter();
                TaskAttemptContext committerContext = outputContainer.getContext();
                if (baseCommitter.needsTaskCommit(committerContext)) {
                    LOGGER.info("Calling commitTask for alias: " + alias);
                    baseCommitter.commitTask(committerContext);
                }
            }
        }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()

                bwriter.close(reporter);
            }
            for (Map.Entry<String, org.apache.hadoop.mapred.OutputCommitter> entry : baseDynamicCommitters.entrySet()) {
                org.apache.hadoop.mapred.TaskAttemptContext currContext = dynamicContexts.get(entry.getKey());
                OutputCommitter baseOutputCommitter = entry.getValue();
                if (baseOutputCommitter.needsTaskCommit(currContext)) {
                    baseOutputCommitter.commitTask(currContext);
                }
            }
        } else {
            getBaseRecordWriter().close(reporter);

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.