List of needsTaskCommit() Examples

Examples of needsTaskCommit()

org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()
Check whether task needs a commit. This is called from each individual task's process that will output to HDFS, and it is called just for that task. @param taskContext @return true/false @throws IOException
org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter.needsTaskCommit()
Did this task write any files in the work directory? @param context the task's context

Examples of org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()

            }
            rw.close(tac);
        } catch (InterruptedException e) {
            throw new IOException(e);
        }
        if(oc.needsTaskCommit(tac)) {
            oc.commitTask(tac);
        }
        oc.cleanupJob(jc);
    }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()

      while (recordItr.hasNext()) {
        HCatRecord rec = recordItr.next();
        writer.write(null, rec);
      }
      writer.close(cntxt);
      if (committer.needsTaskCommit(cntxt)) {
        committer.commitTask(cntxt);
      }
    } catch (IOException e) {
      if (null != committer) {
        try {

View Full Code Here

Examples of org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()

          baseOutputCommitter.setupTask(taskContext);
        }


        @Override
        public boolean needsTaskCommit(TaskAttemptContext taskContext) throws IOException {
          return baseOutputCommitter.needsTaskCommit(taskContext);
        }


        @Override
        public void commitTask(TaskAttemptContext taskContext) throws IOException {
          baseOutputCommitter.commitTask(taskContext);

View Full Code Here

Examples of org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()

        for (RecordWriter<? super WritableComparable<?>, ? super Writable> bwriter : baseDynamicWriters.values()){
          bwriter.close(context);
        }
        for (HCatOutputStorageDriver osd : baseDynamicStorageDrivers.values()){
          OutputCommitter baseOutputCommitter = osd.getOutputFormat().getOutputCommitter(context);
          if (baseOutputCommitter.needsTaskCommit(context)){
            baseOutputCommitter.commitTask(context);
          }
        }
      } else {
        baseWriter.close(context);

View Full Code Here

Examples of org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()

                    baseOutputCommitter.setupTask(taskContext);
                }


                @Override
                public boolean needsTaskCommit(TaskAttemptContext taskContext) throws IOException {
                    return baseOutputCommitter.needsTaskCommit(taskContext);
                }


                @Override
                public void commitTask(TaskAttemptContext taskContext) throws IOException {
                    baseOutputCommitter.commitTask(taskContext);

View Full Code Here

Examples of org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()

        public void commitTask(TaskAttemptContext taskContext) throws IOException {
            for (String alias : outputCommitters.keySet()) {
                BaseOutputCommitterContainer outputContainer = outputCommitters.get(alias);
                OutputCommitter baseCommitter = outputContainer.getBaseCommitter();
                TaskAttemptContext committerContext = outputContainer.getContext();
                if (baseCommitter.needsTaskCommit(committerContext)) {
                    LOGGER.info("Calling commitTask for alias: " + alias);
                    baseCommitter.commitTask(committerContext);
                }
            }
        }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()

                bwriter.close(reporter);
            }
            for (Map.Entry<String, org.apache.hadoop.mapred.OutputCommitter> entry : baseDynamicCommitters.entrySet()) {
                org.apache.hadoop.mapred.TaskAttemptContext currContext = dynamicContexts.get(entry.getKey());
                OutputCommitter baseOutputCommitter = entry.getValue();
                if (baseOutputCommitter.needsTaskCommit(currContext)) {
                    baseOutputCommitter.commitTask(currContext);
                }
            }
        } else {
            getBaseRecordWriter().close(reporter);

View Full Code Here

Examples of org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()

            while (recordItr.hasNext()) {
                HCatRecord rec = recordItr.next();
                writer.write(null, rec);
            }
            writer.close(cntxt);
            if (committer.needsTaskCommit(cntxt)) {
                committer.commitTask(cntxt);
            }
        } catch (IOException e) {
            if (null != committer) {
                try {

View Full Code Here

Examples of org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()

    if (getConfiguration().isPureYarnJob() &&
      getConfiguration().getVertexOutputFormatClass() != null) {
      try {
        OutputCommitter outputCommitter =
          vertexOutputFormat.getOutputCommitter(getContext());
        if (outputCommitter.needsTaskCommit(getContext())) {
          LoggerUtils.setStatusAndLog(getContext(), LOG, Level.INFO,
            "OutputCommitter: committing task output.");
          // transfer from temp dirs to "task commit" dirs to prep for
          // the master's OutputCommitter#commitJob(context) call to finish.
          outputCommitter.commitTask(getContext());

View Full Code Here

Examples of org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()

            }
            rw.close(tac);
        } catch (InterruptedException e) {
            throw new IOException(e);
        }
        if(oc.needsTaskCommit(tac)) {
            oc.commitTask(tac);
        }
        HadoopShims.commitOrCleanup(oc, jc);
    }

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.