List of needsTaskCommit() Examples

Examples of needsTaskCommit()

org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()
Check whether task needs a commit. This is called from each individual task's process that will output to HDFS, and it is called just for that task. @param taskContext @return true/false @throws IOException
org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter.needsTaskCommit()
Did this task write any files in the work directory? @param context the task's context

Examples of org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()

            }
            rw.close(tac);
        } catch (InterruptedException e) {
            throw new IOException(e);
        }
        if(oc.needsTaskCommit(tac)) {
            oc.commitTask(tac);
        }
        oc.cleanupJob(jc);
    }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()

      while(recordItr.hasNext()){
        HCatRecord rec = recordItr.next();
        writer.write(null, rec);
      }
      writer.close(cntxt);
      if(committer.needsTaskCommit(cntxt)){
        committer.commitTask(cntxt);        
      }
    } catch (IOException e) {
      if(null != committer) {
        try {

View Full Code Here

Examples of org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()

                bwriter.close(reporter);
            }
            for(Map.Entry<String,org.apache.hadoop.mapred.OutputCommitter>entry : baseDynamicCommitters.entrySet()) {
                org.apache.hadoop.mapred.TaskAttemptContext currContext = dynamicContexts.get(entry.getKey());
                OutputCommitter baseOutputCommitter = entry.getValue();
                if (baseOutputCommitter.needsTaskCommit(currContext)){
                    baseOutputCommitter.commitTask(currContext);
                }
            }
        } else {
            getBaseRecordWriter().close(reporter);

View Full Code Here

Examples of org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()

                    baseOutputCommitter.setupTask(taskContext);
                }


                @Override
                public boolean needsTaskCommit(TaskAttemptContext taskContext) throws IOException {
                    return baseOutputCommitter.needsTaskCommit(taskContext);
                }


                @Override
                public void commitTask(TaskAttemptContext taskContext) throws IOException {
                    baseOutputCommitter.commitTask(taskContext);

View Full Code Here

Examples of org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()

          baseOutputCommitter.setupTask(taskContext);
        }


        @Override
        public boolean needsTaskCommit(TaskAttemptContext taskContext) throws IOException {
          return baseOutputCommitter.needsTaskCommit(taskContext);
        }


        @Override
        public void commitTask(TaskAttemptContext taskContext) throws IOException {
          baseOutputCommitter.commitTask(taskContext);

View Full Code Here

Examples of org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()

    public void commitTask(TaskAttemptContext taskContext) throws IOException {
      for (String alias : outputCommitters.keySet()) {
        BaseOutputCommitterContainer outputContainer = outputCommitters.get(alias);
        OutputCommitter baseCommitter = outputContainer.getBaseCommitter();
        TaskAttemptContext committerContext = outputContainer.getContext();
        if (baseCommitter.needsTaskCommit(committerContext)) {
          LOGGER.info("Calling commitTask for alias: " + alias);
          baseCommitter.commitTask(committerContext);
        }
      }
    }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()

    public void commitTask(TaskAttemptContext taskContext) throws IOException {
      for (String alias : outputCommitters.keySet()) {
        BaseOutputCommitterContainer outputContainer = outputCommitters.get(alias);
        OutputCommitter baseCommitter = outputContainer.getBaseCommitter();
        TaskAttemptContext committerContext = outputContainer.getContext();
        if (baseCommitter.needsTaskCommit(committerContext)) {
          LOGGER.info("Calling commitTask for alias: " + alias);
          baseCommitter.commitTask(committerContext);
        }
      }
    }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()

      while (recordItr.hasNext()) {
        HCatRecord rec = recordItr.next();
        writer.write(null, rec);
      }
      writer.close(cntxt);
      if (committer.needsTaskCommit(cntxt)) {
        committer.commitTask(cntxt);
      }
    } catch (IOException e) {
      if (null != committer) {
        try {

View Full Code Here

Examples of org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()

      while (recordItr.hasNext()) {
        HCatRecord rec = recordItr.next();
        writer.write(null, rec);
      }
      writer.close(cntxt);
      if (committer.needsTaskCommit(cntxt)) {
        committer.commitTask(cntxt);
      }
    } catch (IOException e) {
      if (null != committer) {
        try {

View Full Code Here

Examples of org.apache.hadoop.mapreduce.OutputCommitter.needsTaskCommit()

    public void commitTask(TaskAttemptContext taskContext) throws IOException {
      for (String alias : outputCommitters.keySet()) {
        BaseOutputCommitterContainer outputContainer = outputCommitters.get(alias);
        OutputCommitter baseCommitter = outputContainer.getBaseCommitter();
        TaskAttemptContext committerContext = outputContainer.getContext();
        if (baseCommitter.needsTaskCommit(committerContext)) {
          LOGGER.info("Calling commitTask for alias: " + alias);
          baseCommitter.commitTask(committerContext);
        }
      }
    }

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.