Examples of FileOutputCommitter

org.apache.hadoop.mapred.FileOutputCommitter
An {@link OutputCommitter} that commits files specified in job output directory i.e. ${mapred.output.dir}.
org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter
An {@link OutputCommitter} that commits files specified in job output directory i.e. ${mapreduce.output.fileoutputformat.outputdir}.

Examples of org.apache.hadoop.mapred.FileOutputCommitter

  public static class HBaseBulkOutputCommitter extends OutputCommitter {


    private final OutputCommitter baseOutputCommitter;


    public HBaseBulkOutputCommitter() {
      baseOutputCommitter = new FileOutputCommitter();
    }

View Full Code Here

Examples of org.apache.hadoop.mapred.FileOutputCommitter

    public void checkOutputSpecs(JobContext context) throws IOException, InterruptedException {
    }


    @Override
    public OutputCommitter getOutputCommitter(TaskAttemptContext context) throws IOException, InterruptedException {
        return new FileOutputCommitter();
    }

View Full Code Here

Examples of org.apache.hadoop.mapred.FileOutputCommitter

    public void checkOutputSpecs(JobContext context) throws IOException, InterruptedException {
    }


    @Override
    public OutputCommitter getOutputCommitter(TaskAttemptContext context) throws IOException, InterruptedException {
        return new FileOutputCommitter();
    }

View Full Code Here

Examples of org.apache.hadoop.mapred.FileOutputCommitter

    public static class HBaseBulkOutputCommitter extends OutputCommitter {


        private final OutputCommitter baseOutputCommitter;


        public HBaseBulkOutputCommitter() {
            baseOutputCommitter = new FileOutputCommitter();
        }

View Full Code Here

Examples of org.apache.hadoop.mapred.FileOutputCommitter

    public static class HBaseBulkOutputCommitter extends OutputCommitter {


        private final OutputCommitter baseOutputCommitter;


        public HBaseBulkOutputCommitter() {
            baseOutputCommitter = new FileOutputCommitter();
        }

View Full Code Here

Examples of org.apache.hadoop.mapred.FileOutputCommitter

  public static class HBaseBulkOutputCommitter extends OutputCommitter {


    private final OutputCommitter baseOutputCommitter;


    public HBaseBulkOutputCommitter() {
      baseOutputCommitter = new FileOutputCommitter();
    }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter


  public RecordWriter<ImmutableBytesWritable, KeyValue> getRecordWriter(final TaskAttemptContext context)
  throws IOException, InterruptedException {
    // Get the path of the temporary output file
    final Path outputPath = FileOutputFormat.getOutputPath(context);
    final Path outputdir = new FileOutputCommitter(outputPath, context).getWorkPath();
    final Configuration conf = context.getConfiguration();
    final FileSystem fs = outputdir.getFileSystem(conf);
    // These configs. are from hbase-*.xml
    final long maxsize = conf.getLong(HConstants.HREGION_MAX_FILESIZE,
        HConstants.DEFAULT_MAX_FILE_SIZE);

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

  public void checkOutputSpecs(JobContext context) 
      throws IOException, InterruptedException {}


  public OutputCommitter getOutputCommitter(TaskAttemptContext context) 
      throws IOException, InterruptedException {
    return new FileOutputCommitter(FileOutputFormat.getOutputPath(context),
                                   context);
  }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

    Path p1;
    isAppend = context.getConfiguration().get(HIHOConf.IS_APPEND);
    if (isAppend.equalsIgnoreCase("false")) {
      p1 = super.getDefaultWorkFile(context, extension);
    } else {
      FileOutputCommitter committer = (FileOutputCommitter) getOutputCommitter(context);
      Path p = committer.getWorkPath();
      fileCount = p.getFileSystem(context.getConfiguration())
          .getContentSummary(getOutputPath(context)).getFileCount();
      if (fileCount > 1) {
        fileCount = fileCount - 1;
      }
      p1 = new Path(committer.getWorkPath(), getUniqueFile(context,
          "part", extension));
    }
    return p1;
  }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

    Path p1;
    isAppend = context.getConfiguration().get(HIHOConf.IS_APPEND, "false");
    if (isAppend.equalsIgnoreCase("false")) {
      p1 = super.getDefaultWorkFile(context, extension);
    } else {
      FileOutputCommitter committer = (FileOutputCommitter) getOutputCommitter(context);
      Path p = committer.getWorkPath();
      fileCount = p.getFileSystem(context.getConfiguration())
          .getContentSummary(getOutputPath(context)).getFileCount();
      if (fileCount > 1)
        fileCount = fileCount - 1;
      p1 = new Path(committer.getWorkPath(), getUniqueFile(context,
          "part", extension));
    }
    return p1;
  }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.