Package org.apache.hadoop.mapreduce.lib.output

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

196
197
198
199
200
201
202
203
204
205
206
207
208
    String outputPath = getOutputDir(conf);
    // we need to do this to get the task path and set it for mapred
    // implementation since it can't be done automatically because of
    // mapreduce->mapred abstraction
    if (outputPath != null) {
      FileOutputCommitter foc =
          new FileOutputCommitter(getOutputPath(conf), context);
      Path path = foc.getWorkPath();
      FileSystem fs = path.getFileSystem(conf);
      fs.mkdirs(path);
      conf.set("mapred.work.output.dir", path.toString());
      LOG.info("Setting mapred.work.output.dir to {}", path.toString());
    }
View Full Code Here
236
237
238
239
240
241
242
243
  }

  /** (@inheritDoc) */
  public OutputCommitter getOutputCommitter(TaskAttemptContext context)
      throws IOException, InterruptedException {
    return new FileOutputCommitter(FileOutputFormat.getOutputPath(context),
        context);
  }
View Full Code Here
73
74
75
76
77
78
79
80
    }
  }

  public OutputCommitter getOutputCommitter(TaskAttemptContext context)
      throws IOException, InterruptedException {
    return new FileOutputCommitter(FileOutputFormat.getOutputPath(context),
        context);
  }
View Full Code Here
54
55
56
57
58
59
60
61
  public void checkOutputSpecs(JobContext context)
      throws IOException, InterruptedException {}

  public OutputCommitter getOutputCommitter(TaskAttemptContext context)
      throws IOException, InterruptedException {
    return new FileOutputCommitter(FileOutputFormat.getOutputPath(context),
                                   context);
  }
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
108
  public OutputCommitter getOutputCommitter(TaskAttemptContext context) throws IOException {
    createOutputFormatIfNeeded(context);

    String outDir = context.getConfiguration().get("mapred.output.dir");
    originalDir = outDir;
    FileOutputCommitter committer = (FileOutputCommitter) super.getOutputCommitter(context);
    baseDir = committer.getWorkPath() + "";
   
    Configuration conf = new Configuration(context.getConfiguration());
    TaskAttemptContext reContext;
    try {
      reContext = TaskAttemptContextFactory.get(conf, context.getTaskAttemptID());
View Full Code Here
50
51
52
53
54
55
56
57
58
59
60
public class HFileOutputFormat extends FileOutputFormat<ImmutableBytesWritable, KeyValue> {
  public RecordWriter<ImmutableBytesWritable, KeyValue> getRecordWriter(TaskAttemptContext context)
  throws IOException, InterruptedException {
    // Get the path of the temporary output file
    final Path outputPath = FileOutputFormat.getOutputPath(context);
    final Path outputdir = new FileOutputCommitter(outputPath, context).getWorkPath();
    Configuration conf = context.getConfiguration();
    final FileSystem fs = outputdir.getFileSystem(conf);
    // These configs. are from hbase-*.xml
    final long maxsize = conf.getLong("hbase.hregion.max.filesize", 268435456);
    final int blocksize = conf.getInt("hfile.min.blocksize.size", 65536);
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
108
  public OutputCommitter getOutputCommitter(TaskAttemptContext context) throws IOException {
    createOutputFormatIfNeeded(context);

    String outDir = context.getConfiguration().get("mapred.output.dir");
    originalDir = outDir;
    FileOutputCommitter committer = (FileOutputCommitter) super.getOutputCommitter(context);
    baseDir = committer.getWorkPath() + "";
    Configuration conf = new Configuration(context.getConfiguration());
    TaskAttemptContext reContext;
    try {
      reContext = TaskAttemptContextFactory.get(conf, context.getTaskAttemptID());
    } catch(Exception e) {
View Full Code Here
85
86
87
88
89
90
91
  @Override
  public OutputCommitter getOutputCommitter(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException
  {
    // Is there a programmatic way to get the temp dir? I see it hardcoded everywhere in Hadoop, Hive, and Pig.
    return new FileOutputCommitter(new Path("/tmp/" + taskAttemptContext.getTaskAttemptID().getJobID().toString()), taskAttemptContext);
  }
View Full Code Here
113
114
115
116
117
118
119
120
121
122
123
      this.context = HadoopUtils.instantiateTaskAttemptContext(this.configuration, taskAttemptID);
    } catch (Exception e) {
      throw new RuntimeException(e);
    }
   
    this.fileOutputCommitter = new FileOutputCommitter(new Path(this.configuration.get("mapred.output.dir")), context);
   
    try {
      this.fileOutputCommitter.setupJob(HadoopUtils.instantiateJobContext(this.configuration, new JobID()));
    } catch (Exception e) {
      throw new RuntimeException(e);
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
88
      private void initSql(int partition) throws IOException, InterruptedException {
        // HDFS final location of the generated partition file. It will be
        // loaded to the temporary folder in the HDFS than finally will be
        // committed by the OutputCommitter to the proper location.
        FileOutputCommitter committer = (FileOutputCommitter) getOutputCommitter(SploutSQLProxyOutputFormat.this.context);
        Path perm = new Path(committer.getWorkPath(), partition + ".db");
        FileSystem fs = perm.getFileSystem(conf);

        // Make a task unique name that contains the actual index output name to
        // make debugging simpler
        // Note: if using JVM reuse, the sequence number will not be reset for a
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

  • co.nubetech.hiho.mapreduce.lib.db.apache.DBOutputFormat
  • co.nubetech.hiho.mapreduce.lib.output.AppendSequenceFileOutputFormat
  • co.nubetech.hiho.mapreduce.lib.output.AppendTextOutputFormat
  • com.asakusafw.runtime.stage.output.StageOutputFormat
  • com.asakusafw.runtime.stage.output.TemporaryOutputFormat
  • com.datasalt.pangool.tuplemr.mapred.lib.output.ProxyOutputFormat
  • com.facebook.giraph.hive.common.HadoopUtils
  • com.facebook.giraph.hive.impl.common.HadoopUtils
  • com.facebook.hiveio.common.HadoopUtils
  • com.splout.db.hadoop.engine.SploutSQLProxyOutputFormat
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.