Package org.apache.hadoop.mapreduce.lib.output

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

97
98
99
100
101
102
103
104
105
106
107
108
  public OutputCommitter getOutputCommitter(TaskAttemptContext context) throws IOException {
    createOutputFormatIfNeeded(context);

    String outDir = context.getConfiguration().get("mapred.output.dir");
    originalDir = outDir;
    FileOutputCommitter committer = (FileOutputCommitter) super.getOutputCommitter(context);
    baseDir = committer.getWorkPath() + "";

    Configuration conf = new Configuration(context.getConfiguration());
    TaskAttemptContext reContext;
    try {
      reContext = TaskAttemptContextFactory.get(conf, context.getTaskAttemptID());
View Full Code Here
142
143
144
145
146
147
148
149
150
151
    String outputPath = getOutputDir(conf);
    // we need to do this to get the task path and set it for mapred
    // implementation since it can't be done automatically because of
    // mapreduce->mapred abstraction
    if (outputPath != null) {
      FileOutputCommitter foc =
          new FileOutputCommitter(getOutputPath(conf), context);
      conf.set("mapred.work.output.dir", foc.getWorkPath().toString());
    }
  }
View Full Code Here
844
845
846
847
848
849
850
851
852
853
        }

        @Override
        public Path getDefaultWorkFile(TaskAttemptContext context,
                String extension) throws IOException {
            FileOutputCommitter committer =
                    (FileOutputCommitter) super.getOutputCommitter(context);
            return new Path(committer.getWorkPath(), getUniqueFile(context,
                    "part", extension));
        }
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
109
110
      throws IOException {
    createOutputFormatIfNeeded(context);

    String outDir = context.getConfiguration().get("mapred.output.dir");
    originalDir = outDir;
    FileOutputCommitter committer = (FileOutputCommitter) super
        .getOutputCommitter(context);
    baseDir = committer.getWorkPath() + "";
    Configuration conf = new Configuration(context.getConfiguration());
    TaskAttemptContext reContext = new TaskAttemptContext(conf,
        context.getTaskAttemptID());
    reContext.getConfiguration().set("mapred.output.dir", baseDir);
View Full Code Here
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
          // orbLogger.info("Partition: " + Integer.toString(partitionId) + "writing: " +
          // oc.getKey().toString() + ", " + oc.getValue().toString());
        }
        rw.close(tao);
       
        FileOutputCommitter cm = (FileOutputCommitter) outputFormat.getOutputCommitter(tao);
        if (cm.needsTaskCommit(tao)) {
          cm.commitTask(tao);
          cm.cleanupJob(jobContext);
        } else {
          cm.cleanupJob(jobContext);
          tryAgain = true;
        }
       
      } catch (IOException e) {
        tryAgain = true;
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
72
  public org.apache.hadoop.mapreduce.RecordWriter<WritableComparable<?>, BytesRefArrayWritable> getRecordWriter(
    TaskAttemptContext task) throws IOException, InterruptedException {

    //FileOutputFormat.getWorkOutputPath takes TaskInputOutputContext instead of
    //TaskAttemptContext, so can't use that here
    FileOutputCommitter committer = (FileOutputCommitter) getOutputCommitter(task);
    Path outputPath = committer.getWorkPath();

    FileSystem fs = outputPath.getFileSystem(task.getConfiguration());

    if (!fs.exists(outputPath)) {
      fs.mkdirs(outputPath);
View Full Code Here
213
214
215
216
217
218
219
220
221
222
223
          HCatMapRedUtil.createTaskAttemptContext(currJobContext.getJobConf(),
            currTaskContext.getTaskAttemptID(),
            currTaskContext.getProgressible());
        //set temp location
        currTaskContext.getConfiguration().set("mapred.work.output.dir",
          new FileOutputCommitter(new Path(localJobInfo.getLocation()), currTaskContext).getWorkPath().toString());
        //setupTask()
        baseOutputCommitter.setupTask(currTaskContext);

        Path parentDir = new Path(currTaskContext.getConfiguration().get("mapred.work.output.dir"));
        Path childPath = new Path(parentDir,FileOutputFormat.getUniqueFile(currTaskContext, "part", ""));
View Full Code Here
212
213
214
215
216
217
218
219
220
221
222
          HCatMapRedUtil.createTaskAttemptContext(currJobContext.getJobConf(),
            currTaskContext.getTaskAttemptID(),
            currTaskContext.getProgressible());
        //set temp location
        currTaskContext.getConfiguration().set("mapred.work.output.dir",
          new FileOutputCommitter(new Path(localJobInfo.getLocation()), currTaskContext).getWorkPath().toString());
        //setupTask()
        baseOutputCommitter.setupTask(currTaskContext);

        Path parentDir = new Path(currTaskContext.getConfiguration().get("mapred.work.output.dir"));
        Path childPath = new Path(parentDir,FileOutputFormat.getUniqueFile(currTaskContext, "part", ""));
View Full Code Here
161
162
163
164
165
166
167
168
169
170
    String outputPath = getOutputDir(conf);
    // we need to do this to get the task path and set it for mapred
    // implementation since it can't be done automatically because of
    // mapreduce->mapred abstraction
    if (outputPath != null) {
      FileOutputCommitter foc =
          new FileOutputCommitter(getOutputPath(conf), context);
      conf.set("mapred.work.output.dir", foc.getWorkPath().toString());
    }
  }
View Full Code Here
142
143
144
145
146
147
148
149
150
151
    String outputPath = getOutputDir(conf);
    // we need to do this to get the task path and set it for mapred
    // implementation since it can't be done automatically because of
    // mapreduce->mapred abstraction
    if (outputPath != null) {
      FileOutputCommitter foc =
          new FileOutputCommitter(getOutputPath(conf), context);
      conf.set("mapred.work.output.dir", foc.getWorkPath().toString());
    }
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

  • co.nubetech.hiho.mapreduce.lib.db.apache.DBOutputFormat
  • co.nubetech.hiho.mapreduce.lib.output.AppendSequenceFileOutputFormat
  • co.nubetech.hiho.mapreduce.lib.output.AppendTextOutputFormat
  • com.asakusafw.runtime.stage.output.StageOutputFormat
  • com.asakusafw.runtime.stage.output.TemporaryOutputFormat
  • com.datasalt.pangool.tuplemr.mapred.lib.output.ProxyOutputFormat
  • com.facebook.giraph.hive.common.HadoopUtils
  • com.facebook.giraph.hive.impl.common.HadoopUtils
  • com.facebook.hiveio.common.HadoopUtils
  • com.splout.db.hadoop.engine.SploutSQLProxyOutputFormat
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.