Package org.apache.hadoop.mapreduce.lib.output

Examples of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

245
246
247
248
249
250
251
        String outputPath = context.getConfiguration().get("mapred.output.dir");
        //we need to do this to get the task path and set it for mapred implementation
        //since it can't be done automatically because of mapreduce->mapred abstraction
        if (outputPath != null)
            context.getConfiguration().set("mapred.work.output.dir",
                new FileOutputCommitter(new Path(outputPath), context).getWorkPath().toString());
    }
View Full Code Here
212
213
214
215
216
217
218
219
220
221
222
                    HCatMapRedUtil.createTaskAttemptContext(currJobContext.getJobConf(),
                        currTaskContext.getTaskAttemptID(),
                        currTaskContext.getProgressible());
                //set temp location
                currTaskContext.getConfiguration().set("mapred.work.output.dir",
                    new FileOutputCommitter(new Path(localJobInfo.getLocation()), currTaskContext).getWorkPath().toString());
                //setupTask()
                baseOutputCommitter.setupTask(currTaskContext);

                Path parentDir = new Path(currTaskContext.getConfiguration().get("mapred.work.output.dir"));
                Path childPath = new Path(parentDir,FileOutputFormat.getUniqueFile(currTaskContext, "part", ""));
View Full Code Here
138
139
140
141
142
143
144
145
146
147
148
   
    // Setup the Mapper
    WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(tmpFile, 0, fs.pathToFile(tmpFile).length(), null),0);
    AggregatingRecordReader rr = new AggregatingRecordReader();
    Path ocPath = new Path(tmpFile, "oc");
    OutputCommitter oc = new FileOutputCommitter(ocPath, context);
    fs.deleteOnExit(ocPath);
    StandaloneStatusReporter sr = new StandaloneStatusReporter();
    rr.initialize(split, context);
    MockAccumuloRecordWriter rw = new MockAccumuloRecordWriter();
    WikipediaMapper mapper = new WikipediaMapper();
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
108
109
      throws IOException {
    createOutputFormatIfNeeded(context);

    String outDir = context.getConfiguration().get("mapred.output.dir");
    originalDir = outDir;
    FileOutputCommitter committer = (FileOutputCommitter) super
        .getOutputCommitter(context);
    baseDir = committer.getWorkPath() + "";
    Configuration conf = new Configuration(context.getConfiguration());
    TaskAttemptContext reContext = new TaskAttemptContext(conf,
        context.getTaskAttemptID());
    reContext.getConfiguration().set("mapred.output.dir", baseDir);
View Full Code Here
65
66
67
68
69
70
71
  @Override
  public OutputCommitter getOutputCommitter(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException
  {
    // Is there a programmatic way to get the temp dir? I see it hardcoded everywhere in Hadoop, Hive, and Pig.
    return new FileOutputCommitter(new Path("/tmp/" + taskAttemptContext.getTaskAttemptID().getJobID().toString()), taskAttemptContext);
  }
View Full Code Here
106
107
108
109
110
111
112
113
114
115
116
    return writer;
  }

  @Override
  public RecordWriter<K, V> getRecordWriter(TaskAttemptContext job) throws IOException {
    FileOutputCommitter committer = (FileOutputCommitter) this.getOutputCommitter(job);
    File tmpDirFile = Files.createTempDir();
    Directory directory = getDirectoryImplementation(tmpDirFile);
    IndexWriter writer = createIndexWriter(directory, newAnalyzer(HadoopCompat.getConfiguration(job)));
    return new IndexRecordWriter(writer, committer, tmpDirFile);
  }
View Full Code Here
84
85
86
87
88
89
90
91
      org.apache.hadoop.mapreduce.lib.output.FileOutputFormat<?, ?> outputFormat,
      String name,
      TaskAttemptContext context,
      String extension
    ) throws IOException {
      FileOutputCommitter committer = (FileOutputCommitter) outputFormat.getOutputCommitter(context);
      return new Path(committer.getWorkPath(), name + extension);
    }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter

  • co.nubetech.hiho.mapreduce.lib.db.apache.DBOutputFormat
  • co.nubetech.hiho.mapreduce.lib.output.AppendSequenceFileOutputFormat
  • co.nubetech.hiho.mapreduce.lib.output.AppendTextOutputFormat
  • com.asakusafw.runtime.stage.output.StageOutputFormat
  • com.asakusafw.runtime.stage.output.TemporaryOutputFormat
  • com.datasalt.pangool.tuplemr.mapred.lib.output.ProxyOutputFormat
  • com.facebook.giraph.hive.common.HadoopUtils
  • com.facebook.giraph.hive.impl.common.HadoopUtils
  • com.facebook.hiveio.common.HadoopUtils
  • com.splout.db.hadoop.engine.SploutSQLProxyOutputFormat
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.