Package org.apache.flink.hadoopcompatibility.mapred.wrapper

Examples of org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopDummyReporter

91
92
93
94
95
96
97
98
99
100
  @Override
  public void open(Configuration parameters) throws Exception {
    super.open(parameters);
    this.reducer.configure(jobConf);
   
    this.reporter = new HadoopDummyReporter();
    this.reduceCollector = new HadoopOutputCollector<KEYOUT, VALUEOUT>();
    Class<KEYIN> inKeyClass = (Class<KEYIN>) TypeExtractor.getParameterType(Reducer.class, reducer.getClass(), 0);
    this.valueIterator = new HadoopTupleUnwrappingIterator<KEYIN, VALUEIN>(inKeyClass);
  }
View Full Code Here
103
104
105
106
107
108
109
110
111
112
113
  public void open(Configuration parameters) throws Exception {
    super.open(parameters);
    this.reducer.configure(jobConf);
    this.combiner.configure(jobConf);
   
    this.reporter = new HadoopDummyReporter();
    Class<KEYIN> inKeyClass = (Class<KEYIN>) TypeExtractor.getParameterType(Reducer.class, reducer.getClass(), 0);
    this.valueIterator = new HadoopTupleUnwrappingIterator<KEYIN, VALUEIN>(inKeyClass);
    this.combineCollector = new HadoopOutputCollector<KEYIN, VALUEIN>();
    this.reduceCollector = new HadoopOutputCollector<KEYOUT, VALUEOUT>();
  }
View Full Code Here
89
90
91
92
93
94
95
96
  @Override
  public void open(Configuration parameters) throws Exception {
    super.open(parameters);
    this.mapper.configure(jobConf);
   
    this.reporter = new HadoopDummyReporter();
    this.outputCollector = new HadoopOutputCollector<KEYOUT, VALUEOUT>();
  }
View Full Code Here
134
135
136
137
138
139
140
141
142
143
144
   * commit the task by moving the output file out from the temporary directory.
   * @throws IOException
   */
  @Override
  public void close() throws IOException {
    this.recordWriter.close(new HadoopDummyReporter());
   
    if (this.fileOutputCommitter.needsTaskCommit(this.context)) {
      this.fileOutputCommitter.commitTask(this.context);
    }
    this.fileOutputCommitter.commitJob(this.jobContext);
View Full Code Here
90
91
92
93
94
95
96
97
98
99
    return new DefaultInputSplitAssigner(inputSplits);
  }

  @Override
  public void open(HadoopInputSplit split) throws IOException {
    this.recordReader = this.hadoopInputFormat.getRecordReader(split.getHadoopInputSplit(), jobConf, new HadoopDummyReporter());
    key = this.recordReader.createKey();
    value = this.recordReader.createValue();
    this.fetched = false;
  }
View Full Code Here
153
154
155
156
157
158
159
160
161
162
163
    return new DefaultInputSplitAssigner(inputSplits);
  }
 
  @Override
  public void open(HadoopInputSplit split) throws IOException {
    this.recordReader = this.mapredInputFormat.getRecordReader(split.getHadoopInputSplit(), jobConf, new HadoopDummyReporter());
    if (this.recordReader instanceof Configurable) {
      ((Configurable) this.recordReader).setConf(jobConf);
    }
    key = this.recordReader.createKey();
    value = this.recordReader.createValue();
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
   * commit the task by moving the output file out from the temporary directory.
   * @throws IOException
   */
  @Override
  public void close() throws IOException {
    this.recordWriter.close(new HadoopDummyReporter());
    if (this.fileOutputCommitterWrapper.needsTaskCommit(this.jobConf, TaskAttemptID.forName(this.jobConf.get("mapred.task.id")))) {
      this.fileOutputCommitterWrapper.commitTask(this.jobConf, TaskAttemptID.forName(this.jobConf.get("mapred.task.id")));
    }
  //TODO: commitjob when all the tasks are finished
  }
View Full Code Here
TOP

Related Classes of org.apache.flink.hadoopcompatibility.mapred.wrapper.HadoopDummyReporter

  • org.apache.flink.hadoopcompatibility.mapred.HadoopInputFormat
  • org.apache.flink.hadoopcompatibility.mapred.HadoopMapFunction
  • org.apache.flink.hadoopcompatibility.mapred.HadoopOutputFormat
  • org.apache.flink.hadoopcompatibility.mapred.HadoopReduceCombineFunction
  • org.apache.flink.hadoopcompatibility.mapred.HadoopReduceFunction
  • org.apache.flink.hadoopcompatibility.mapred.record.HadoopRecordInputFormat
  • org.apache.flink.hadoopcompatibility.mapred.record.HadoopRecordOutputFormat
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.