Package org.apache.hadoop.mapred

Examples of org.apache.hadoop.mapred.InputSplit

222
223
224
225
226
227
228
229
230
231
232
    secondOutput = new PartialOutputCollector(total);
    Reporter reporter = Reporter.NULL;
    long slowest = 0; // duration of slowest map

    for (int partition = 0; partition < splits.length; partition++) {
      InputSplit split = splits[partition];
      RecordReader<LongWritable, Text> reader = input.getRecordReader(split,
          job, reporter);

      LongWritable key = reader.createKey();
      Text value = reader.createValue();
View Full Code Here
79
80
81
82
83
84
85
86
87
88
89
    Step0OutputCollector collector = new Step0OutputCollector(numMaps);
    Reporter reporter = Reporter.NULL;

    for (int p = 0; p < numMaps; p++) {
      InputSplit split = sorted[p];
      RecordReader<LongWritable, Text> reader = input.getRecordReader(split, job, reporter);

      LongWritable key = reader.createKey();
      Text value = reader.createValue();
View Full Code Here
152
153
154
155
156
157
158
159
160
161
162
    Step0Output[] values = new Step0Output[numMaps];
   
    int[] expectedIds = new int[numMaps];
   
    for (int p = 0; p < numMaps; p++) {
      InputSplit split = sorted[p];
      RecordReader<LongWritable, Text> reader = input.getRecordReader(split, job, reporter);

      LongWritable key = reader.createKey();
      Text value = reader.createValue();
View Full Code Here
74
75
76
77
78
79
80
81
82
83
84
  }

  @Override
  public RecordReader<LongWritable, Text> getRecordReader(
      InputSplit split, JobConf job, Reporter reporter) throws IOException {
    InputSplit targetSplit = ((SymlinkTextInputSplit)split).getTargetSplit();

    // The target data is in TextInputFormat.
    TextInputFormat inputFormat = new TextInputFormat();
    inputFormat.configure(job);
    return inputFormat.getRecordReader(targetSplit, job, reporter);
View Full Code Here
123
124
125
126
127
128
129
130
131
132
      } catch (HiveException e) {
        throw new RuntimeException(
            "Unable to get metadata for input table split" + split.getPath());
      }
    }
    InputSplit retA[] = newSplits.toArray((new FileSplit[newSplits.size()]));
    l4j.info("Number of input splits: " + splits.length + " new input splits: "
        + retA.length);
    return retA;
  }
View Full Code Here
203
204
205
206
207
208
209
210
211
212
213
  public RecordReader getRecordReader(InputSplit split, JobConf job,
      Reporter reporter) throws IOException {

    HiveInputSplit hsplit = (HiveInputSplit) split;

    InputSplit inputSplit = hsplit.getInputSplit();
    String inputFormatClassName = null;
    Class inputFormatClass = null;
    try {
      inputFormatClassName = hsplit.inputFormatClassName();
      inputFormatClass = job.getClassByName(inputFormatClassName);
View Full Code Here
166
167
168
169
170
171
172
173
174
175
176
  public RecordReader getRecordReader(InputSplit split, JobConf job,
      Reporter reporter) throws IOException {

    HiveInputSplit hsplit = (HiveInputSplit)split;

    InputSplit inputSplit = hsplit.getInputSplit();
    String inputFormatClassName = null;
    Class inputFormatClass = null;
    try {
      inputFormatClassName = hsplit.inputFormatClassName();
      inputFormatClass = Class.forName(inputFormatClassName);
View Full Code Here
205
206
207
208
209
210
211
212
213
214
215
  public RecordReader getRecordReader(InputSplit split, JobConf job,
      Reporter reporter) throws IOException {

    HiveInputSplit hsplit = (HiveInputSplit) split;

    InputSplit inputSplit = hsplit.getInputSplit();
    String inputFormatClassName = null;
    Class inputFormatClass = null;
    try {
      inputFormatClassName = hsplit.inputFormatClassName();
      inputFormatClass = job.getClassByName(inputFormatClassName);
View Full Code Here
314
315
316
317
318
319
320
321
322
323
324
325
326
    InputSplit[] splits = aif.getSplits(job, 1);

    Assert.assertEquals(1, splits.length);

    InputSplit split = splits[0];

    Assert.assertEquals(RangeInputSplit.class, split.getClass());

    RangeInputSplit risplit = (RangeInputSplit) split;

    Assert.assertEquals(username, risplit.getPrincipal());
    Assert.assertEquals(table, risplit.getTable());
View Full Code Here
420
421
422
423
424
425
426
427
428
429
430
    {
      JobConf job = HadoopPlugin.getJobFactory(BitPostingIndexInputFormat.class.getSimpleName()).newJob();
      setStructures(job, "linksin", "linksin-lookup");
      HadoopUtility.toHConfiguration(index, job);
      index.close();
      InputSplit s = new BitPostingIndexInputSplit(
          new Path(args[3]), Long.parseLong(args[4]), Long.parseLong(args[5]),
          new String[0], Integer.parseInt(args[6]), Integer.parseInt(args[7]));
      RecordReader<IntWritable, IntObjectWrapper<IterablePosting>> rr = new BitPostingIndexInputFormat().getRecordReader(s, job, new Reporter(){
        public InputSplit getInputSplit() throws UnsupportedOperationException {return null;}
        @SuppressWarnings("unchecked")
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapred.InputSplit

  • com.ebay.erl.mobius.core.mapred.MobiusInputSampler
  • com.sap.hadoop.windowing.io.TableWindowingInput
  • com.taobao.zeus.jobs.sub.tool.DataPreviewJob
  • org.apache.accumulo.core.client.mapred.AccumuloInputFormatTest
  • org.apache.drill.exec.store.hive.HiveSubScan
  • org.apache.hadoop.contrib.mongoreduce.MongoStreamInputFormat
  • org.apache.hadoop.hbase.mapred.TestTableSnapshotInputFormat
  • org.apache.hadoop.hive.accumulo.mr.TestHiveAccumuloTableInputFormat
  • org.apache.hadoop.hive.ql.exec.StatsNoJobTask
  • org.apache.hadoop.hive.ql.exec.StatsNoJobTask$StatsCollection
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.