Package org.apache.hadoop.mapred

Examples of org.apache.hadoop.mapred.Reporter

299
300
301
302
303
304
305
306
307
308
309
          public void collect(Object key, Object value)
            throws IOException {
            //just consume it, no need to write the record anywhere
          }
        };
        Reporter reporter = Reporter.NULL;//dummy reporter
        startOutputThreads(collector, reporter);
      }
      int exitVal = sim.waitFor();
      // how'd it go?
      if (exitVal != 0) {
View Full Code Here
350
351
352
353
354
355
356
357
358
359
360
          public void collect(Object key, Object value)
            throws IOException {
            //just consume it, no need to write the record anywhere
          }
        };
        Reporter reporter = Reporter.NULL;//dummy reporter
        startOutputThreads(collector, reporter);
      }
      int exitVal = sim.waitFor();
      // how'd it go?
      if (exitVal != 0) {
View Full Code Here
308
309
310
311
312
313
314
315
316
317
318
          public void collect(Object key, Object value)
            throws IOException {
            //just consume it, no need to write the record anywhere
          }
        };
        Reporter reporter = Reporter.NULL;//dummy reporter
        startOutputThreads(collector, reporter);
      }
      int exitVal = sim.waitFor();
      // how'd it go?
      if (exitVal != 0) {
View Full Code Here
71
72
73
74
75
76
77
78
79
80
81
        FileWriter fileWriter = new FileWriter(files[i]);
        fileWriter.close();
        paths[i] = new Path(outDir+"/testfile"+i);
      }
      CombineFileSplit combineFileSplit = new CombineFileSplit(conf, paths, fileLength);
      Reporter reporter = Mockito.mock(Reporter.class);
      CombineFileRecordReader cfrr = new CombineFileRecordReader(conf, combineFileSplit,
        reporter,  TextRecordReaderWrapper.class);
      verify(reporter).progress();
      Assert.assertFalse(cfrr.next(key,value));
      verify(reporter, times(3)).progress();
View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
   
    DBInputFormat<NullDBWritable> format = new DBInputFormat<NullDBWritable>();
    format.setConf(configuration);
    format.setConf(configuration);
    DBInputFormat.DBInputSplit splitter = new DBInputFormat.DBInputSplit(1, 10);
    Reporter reporter = mock(Reporter.class);
    RecordReader<LongWritable, NullDBWritable> reader = format.getRecordReader(
        splitter, configuration, reporter);

    configuration.setInt(MRJobConfig.NUM_MAPS, 3);
    InputSplit[] lSplits = format.getSplits(configuration, 3);
View Full Code Here
37
38
39
40
41
42
43
44
45
46
  public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    return PrimitiveObjectInspectorFactory.writableLongObjectInspector;
  }

  public Object evaluate(DeferredObject[] arguments) throws HiveException {
    Reporter reporter = context.getReporter();
    Counters.Counter counter = reporter.getCounter("org.apache.hadoop.mapred.Task$Counter", "MAP_INPUT_RECORDS");
    result.set(counter.getValue());
    return result;
  }
View Full Code Here
47
48
49
50
51
52
53
54
    super(baseWriter, context);
  }

  @Override
  public void close(TaskAttemptContext context) throws IOException, InterruptedException {
    Reporter reporter = InternalUtil.createReporter(context);
    getBaseRecordWriter().close(reporter);
  }
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
    this.dynamicOutputJobInfo = new HashMap<String, OutputJobInfo>();
  }

  @Override
  public void close(TaskAttemptContext context) throws IOException, InterruptedException {
    Reporter reporter = InternalUtil.createReporter(context);
    for (RecordWriter<? super WritableComparable<?>, ? super Writable> bwriter : baseDynamicWriters
        .values()) {
      // We are in RecordWriter.close() make sense that the context would be
      // TaskInputOutput.
      bwriter.close(reporter);
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
  public void shouldCollectPredefinedTimes() throws IOException {
    int recordNumber = 999;
    Result resultMock = mock(Result.class);
    IdentityTableMap identityTableMap = null;
    try {
      Reporter reporterMock = mock(Reporter.class);
      identityTableMap = new IdentityTableMap();
      ImmutableBytesWritable bytesWritableMock = mock(ImmutableBytesWritable.class);
      OutputCollector<ImmutableBytesWritable, Result> outputCollectorMock =
          mock(OutputCollector.class);
 
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
  @Test
  @SuppressWarnings({ "deprecation", "unchecked" })
  public void shouldRegInReportEveryIncomingRow() throws IOException {
    int iterationNumber = 999;
    RowCounter.RowCounterMapper mapper = new RowCounter.RowCounterMapper();
    Reporter reporter = mock(Reporter.class);
    for (int i = 0; i < iterationNumber; i++)
      mapper.map(mock(ImmutableBytesWritable.class), mock(Result.class),
          mock(OutputCollector.class), reporter);

    Mockito.verify(reporter, times(iterationNumber)).incrCounter(
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapred.Reporter

  • co.nubetech.hiho.mapreduce.TestDBInputAvroMapper
  • com.facebook.giraph.hive.impl.input.HiveApiInputSplit
  • com.facebook.giraph.hive.input.HInputSplit
  • com.facebook.giraph.hive.output.HiveApiOutputFormat
  • com.facebook.hiveio.input.HInputSplit
  • com.facebook.hiveio.output.HiveApiOutputFormat
  • com.senseidb.indexing.hadoop.reduce.SenseiReducer
  • com.twitter.elephantbird.mapreduce.input.MapReduceInputFormatWrapper$RecordReaderWrapper
  • org.apache.hadoop.contrib.index.mapred.IndexUpdateReducer
  • org.apache.hadoop.hbase.mapred.TestGroupingTableMap
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.