Package org.apache.hadoop.mapred

Examples of org.apache.hadoop.mapred.Reporter

87
88
89
90
91
92
93
94
95
96
97
  @SuppressWarnings({ "deprecation", "unchecked" })
  public void shouldCreateNewKeyAlthoughExtraKey() throws Exception {
    GroupingTableMap gTableMap = null;
    try {
      Result result = mock(Result.class);
      Reporter reporter = mock(Reporter.class);
      gTableMap = new GroupingTableMap();
      Configuration cfg = new Configuration();
      cfg.set(GroupingTableMap.GROUP_COLUMNS, "familyA:qualifierA familyB:qualifierB");
      JobConf jobConf = new JobConf(cfg);
      gTableMap.configure(jobConf);
View Full Code Here
119
120
121
122
123
124
125
126
127
128
129
  @SuppressWarnings({ "deprecation" })
  public void shouldCreateNewKey() throws Exception {
    GroupingTableMap gTableMap = null; 
    try {
      Result result = mock(Result.class);
      Reporter reporter = mock(Reporter.class);
      final byte[] bSeparator = Bytes.toBytes(" ");
      gTableMap = new GroupingTableMap();
      Configuration cfg = new Configuration();
      cfg.set(GroupingTableMap.GROUP_COLUMNS, "familyA:qualifierA familyB:qualifierB");
      JobConf jobConf = new JobConf(cfg);
View Full Code Here
189
190
191
192
193
194
195
196
197
198
199
      InputSplit split = splits[i];
      Assert.assertTrue(split instanceof TableSnapshotInputFormat.TableSnapshotRegionSplit);

      // validate record reader
      OutputCollector collector = mock(OutputCollector.class);
      Reporter reporter = mock(Reporter.class);
      RecordReader<ImmutableBytesWritable, Result> rr = tsif.getRecordReader(split, job, reporter);

      // validate we can read all the data back
      ImmutableBytesWritable key = rr.createKey();
      Result value = rr.createValue();
View Full Code Here
921
922
923
924
925
926
927
928
929
930
931
                  getValueClass(),
                  allocateRawComparator());
           
            try {
              LOG.info("Execute Local for Query:" + getQueryId() +" Running Merger");
              merger.mergeAndSpill(new Reporter() {
               
                @Override
                public void progress() {
                }
               
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
55
                        Properties tableProperties,
                        Progressable progress) throws IOException {
    final RecordWriter result =
      super.getHiveRecordWriter(jc,outPath,valueClass,isCompressed,
        tableProperties,progress);
    final Reporter reporter = (Reporter) progress;
    reporter.setStatus("got here");
    System.out.println("Got a reporter " + reporter);
    return new RecordWriter() {
      @Override
      public void write(Writable w) throws IOException {
        if (w instanceof Text) {
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
    JobConf job = new JobConf(conf);
    FileSystem fs = FileSystem.getNamed("local", conf);
    Path dir = new Path(System.getProperty("test.build.data",".") + "/mapred");
    Path file = new Path(dir, "test.seq");
   
    Reporter reporter = new Reporter() {
        public void setStatus(String status) throws IOException {}
        public void progress() throws IOException {}
      };
   
    int seed = new Random().nextInt();
View Full Code Here
35
36
37
38
39
40
41
42
43
44
  public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    return PrimitiveObjectInspectorFactory.writableLongObjectInspector;
  }

  public Object evaluate(DeferredObject[] arguments) throws HiveException {
    Reporter reporter = context.getReporter();
    Counters.Counter counter = reporter.getCounter("org.apache.hadoop.mapred.Task$Counter", "MAP_INPUT_RECORDS");
    result.set(counter.getValue());
    return result;
  }
View Full Code Here
404
405
406
407
408
409
410
411
412
413
414
      conf.set(MRJobConfig.TASK_ATTEMPT_ID, taskName);
      initStdOut(conf);
      conf.setBoolean(MRJobConfig.SKIP_RECORDS, true);
      CombineOutputCollector<IntWritable, Text> output = new CombineOutputCollector<IntWritable, Text>(
              new Counters.Counter(), new Progress());
      Reporter reporter = new TestTaskReporter();
      List<Text> texts = new ArrayList<Text>();
      texts.add(new Text("first"));
      texts.add(new Text("second"));
      texts.add(new Text("third"));
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
  public void testFormat() throws IOException {

    PipesNonJavaInputFormat inputFormat = new PipesNonJavaInputFormat();
    JobConf conf = new JobConf();

    Reporter reporter= mock(Reporter.class);
    RecordReader<FloatWritable, NullWritable> reader = inputFormat
        .getRecordReader(new FakeSplit(), conf, reporter);
    assertEquals(0.0f, reader.getProgress(), 0.001);

    // input and output files
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
    JobConf job = new JobConf(conf);
    FileSystem fs = FileSystem.getNamed("local", conf);
    Path dir = new Path(System.getProperty("test.build.data",".") + "/mapred");
    Path file = new Path(dir, "test.seq");
   
    Reporter reporter = new Reporter() {
        public void setStatus(String status) throws IOException {}
      };
   
    int seed = new Random().nextInt();
    //LOG.info("seed = "+seed);
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapred.Reporter

  • co.nubetech.hiho.mapreduce.TestDBInputAvroMapper
  • com.facebook.giraph.hive.impl.input.HiveApiInputSplit
  • com.facebook.giraph.hive.input.HInputSplit
  • com.facebook.giraph.hive.output.HiveApiOutputFormat
  • com.facebook.hiveio.input.HInputSplit
  • com.facebook.hiveio.output.HiveApiOutputFormat
  • com.senseidb.indexing.hadoop.reduce.SenseiReducer
  • com.twitter.elephantbird.mapreduce.input.MapReduceInputFormatWrapper$RecordReaderWrapper
  • org.apache.hadoop.contrib.index.mapred.IndexUpdateReducer
  • org.apache.hadoop.hbase.mapred.TestGroupingTableMap
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.