Examples of OutputCollector

backtype.storm.task.OutputCollector
This output collector exposes the API for emitting tuples from an IRichBolt. This is the core API for emitting tuples. For a simpler API, and a more restricted form of stream processing, see IBasicBolt and BasicOutputCollector.
com.btaz.datautil.files.mapreduce.OutputCollector
User: msundell
com.btaz.util.mr.OutputCollector
User: msundell
org.apache.hadoop.mapred.OutputCollector
Collects the <key, value> pairs output by {@link Mapper}s and {@link Reducer}s.
OutputCollector is the generalization of the facility provided by the Map-Reduce framework to collect data output by either the Mapper or the Reducer i.e. intermediate outputs or the output of the job.

Examples of org.apache.hadoop.mapred.OutputCollector

    Reducer reducer =
        ReflectionUtils.newInstance(job.getReducerClass(), job);


    // make output collector


    OutputCollector collector =
        new OutputCollector() {
      public void collect(Object key, Object value)
          throws IOException {
        output.write(key, value);
      }
    };

View Full Code Here

Examples of org.apache.hadoop.mapred.OutputCollector

    
    updateJobWithSplit(job, inputSplit);


    RecordReader in = new OldRecordReader(input);


    OutputCollector collector = new OldOutputCollector(output);


    MapRunnable runner =
        (MapRunnable)ReflectionUtils.newInstance(job.getMapRunnerClass(), job);


    runner.run(in, collector, (Reporter)reporter);

View Full Code Here

Examples of org.apache.hadoop.mapred.OutputCollector

    Reducer reducer =
        ReflectionUtils.newInstance(job.getReducerClass(), job);


    // make output collector


    OutputCollector collector =
        new OutputCollector() {
      public void collect(Object key, Object value)
          throws IOException {
        output.write(key, value);
      }
    };

View Full Code Here

Examples of org.apache.hadoop.mapred.OutputCollector

        // called at all in this task). If reducer still generates output,
        // which is very uncommon and we may not have to support this case.
        // So we don't write this output to HDFS, but we consume/collect
        // this output just to avoid reducer hanging forever.


        OutputCollector collector = new OutputCollector() {
          public void collect(Object key, Object value)
            throws IOException {
            //just consume it, no need to write the record anywhere
          }
        };

View Full Code Here

Examples of org.apache.hadoop.mapred.OutputCollector

    // TODO use new method in MRInput to get required info
    //input.initialize(job, master);


    RecordReader in = new OldRecordReader(input);


    OutputCollector collector = new OldOutputCollector(output);


    MapRunnable runner =
        (MapRunnable)ReflectionUtils.newInstance(job.getMapRunnerClass(), job);


    runner.run(in, collector, (Reporter)reporter);

View Full Code Here

Examples of org.apache.hadoop.mapred.OutputCollector


  @Override
  public void sink(FlowProcess<JobConf> flowProcess, SinkCall<Object[], OutputCollector> sinkCall)
      throws IOException {
    TupleEntry tupleEntry = sinkCall.getOutgoingEntry();
    OutputCollector outputCollector = sinkCall.getOutput();
    Tuple key = tupleEntry.selectTuple(keyField);
    byte[] keyBytes = Bytes.toBytes(key.getString(0));
    Put put = new Put(keyBytes);


    for (int i = 0; i < valueFields.length; i++) {
      Fields fieldSelector = valueFields[i];
      TupleEntry values = tupleEntry.selectEntry(fieldSelector);
      
      for (int j = 0; j < values.getFields().size(); j++) {
        Fields fields = values.getFields();
        Tuple tuple = values.getTuple();


        String value = tuple.getString(j);
        byte[] asBytes = value == null ? null : Bytes.toBytes(value);
        put.add(Bytes.toBytes(familyNames[i]), Bytes.toBytes((String) fields.get(j)), asBytes);
      }
    }


    outputCollector.collect(null, put);
  }

View Full Code Here

Examples of org.apache.hadoop.mapred.OutputCollector


    @Override
    public void sink( FlowProcess<JobConf> flowProcess, SinkCall<Object[], OutputCollector> sinkCall ) throws IOException {
        // it's ok to use NULL here so the collector does not write anything
        TupleEntry tupleEntry = sinkCall.getOutgoingEntry();
        OutputCollector outputCollector = sinkCall.getOutput();
        if( updateBy != null )
        {
            Tuple allValues = tupleEntry.selectTuple( updateValueFields );
            Tuple updateValues = tupleEntry.selectTuple( updateByFields );


            allValues = cleanTuple( allValues );


            TupleRecord key = new TupleRecord( allValues );


            if( updateValues.equals( updateIfTuple ) )
                outputCollector.collect( key, null );
            else
                outputCollector.collect( key, key );


            return;
        }


        Tuple result = tupleEntry.selectTuple( getSinkFields() );


        result = cleanTuple( result );


        outputCollector.collect( new TupleRecord( result ), null );
    }

View Full Code Here

Examples of org.apache.hadoop.mapred.OutputCollector

      // validate input split
      InputSplit split = splits[i];
      Assert.assertTrue(split instanceof TableSnapshotInputFormat.TableSnapshotRegionSplit);


      // validate record reader
      OutputCollector collector = mock(OutputCollector.class);
      Reporter reporter = mock(Reporter.class);
      RecordReader<ImmutableBytesWritable, Result> rr = tsif.getRecordReader(split, job, reporter);


      // validate we can read all the data back
      ImmutableBytesWritable key = rr.createKey();

View Full Code Here

Examples of org.apache.hadoop.mapred.OutputCollector

  private void runOldCombiner(final TezRawKeyValueIterator rawIter, final Writer writer) throws IOException {
    Class<? extends Reducer> reducerClazz = (Class<? extends Reducer>) conf.getClass("mapred.combiner.class", null, Reducer.class);
    
    Reducer combiner = ReflectionUtils.newInstance(reducerClazz, conf);
    
    OutputCollector collector = new OutputCollector() {
      @Override
      public void collect(Object key, Object value) throws IOException {
        writer.append(key, value);
      }
    };

View Full Code Here

Examples of org.apache.hadoop.mapred.OutputCollector

    Reducer reducer =
        ReflectionUtils.newInstance(job.getReducerClass(), job);


    // make output collector


    OutputCollector collector =
        new OutputCollector() {
      public void collect(Object key, Object value)
          throws IOException {
        output.write(key, value);
      }
    };

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.