Examples of DoFn

com.tdunning.plume.DoFn
Describes the key functional object that processes an input record and outputs zero or more objects by means of an emitter object.
org.apache.crunch.DoFn
Base class for all data processing functions in Crunch.
Note that all {@code DoFn} instances implement {@link Serializable}, and thus all of their non-transient member variables must implement {@code Serializable} as well. If your DoFn depends on non-serializableclasses for data processing, they may be declared as {@code transient} andinitialized in the DoFn's {@code initialize} method.

Examples of com.tdunning.plume.DoFn

    PCollection col  = mscr.getChannelByNumber().get(arg0.sourceId);
    OutputChannel oC = mscr.getOutputChannels().get(col);
    if(oC.reducer != null) {
      // apply reducer
      ParallelDo pDo = oC.reducer;
      DoFn reducer = pDo.getFunction(); // TODO how to check / report this
      List<WritableComparable> vals = Lists.newArrayList();
      for(PlumeObject val: values) {
        vals.add(val.obj);
      }
      reducer.process(Pair.create(arg0.obj, vals), new EmitFn() {
      @Override
        public void emit(Object v) {
          try {
            if(v instanceof Pair) {
              Pair p = (Pair)v;

View Full Code Here

Examples of com.tdunning.plume.DoFn

      // MultipleParallelDo -> parallel operations that read the same collection
      // In this version of executor, we will only compute the current collection, not its neighbors
      } else if(op instanceof MultipleParallelDo) {
        MultipleParallelDo mPDo = (MultipleParallelDo) op;
        parent = execute((LazyCollection)mPDo.getOrigin());
        DoFn function = (DoFn)mPDo.getDests().get(output); // get the function that corresponds to this collection
        for (Object obj : parent) {
          function.process(obj, emitter);
        }
      // GroupByKey
      } else if(op instanceof GroupByKey) {
        GroupByKey gBK = (GroupByKey) op;
        parent = execute(gBK.getOrigin());

View Full Code Here

Examples of com.tdunning.plume.DoFn

        // Upper parallel do is CombineValues and follows a GroupByKey -> don't join
        fuseParallelDos(orig1);
        return;
      }
    }
    final DoFn f1 = p1.getFunction();
    final DoFn f2 = p2.getFunction();
    // Define the joined function
    DoFn newFn = new DoFn() {
      @Override
      public void process(Object v, final EmitFn emitter) {
        f2.process(v, new EmitFn() {
          @Override
          public void emit(Object v) {

View Full Code Here

Examples of com.tdunning.plume.DoFn

      } catch (IOException e) {
        throw new RuntimeException();
      }
      
      // Define the wordcount map
      DoFn wordCountMap = new DoFn() {
        @Override
        public void process(Object v, EmitFn emitter) {
          StringTokenizer itr = new StringTokenizer(v.toString());
          while (itr.hasMoreTokens()) {
            emitter.emit(Pair.create(new Text(itr.nextToken()), new IntWritable(1)));

View Full Code Here

Examples of com.tdunning.plume.DoFn

        addInput(input);
      } catch (IOException e) {
        throw new RuntimeException();
      }
      
      PCollection output = input.map(new DoFn() {
        @Override
        public void process(Object v, EmitFn emitter) {
          Text t = (Text)v;
          // do some foo processing
          emitter.emit(Pair.create(t, new Text("foo")));
        }}, tableOf(strings(), strings()))
         .groupByKey()
         .map(new DoFn() { 
        public void process(Object v, EmitFn emitter) {
          Pair p = (Pair)v;
          // do some more foo processing         
          emitter.emit(Pair.create(p.getKey(), new Text("bar")));
        }
      }, tableOf(strings(), strings()))
        // second group by key
         .groupByKey()
         .map(new DoFn() { 
        public void process(Object v, EmitFn emitter) {
          Pair p = (Pair)v;
          // do some more foo processing         
          emitter.emit(Pair.create(p.getKey(), new Text("bar 2")));
        }

View Full Code Here

Examples of com.tdunning.plume.DoFn

      }
      
      final IntWritable one = new IntWritable(1);
      
      // Define a map that counts and group by #chars of line
      PCollection po1 = input.map(new DoFn() {
        @Override
        public void process(Object v, EmitFn emitter) {
          StringTokenizer itr = new StringTokenizer(v.toString());
          int length = 0;
          while (itr.hasMoreTokens()) {
            length += itr.nextToken().length();
          }
          emitter.emit(Pair.create(new IntWritable(length), one));
        }
      }, tableOf(integers(), integers()))
       .groupByKey()
       .map(countReduceToText, tableOf(integers(), strings()));
      
      // Define a map that counts and group by #tokens of line
      PCollection po2 = input.map(new DoFn() {
        @Override
        public void process(Object v, EmitFn emitter) {
          StringTokenizer itr = new StringTokenizer(v.toString());
          int length = 0;
          while (itr.hasMoreTokens()) {
            length ++;
            itr.nextToken();
          }
          emitter.emit(Pair.create(new IntWritable(length), one));
        }
      }, tableOf(integers(), integers()))
       .groupByKey()
       .map(countReduceToText, tableOf(integers(), strings()));
      
      // Define a map that counts appearances of chars
      PCollection po3 = input.map(new DoFn() {
        @Override
        public void process(Object v, EmitFn emitter) {
          StringTokenizer itr = new StringTokenizer(v.toString());
          while (itr.hasMoreTokens()) {
            String token = itr.nextToken();

View Full Code Here

Examples of com.tdunning.plume.DoFn

        addInput(input);
      } catch (IOException e) {
        throw new RuntimeException();
      }
      
      PCollection transform = input.map(new DoFn() {
        @Override
        public void process(Object v, EmitFn emitter) {
          Text t = (Text)v;
          emitter.emit(new Text(t.toString()+"-bar"));
        }}, collectionOf(strings()));
      
      addOutput(plume.flatten(input2, transform)); // flatten with another file
      
      PCollection groupedTransform = input.map(new DoFn() {
        @Override
        public void process(Object v, EmitFn emitter) {
          Text t = (Text)v;
          emitter.emit(Pair.create(t, new Text("foo")));
        }}, tableOf(strings(), strings())).groupByKey();

View Full Code Here

Examples of com.tdunning.plume.DoFn

        addInput(input);
      } catch (IOException e) {
        throw new RuntimeException();
      }
      
      PCollection bypassTransform = input.map(new DoFn() {
        @Override
        public void process(Object v, EmitFn emitter) {
          Text t = (Text)v;
          emitter.emit(Pair.create(new Text(t + "-blah"), new Text(t + "-bloh")));
        }}, tableOf(strings(), strings()));
      
      addOutput(bypassTransform);
      
      PCollection groupedTransform = input.map(new DoFn() {
        @Override
        public void process(Object v, EmitFn emitter) {
          Text t = (Text)v;
          emitter.emit(Pair.create(t, new Text("foo")));
        }}, tableOf(strings(), strings())).groupByKey();

View Full Code Here

Examples of org.apache.crunch.DoFn

      }
      
      Map<String, Object> settings = new HashMap<String, Object>();
      settings.put(TypedSettings.DRY_RUN_SETTING_NAME, opts.isDryRun);
      
      DoFn morphlineFn = new MorphlineFn(
          morphlineFileContents, 
          opts.morphlineId, 
          morphlineVariables,
          settings,
          opts.inputFileFormat != null

View Full Code Here

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.