Package com.tdunning.plume

Examples of com.tdunning.plume.PCollection

60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
    @Override
    public void build() {
      init();
     
      LazyPlume plume = new LazyPlume();
      PCollection input;
      try {
        // Read input
        input = plume.readFile("/tmp/input-wordcount.txt", collectionOf(strings()));
        // Add it as workflow's input
        addInput(input);
      } catch (IOException e) {
        throw new RuntimeException();
      }
     
      PCollection bypassTransform = input.map(new DoFn() {
        @Override
        public void process(Object v, EmitFn emitter) {
          Text t = (Text)v;
          emitter.emit(Pair.create(new Text(t + "-blah"), new Text(t + "-bloh")));
        }}, tableOf(strings(), strings()));
     
      addOutput(bypassTransform);
     
      PCollection groupedTransform = input.map(new DoFn() {
        @Override
        public void process(Object v, EmitFn emitter) {
          Text t = (Text)v;
          emitter.emit(Pair.create(t, new Text("foo")));
        }}, tableOf(strings(), strings())).groupByKey();
View Full Code Here
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
    public void build() {
      init();

      LazyPlume plume = new LazyPlume();
      // Get input files
      PCollection inputEvent2, inputLogFile, inputLogFile2;
      try {
        inputEvent2   = plume.readFile(inputPathEvent2, collectionOf(strings()));
        inputLogFile2 = plume.readFile(inputPathLogFile2, collectionOf(strings()));
        inputLogFile  = plume.readFile(inputPathLogFile, collectionOf(strings()));
        // Add as inputs
        addInput(inputEvent2);
        addInput(inputLogFile);
      } catch (IOException e) {
        throw new RuntimeException();
      }     
     
      /**
       * Emit the user of the log file - flatten it with users file
       */
      PCollection output = plume.flatten(collectionOf(strings()),
        inputEvent2,
        inputLogFile.map(new DoFn<Text, Text>() {
          @Override
          public void process(Text v, EmitFn<Text> emitter) {
            String[] splittedLine = v.toString().split("\t");
            emitter.emit(new Text(splittedLine[2]));
          }
        }, collectionOf(strings())));
     
      /**
       * Flatten two log files
       */
      PCollection output2 = plume.flatten(collectionOf(strings()),
        inputLogFile2,
        inputLogFile);
     
      addOutput(output);
      addOutput(output2);
View Full Code Here
TOP

Related Classes of com.tdunning.plume.PCollection

  • com.tdunning.plume.local.lazy.MapRedBypassTest$MapRedBypassWorkflow
  • com.tdunning.plume.local.lazy.MapRedFlattenTest$MapRedFlattenTestWorkflow
  • com.tdunning.plume.local.lazy.MapRedMultipleGroupsTest$MultipleGroupsWorkflow
  • com.tdunning.plume.local.lazy.MapRedOnlyFlattensTest$MapRedOnlyFlattensTestWorkflow
  • com.tdunning.plume.local.lazy.MapRedSequenceFileTest$OtherWorkflow
  • com.tdunning.plume.local.lazy.MapRedSingleFlattenChannelTest$MapRedSingleFlattenChannelTestWorkflow
  • com.tdunning.plume.local.lazy.MapRedTwoSequentialGBKTest$TwoSequentialGBKWorkflow
  • com.tdunning.plume.local.lazy.MapRedWordCountTest$WordCountWorkflow
  • com.tdunning.plume.local.lazy.MSCRCombiner
  • com.tdunning.plume.local.lazy.MSCRReducer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.