Package cascading.pipe

Examples of cascading.pipe.Each

54
55
56
57
58
59
60
61
62
63
64
    {
    Tap source = getPlatform().getTextFile( inputFileIps );

    Pipe pipe = new Pipe( "first" );

    pipe = new Each( pipe, new Fields( "line" ), new Identity( new Fields( "ip" ) ), new Fields( "ip" ) );

    Tap sink = getPlatform().getTabDelimitedFile( new Fields( "ip" ), getOutputPath( path + "/first" ), SinkMode.REPLACE );

    return getPlatform().getFlowConnector().connect( source, sink, pipe );
    }
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
  private Flow secondFlow( Tap source, String path )
    {
    Pipe pipe = new Pipe( "second" );

    pipe = new Each( pipe, new RegexSplitter( new Fields( "first", "second", "third", "fourth" ), "\\." ) );

    Tap sink = getPlatform().getTabDelimitedFile( new Fields( "first", "second", "third", "fourth" ), getOutputPath( path + "/second" ), SinkMode.REPLACE );

    return getPlatform().getFlowConnector().connect( source, sink, pipe );
    }
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
  private Flow thirdFlow( Tap source, String path )
    {
    Pipe pipe = new Pipe( "third" );

    pipe = new Each( pipe, new FieldJoiner( new Fields( "mangled" ), "-" ) );

    Tap sink = getPlatform().getTabDelimitedFile( new Fields( "mangled" ), getOutputPath( path + "/third" ), SinkMode.REPLACE );

    return getPlatform().getFlowConnector().connect( source, sink, pipe );
    }
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
  private Flow fourthFlow( Tap source, String path )
    {
    Pipe pipe = new Pipe( "fourth" );

    pipe = new Each( pipe, new Identity() );

    Tap sink = getPlatform().getTextFile( getOutputPath( path + "/fourth" ), SinkMode.REPLACE );

    return getPlatform().getFlowConnector().connect( source, sink, pipe );
    }
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
    getPlatform().copyFromLocal( inputFileApache );

    Tap source = getPlatform().getTextFile( inputFileApache );

    Pipe pipe = new Pipe( "failing mapper" );
    pipe = new Each( pipe, new Fields( "line" ), new FailFunction( new Fields( "ip" ) ), new Fields( "ip" ) );

    Tap sink = getPlatform().getTextFile( getOutputPath( "mapperfail" ), SinkMode.REPLACE );

    Flow flow = getPlatform().getFlowConnector().connect( "mapper fail test", source, sink, pipe );
View Full Code Here
117
118
119
120
121
122
123
124
125
126
127
    getPlatform().copyFromLocal( inputFileApache );

    Tap source = getPlatform().getTextFile( inputFileApache );

    Pipe pipe = new Pipe( "failing reducer" );
    pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) );
    pipe = new GroupBy( pipe, new Fields( "ip" ) );
    pipe = new Every( pipe, new TestFailAggregator( new Fields( "count" ), 1 ) );

    Tap sink = getPlatform().getTextFile( getOutputPath( "reducerfail" ), SinkMode.REPLACE );

View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
87
88
89
    {
    Tap source = getPlatform().getTextFile( inputFileIps );

    Pipe pipe = new Pipe( "first" );

    pipe = new Each( pipe, new Fields( "line" ), new Identity( new Fields( "ip" ) ), new Fields( "ip" ) );

    if( doFail )
      pipe = new Each( pipe, new Fields( "ip" ), new FailFunction( Fields.ARGS ), Fields.REPLACE );

    Tap sink = getPlatform().getTabDelimitedFile( new Fields( "ip" ), getOutputPath( path ), SinkMode.REPLACE );

    return getPlatform().getFlowConnector().connect( source, sink, pipe );
    }
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
  private Flow secondFlow( Tap source, String path )
    {
    Pipe pipe = new Pipe( "second" );

    pipe = new Each( pipe, new RegexSplitter( new Fields( "first", "second", "third", "fourth" ), "\\." ) );

    Tap sink = getPlatform().getTabDelimitedFile( new Fields( "first", "second", "third", "fourth" ), getOutputPath( path ), SinkMode.REPLACE );

    return getPlatform().getFlowConnector().connect( source, sink, pipe );
    }
View Full Code Here
101
102
103
104
105
106
107
108
109
110
111
  private Flow thirdFlow( Tap source, String path )
    {
    Pipe pipe = new Pipe( "third" );

    pipe = new Each( pipe, new FieldJoiner( new Fields( "mangled" ), "-" ) );

    Tap sink = getPlatform().getTabDelimitedFile( new Fields( "mangled" ), getOutputPath( path ), SinkMode.REPLACE );

    return getPlatform().getFlowConnector().connect( source, sink, pipe );
    }
View Full Code Here
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
  private Flow thirdCheckpointFlow( Tap source, String path )
    {
    Pipe pipe = new Pipe( "third" );

    pipe = new Each( pipe, new FieldJoiner( new Fields( "mangled" ), "-" ) );

    pipe = new Checkpoint( "checkpoint", pipe );

    pipe = new Each( pipe, new Identity() );

    Tap sink = getPlatform().getTabDelimitedFile( new Fields( "mangled" ), getOutputPath( "unusedpath" ), SinkMode.REPLACE );
    Tap checkpoint = getPlatform().getTabDelimitedFile( Fields.ALL, getOutputPath( path ), SinkMode.REPLACE );

    FlowDef flowDef = FlowDef.flowDef()
View Full Code Here
TOP

Related Classes of cascading.pipe.Each

  • cascading.BasicPipesPlatformTest
  • cascading.BasicTrapPlatformTest
  • cascading.BufferPipesPlatformTest
  • cascading.cascade.CascadePlatformTest
  • cascading.cascade.hadoop.RiffleCascadePlatformTest
  • cascading.cascade.ParallelCascadePlatformTest
  • cascading.CoGroupFieldedPipesPlatformTest
  • cascading.DistanceUseCasePlatformTest
  • cascading.FieldedPipesPlatformTest
  • cascading.flow.AssemblyPlannerPlatformTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.