Examples of cascading.pipe.Every

cascading.pipe.Every
The Every operator applies an {@link Aggregator} or {@link Buffer} to every grouping.
Any number of Every instances may follow other Every, {@link GroupBy}, or {@link CoGroup} instances if they apply anAggregator, not a Buffer. If a Buffer, only one Every may follow a GroupBy or CoGroup.
Every operators create aggregate values for every grouping they encounter. This aggregate value is added to the current grouping Tuple.
In the case of a CoGroup, the grouping Tuple will be all the grouping keys from all joined streams, and if an "outer" type join is used, one value on the groupingTuple may be null.
Subsequent Every instances can continue to append values to the grouping Tuple. When an Each follows and Every, the Each applies its operation to the grouping Tuple (thus all child values in the grouping are discarded and only aggregate values are propagated).


    // always fail
    pipe = new Each( pipe, new Fields( "ip" ), new TestFunction( new Fields( "test" ), null ), Fields.ALL );


    pipe = new GroupBy( "reduce", pipe, new Fields( "ip" ) );
    pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) );


    Tap sink = getPlatform().getTextFile( "foosink" );
    Tap trap = getPlatform().getTextFile( "footrap" );


    Map<String, Tap> sources = new HashMap<String, Tap>();

View Full Code Here


    pipe = new Pipe( "middle", pipe );
    pipe = new Each( pipe, new Fields( "ip" ), new TestFunction( new Fields( "test" ), null ), Fields.ALL );


    pipe = new GroupBy( "reduce", pipe, new Fields( "ip" ) );
    pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) );


    Tap sink = getPlatform().getTextFile( "foosink" );
    Tap trap = getPlatform().getTextFile( "footrap" );


    Map<String, Tap> sources = new HashMap<String, Tap>();

View Full Code Here


    pipe = new Each( pipe, new Fields( "ip" ), new TestFunction( new Fields( "test" ), null ), Fields.ALL );


    pipe = new GroupBy( pipe, new Fields( "ip" ) );
    pipe = new Pipe( "first", pipe );
    pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) );
    pipe = new Pipe( "second", pipe );
    pipe = new Every( pipe, new Count( new Fields( "count2" ) ), new Fields( "ip", "count", "count2" ) );


    Tap sink = getPlatform().getTextFile( "foosink" );
    Tap trap = getPlatform().getTextFile( "footrap" );


    Map<String, Tap> sources = new HashMap<String, Tap>();

View Full Code Here


    Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " );


    Pipe pipeLower = new Each( new Pipe( "lower" ), new Fields( "line" ), splitter );
    pipeLower = new GroupBy( pipeLower, new Fields( "num" ) );
    pipeLower = new Every( pipeLower, new Fields( "char" ), new First(), Fields.ALL );


    Pipe pipeUpper = new Each( new Pipe( "upper" ), new Fields( "line" ), splitter );
    pipeUpper = new GroupBy( pipeUpper, new Fields( "num" ) );
    pipeUpper = new Every( pipeUpper, new Fields( "char" ), new First(), Fields.ALL );


    Pipe splice = new HashJoin( pipeLower, new Fields( "num" ), pipeUpper, new Fields( "num" ), Fields.size( 4 ) );


    Flow flow = getPlatform().getFlowConnector().connect( sources, sink, splice );

View Full Code Here

    Pipe pipeUpper = new Each( new Pipe( "upper" ), new Fields( "line" ), splitterUpper );
    Pipe pipeJoined = new Each( new Pipe( "joined" ), new Fields( "line" ), splitterJoined );


    Pipe pipe = new GroupBy( pipeLower, new Fields( "numA" ) );


    pipe = new Every( pipe, Fields.ALL, new TestIdentityBuffer( new Fields( "numA" ), 5, false ), Fields.RESULTS );


    Pipe lhsPipe = new Each( pipe, new Identity() );
    lhsPipe = new HashJoin( "lhs", lhsPipe, new Fields( "numA" ), pipeUpper, new Fields( "numB" ) );


    Pipe rhsPipe = new Each( pipe, new Identity() );

View Full Code Here

//    pruned = new Checkpoint( pruned );
    Pipe merged = new Merge( pruned, rhs );
    Pipe grouped = new GroupBy( merged, new Fields( "id2" ) );
//    Pipe grouped = new GroupBy( Pipe.pipes(  pruned, people  ), new Fields( "id2" ) );
    Aggregator count = new Count( new Fields( "count" ) );
    Pipe counted = new Every( grouped, count );


    String testJoinMerge = "testJoinMergeGroupBy/" + ( ( joined instanceof CoGroup ) ? "cogroup" : "hashjoin" );
    Tap sink = getPlatform().getDelimitedFile( Fields.ALL, true, "\t", null, getOutputPath( testJoinMerge ), SinkMode.REPLACE );


    FlowDef flowDef = FlowDef.flowDef()

View Full Code Here


    pipe = new Checkpoint( pipe );


    pipe = new GroupBy( pipe, new Fields( "ip" ) );


    pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) );


    Tap sink = getPlatform().getTextFile( getOutputPath( "simplecheckpoint" ), SinkMode.REPLACE );


    Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe );

View Full Code Here

    }


    { // job 2
    pipe = new GroupBy( pipe, new Fields( "ip" ) );


    pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) );


    pipe = new Checkpoint( pipe );


    new FlowConnectorProps()
      .setCheckpointTapDecoratorClassName( DecoratorTap.class )

View Full Code Here


    pipe = new Checkpoint( "checkpoint", pipe );


    pipe = new GroupBy( pipe, new Fields( "ip" ) );


    pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) );


    Tap sink = getPlatform().getTextFile( getOutputPath( "checkpoint/sink" ), SinkMode.REPLACE );


    Tap checkpoint = getPlatform().getDelimitedFile( Fields.ALL, true, getOutputPath( "checkpoint/tap" ), SinkMode.REPLACE );

View Full Code Here


    pipe = new Checkpoint( "checkpoint", pipe );


    pipe = new GroupBy( pipe, new Fields( "ip" ) );


    pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) );


    Tap sink = getPlatform().getTextFile( getOutputPath( "failcheckpoint/sink" ), SinkMode.REPLACE );


    Tap checkpoint = getPlatform().getDelimitedFile( Fields.ALL, true, getOutputPath( "failcheckpoint/tap" ), SinkMode.REPLACE );

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of cascading.pipe.Every

bixo.examples.crawl.DemoCrawlWorkflow

bixo.examples.crawl.LatestUrlDatumBufferTest

bixo.examples.webmining.DemoWebMiningWorkflow

bixo.pipes.FetchPipe

cascading.assembly.CrossTab

cascading.BasicPipesPlatformTest

cascading.BasicTrapPlatformTest

cascading.BufferPipesPlatformTest

cascading.CoGroupFieldedPipesPlatformTest

cascading.detail.EveryAssemblyFactory

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.