Examples of cascading.pipe.Merge

cascading.pipe.Merge
The Merge Pipe allows for multiple branches, with the same fields to be spliced back into a single stream.
The behavior is similar to the {@link GroupBy} merging features, but Merge does not perform any grouping orsorting on keys. Thus, when using a MapReduce platform, no Reducer is required.
Merge is non-blocking and performs no processing. Any number of branches and merges may be performed in a {@link cascading.flow.Flow} without triggering any additional MapReduce jobs on the Hadoop platform.
Unlike {@link HashJoin}, no preference need be made for left-hand or right-hand sided-ness of streams in relation to their sizes. @see GroupBy

    if( isMerge && isGroup )
      join = new GroupBy( Pipe.pipes( pipeLower, pipeUpper ), numLHS );
    else if( !isMerge && isGroup )
      join = new CoGroup( pipeLower, numLHS, pipeUpper, numRHS, declaredFields, new InnerJoin() );
    else if( isMerge && !isGroup )
      join = new Merge( pipeLower, pipeUpper );
    else
      join = new HashJoin( pipeLower, numLHS, pipeUpper, numRHS, declaredFields, new InnerJoin() );


    Flow flow = null;
    try

View Full Code Here

    {
    Pipe pipe = new Pipe( "foo" );


    pipe = new Each( pipe, new Fields( "a" ), new Identity() );


    pipe = new Merge( pipe, new Pipe( "bar" ) );


    assertEqualsTrace( "cascading.TraceTest.testPipeMerge(TraceTest.java", pipe.getTrace() );
    }

View Full Code Here

    Tap sink = getPlatform().getTextFile( new Fields( "line" ), getOutputPath( "insensitivegrouping" + NONDETERMINISTIC ), SinkMode.REPLACE );


    Pipe pipeLower = new Pipe( "lower" );
    Pipe pipeUpper = new Pipe( "upper" );


    Pipe merge = new Merge( pipeLower, pipeUpper );


    Fields charFields = new Fields( "char" );
    charFields.setComparator( "char", new LowerComparator() );


    Pipe splice = new GroupBy( "groupby", merge, charFields );

View Full Code Here

//    Pipe joined = new CoGroup( messages, new Fields( "id" ), people, new Fields( "id2" ) );
    Pipe joined = new HashJoin( lhs, new Fields( "id" ), rhs, new Fields( "id2" ) );


    Pipe pruned = new Each( joined, new Fields( "id2" ), new Identity(), Fields.RESULTS );
//    pruned = new Checkpoint( pruned );
    Pipe merged = new Merge( pruned, rhs );
    Pipe grouped = new GroupBy( merged, new Fields( "id2" ) );
//    Pipe grouped = new GroupBy( Pipe.pipes(  pruned, people  ), new Fields( "id2" ) );
    Aggregator count = new Count( new Fields( "count" ) );
    Pipe counted = new Every( grouped, count );

View Full Code Here

    Tap sink = getPlatform().getTextFile( new Fields( "line" ), getOutputPath( "simplemerge" ), SinkMode.REPLACE );


    Pipe pipeLower = new Each( new Pipe( "lower" ), new Fields( "line" ), splitter );
    Pipe pipeUpper = new Each( new Pipe( "upper" ), new Fields( "line" ), splitter );


    Pipe splice = new Merge( "merge", pipeLower, pipeUpper );


    Flow flow = getPlatform().getFlowConnector().connect( sources, sink, splice );


    flow.complete();

View Full Code Here


    Pipe pipeLower = new Each( new Pipe( "lower" ), new Fields( "line" ), splitter );
    Pipe pipeUpper = new Each( new Pipe( "upper" ), new Fields( "line" ), splitter );
    Pipe pipeOffset = new Each( new Pipe( "offset" ), new Fields( "line" ), splitter );


    Pipe splice = new Merge( "merge", pipeLower, pipeUpper, pipeOffset );


    splice = new Each( splice, new Fields( "num", "char" ), new Identity() );


    Flow flow = getPlatform().getFlowConnector().connect( sources, sink, splice );

View Full Code Here


    Pipe pipeLower = new Each( new Pipe( "lower" ), new Fields( "line" ), splitter );
    Pipe pipeUpper = new Each( new Pipe( "upper" ), new Fields( "line" ), splitter );
    Pipe pipeOffset = new Each( new Pipe( "offset" ), new Fields( "line" ), splitter );


    Pipe splice = new Merge( "merge", pipeLower, pipeUpper );


    splice = new Merge( splice, pipeOffset );


    splice = new Each( splice, new Fields( "num", "char" ), new Identity() );


    Flow flow = getPlatform().getFlowConnector().connect( sources, sink, splice );

View Full Code Here


    Pipe pipeLower = new Each( new Pipe( "lower" ), new Fields( "line" ), splitter );
    Pipe pipeUpper = new Each( new Pipe( "upper" ), new Fields( "line" ), splitter );
    Pipe pipeOffset = new Each( new Pipe( "offset" ), new Fields( "line" ), splitter );


    Pipe splice = new Merge( "merge", pipeLower, pipeUpper );


    splice = new Merge( splice, pipeOffset );


    splice = new GroupBy( splice, new Fields( "num" ) );


    Flow flow = getPlatform().getFlowConnector().connect( sources, sink, splice );

View Full Code Here


    Pipe pipeLower = new Each( new Pipe( "lower" ), new Fields( "line" ), new RegexSplitter( new Fields( "num1", "char1" ), " " ) );
    Pipe pipeUpper = new Each( new Pipe( "upper" ), new Fields( "line" ), new RegexSplitter( new Fields( "num1", "char1" ), " " ) );
    Pipe pipeOffset = new Each( new Pipe( "offset" ), new Fields( "line" ), new RegexSplitter( new Fields( "num2", "char2" ), " " ) );


    Pipe splice = new Merge( "merge", pipeLower, pipeUpper );


    splice = new CoGroup( splice, new Fields( "num1" ), pipeOffset, new Fields( "num2" ) );


    Flow flow = getPlatform().getFlowConnector().connect( sources, sink, splice );

View Full Code Here


    Pipe pipeLower = new Each( new Pipe( "lower", pipe ), new Fields( "line" ), splitter );
    Pipe pipeUpper = new Each( new Pipe( "upper", pipe ), new Fields( "line" ), splitter );
    Pipe pipeOffset = new Each( new Pipe( "offset", pipe ), new Fields( "line" ), splitter );


    Pipe splice = new Merge( "merge", pipeLower, pipeUpper );


    //put group before merge to test path counts
    splice = new GroupBy( splice, new Fields( "num" ) );


    splice = new Merge( splice, pipeOffset );


    // this group has its incoming paths counted, gated by the previous group
    splice = new GroupBy( splice, new Fields( "num" ) );


    Flow flow = getPlatform().getFlowConnector().connect( sources, sink, splice );

View Full Code Here

0 1 2

TOP

Related Classes of cascading.pipe.Merge

cascading.FieldedPipesPlatformTest

cascading.flow.iso.graph.HashJoinMergeIntoHashJoinStreamedStreamedMergeGraph

cascading.flow.iso.graph.HashJoinsIntoMerge

cascading.flow.iso.graph.MulitStepFlowGraph

cascading.JoinFieldedPipesPlatformTest

cascading.MergePipesPlatformTest

cascading.pipe.assembly.AssemblyHelpersPlatformTest

cascading.TraceTest

cascading.TypedFieldedPipesPlatformTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.