Examples of cascading.pipe.Merge

cascading.pipe.Merge
The Merge Pipe allows for multiple branches, with the same fields to be spliced back into a single stream.
The behavior is similar to the {@link GroupBy} merging features, but Merge does not perform any grouping orsorting on keys. Thus, when using a MapReduce platform, no Reducer is required.
Merge is non-blocking and performs no processing. Any number of branches and merges may be performed in a {@link cascading.flow.Flow} without triggering any additional MapReduce jobs on the Hadoop platform.
Unlike {@link HashJoin}, no preference need be made for left-hand or right-hand sided-ness of streams in relation to their sizes. @see GroupBy

    pipe = new Each( pipe, new Fields( "line" ), new RegexFilter( "^68.*" ) );


    Pipe left = new Each( new Pipe( "left", pipe ), new Fields( "line" ), new RegexFilter( ".*46.*" ) );
    Pipe right = new Each( new Pipe( "right", pipe ), new Fields( "line" ), new RegexFilter( ".*102.*" ) );


    Pipe merged = new Merge( "merged", left, right );


    merged = new Each( merged, new Fields( "line" ), new Identity() );


    Flow flow = getPlatform().getFlowConnector().connect( source, sink, merged );

View Full Code Here

    pipe = new Each( pipe, new Fields( "line" ), new RegexFilter( "^68.*" ) );


    Pipe left = new Each( new Pipe( "left", pipe ), new Fields( "line" ), new RegexFilter( ".*46.*" ) );
    Pipe right = new Each( new Pipe( "right", pipe ), new Fields( "line" ), new RegexFilter( ".*102.*" ) );


    Pipe merged = new Merge( "merged-first", left, right );


    merged = new Each( merged, new Fields( "line" ), new Identity() );


    left = new Each( new Pipe( "left", merged ), new Fields( "line" ), new RegexFilter( ".*46.*" ) );
    right = new Each( new Pipe( "right", merged ), new Fields( "line" ), new RegexFilter( ".*102.*" ) );


    merged = new Merge( "merged-second", left, right );


    merged = new Each( merged, new Fields( "line" ), new Identity() );


    Flow flow = getPlatform().getFlowConnector().connect( source, sink, merged );

View Full Code Here

    Pipe pipeLower = new Each( new Pipe( "lower" ), new Fields( "line" ), splitter );
    pipeLower = new Rename( pipeLower, new Fields( "num" ), new Fields( "num2" ) );


    Pipe pipeUpper = new Each( new Pipe( "upper" ), new Fields( "line" ), splitter );


    Pipe splice = new Merge( "merge", pipeLower, pipeUpper );


    try
      {
      Flow flow = getPlatform().getFlowConnector().connect( sources, sink, splice );
      fail();

View Full Code Here


    Pipe pipeLower = new Each( new Pipe( "lower" ), new Fields( "line" ), new RegexSplitter( new Fields( "num1", "char1" ), " " ) );
    Pipe pipeUpper = new Each( new Pipe( "upper" ), new Fields( "line" ), new RegexSplitter( new Fields( "num1", "char1" ), " " ) );
    Pipe pipeOffset = new Each( new Pipe( "offset" ), new Fields( "line" ), new RegexSplitter( new Fields( "num2", "char2" ), " " ) );


    Pipe splice = new Merge( "merge", pipeLower, pipeUpper );


    if( streamed )
      splice = new HashJoin( splice, new Fields( "num1" ), pipeOffset, new Fields( "num2" ) );
    else
      splice = new HashJoin( pipeOffset, new Fields( "num2" ), splice, new Fields( "num1" ) );

View Full Code Here


    Pipe splice = new HashJoin( pipeLower, new Fields( "num1" ), pipeOffset, new Fields( "num2" ) );


    splice = new Retain( splice, new Fields( "num1", "char1" ) );


    splice = new Merge( "merge", splice, pipeUpper );


    if( streamed )
      splice = new HashJoin( splice, new Fields( "num1" ), pipeOffset, new Fields( "num2" ) );
    else
      splice = new HashJoin( pipeOffset, new Fields( "num2" ), splice, new Fields( "num1" ) );

View Full Code Here


    Pipe splice = new HashJoin( pipeLower, new Fields( "num1" ), pipeOffset, new Fields( "num2" ) );


    splice = new Retain( splice, new Fields( "num1", "char1" ) );


    splice = new Merge( "merge1", splice, pipeUpper );


    if( firstStreamed )
      splice = new HashJoin( splice, new Fields( "num1" ), pipeOffset, new Fields( "num2" ) );
    else
      splice = new HashJoin( pipeOffset, new Fields( "num2" ), splice, new Fields( "num1" ) );


    splice = new Retain( splice, new Fields( "num1", "char1" ) );


    if( interMerge )
      splice = new Merge( "merge2", splice, pipeUpper );


    if( secondStreamed )
      splice = new HashJoin( splice, new Fields( "num1" ), pipeOffset, new Fields( "num2" ) );
    else
      splice = new HashJoin( pipeOffset, new Fields( "num2" ), splice, new Fields( "num1" ) );

View Full Code Here

    Fields charFields = new Fields( "char" );
    charFields.setComparator( "char", new CustomHasher() );


    if( priorMerge )
      {
      Merge merge = new Merge( Pipe.pipes( lhsPipe, rhsPipe ) );


      pipe = new AggregateBy( "name", merge, charFields, 2, sumPipe, countPipe, averagePipe );
      }
    else
      {

View Full Code Here

    Pipe rhsPipe = new Pipe( "count-rhs", sourcePipe );


    // redundant, but illustrates case
    rhsPipe = new Each( rhsPipe, new Fields( "char" ), new ExpressionFunction( Fields.ARGS, "$0.toLowerCase()", String.class ), Fields.REPLACE );


    Pipe merge = new Merge( "first", lhsPipe, rhsPipe );
    Pipe countPipe = new CountBy( merge, new Fields( "char" ), new Fields( "count" ), 2 );


    lhsPipe = new Each( new Pipe( "lhs", countPipe ), new Identity() );
    rhsPipe = new Each( new Pipe( "rhs", countPipe ), new Identity() );


    merge = new Merge( "second", lhsPipe, rhsPipe );


    countPipe = new CountBy( merge, new Fields( "char" ), new Fields( "count" ), 2 );


    Map<String, Tap> tapMap = Cascades.tapsMap( sourcePipe, source );

View Full Code Here


    Pipe lhsRhs = new HashJoin( pipeLhs, new Fields( "num" ), pipeRhs, new Fields( "num" ), new Fields( "num1", "char1", "num2", "char2" ) );


    lhsRhs = new Each( lhsRhs, new Identity() );


    Pipe merge = new Merge( "sink", Pipe.pipes( upperLower, lhsRhs ) );


    initialize( sources, sinks, merge );
    }

View Full Code Here

    upper = new Checkpoint( upper );


    lower = new Checkpoint( lower );
    upper = new Checkpoint( upper );


    Pipe sink = new Merge( "sink", lower, upper );


    Map<String, Tap> sources = createHashMap();


    sources.put( lower.getName(), new NonTap( new Fields( "offset", "line" ) ) );
    sources.put( upper.getName(), new NonTap( new Fields( "offset", "line" ) ) );


    Map<String, Tap> sinks = createHashMap();


    sinks.put( sink.getName(), new NonTap( new Fields( "offset", "line" ) ) );


    initialize( sources, sinks, sink );
    }

View Full Code Here

0 1 2

TOP

Related Classes of cascading.pipe.Merge

cascading.FieldedPipesPlatformTest

cascading.flow.iso.graph.HashJoinMergeIntoHashJoinStreamedStreamedMergeGraph

cascading.flow.iso.graph.HashJoinsIntoMerge

cascading.flow.iso.graph.MulitStepFlowGraph

cascading.JoinFieldedPipesPlatformTest

cascading.MergePipesPlatformTest

cascading.pipe.assembly.AssemblyHelpersPlatformTest

cascading.TraceTest

cascading.TypedFieldedPipesPlatformTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.