Examples of cascading.flow.Flow

cascading.flow.Flow
A Flow is a logical unit of work declared by an assembly of {@link cascading.pipe.Pipe} instances connected to sourceand sink {@link Tap} instances.
A Flow is then executed to push the incoming source data through the assembly into one or more sinks.
A Flow sub-class instance may not be instantiated directly in most cases, see sub-classes of {@link FlowConnector} classfor supported platforms.
Note that {@link cascading.pipe.Pipe} assemblies can be reused in multiple Flow instances. They maintainno state regarding the Flow execution. Subsequently, {@link cascading.pipe.Pipe} assemblies can be givenparameters through its calling Flow so they can be built in a generic fashion.
When a Flow is created, an optimized internal representation is created that is then executed on the underlying execution platform. This is typically done by creating one or more {@link FlowStep} instances.
Flows are submitted in order of dependency when used with a {@link cascading.cascade.Cascade}. If two or more steps do not share the same dependencies and all can be scheduled simultaneously, the {@link #getSubmitPriority()} value determinesthe order in which all steps will be submitted for execution. The default submit priority is 5.
Use the {@link FlowListener} to receive any events on the life-cycle of the Flow as it executes. Any{@link Tap} instances owned by the Flow also implementing FlowListener will automatically be added to theset of listeners. @see FlowListener @see cascading.flow.FlowConnector

    pipe = new Each( pipe, new Fields( "count1", "count2" ), new ExpressionFunction( new Fields( "sum" ), "count1 + count2", int.class ), Fields.ALL );


    Tap source = getPlatform().getTextFile( new Fields( "offset", "line" ), inputFileApache );
    Tap sink = getPlatform().getTextFile( getOutputPath( "chaineach" ), SinkMode.REPLACE );


    Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe );


    flow.complete();


    validateLength( flow, 8, null );
    }

View Full Code Here


    pipe = new Each( pipe, new RegexSplitter( "\\s+" ), new Fields( 1 ) );


    Tap sink = getPlatform().getTextFile( new Fields( "line" ), getOutputPath( "nogroup" ), SinkMode.REPLACE );


    Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe );


    flow.complete();


    validateLength( flow, 10, null );


    List<Tuple> results = getSinkAsList( flow );

View Full Code Here


    Pipe pipe = new Pipe( "test" );


    Tap sink = getPlatform().getTextFile( getOutputPath( "copy" ), SinkMode.REPLACE );


    Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe );


    flow.complete();


    validateLength( flow, 10, null );
    }

View Full Code Here

    Pipe pipeLower = new Each( new Pipe( "lower" ), new Fields( "line" ), splitter );
    Pipe pipeUpper = new Each( new Pipe( "upper" ), new Fields( "line" ), splitter );


    Pipe splice = new GroupBy( "merge", Pipe.pipes( pipeLower, pipeUpper ), new Fields( "num" ), null, false );


    Flow flow = getPlatform().getFlowConnector().connect( sources, sink, splice );


    flow.complete();


    validateLength( flow, 10 );


    Collection results = getSinkAsList( flow );

View Full Code Here


    splice = new Every( splice, new Fields( "char" ), new First( new Fields( "first" ) ) );


    splice = new Each( splice, new Fields( "num", "first" ), new Identity() );


    Flow flow = getPlatform().getFlowConnector().connect( sources, sink, splice );


    flow.complete();


    validateLength( flow, 6 );


    List<Tuple> tuples = getSinkAsList( flow );

View Full Code Here

    Pipe splice = new GroupBy( Pipe.pipes( pipeLower, pipeUpper ), new Fields( "num" ) );


    // this group has its incoming paths counted, gated by the previous group
    splice = new GroupBy( Pipe.pipes( splice, pipeOffset ), new Fields( "num" ) );


    Flow flow = getPlatform().getFlowConnector().connect( sources, sink, splice );


    if( getPlatform().isMapReduce() )
      assertEquals( "wrong num jobs", 2, flow.getFlowSteps().size() );


    flow.complete();


    validateLength( flow, 15 );
    }

View Full Code Here


    pipe = new Each( pipe, new Fields( "line" ), new RegexSplitter( new Fields( "num", "lower", "upper" ) ) );


    pipe = new Each( pipe, new UnGroup( new Fields( "num", "char" ), new Fields( "num" ), Fields.fields( new Fields( "lower" ), new Fields( "upper" ) ) ) );


    Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe );


    flow.complete();


    validateLength( flow, 10 );
    }

View Full Code Here


    pipe = new Each( pipe, new Fields( "line" ), new RegexSplitter( new Fields( "num", "lower", "upper" ) ) );


    pipe = new Each( pipe, new UnGroup( new Fields( "num" ), Fields.fields( new Fields( "lower" ), new Fields( "upper" ) ) ) );


    Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe );


    flow.complete();


    validateLength( flow, 10 );
    }

View Full Code Here


    pipe = new Each( pipe, new Fields( "line" ), new RegexSplitter( new Fields( "num1", "num2", "lower", "upper" ) ) );


    pipe = new Each( pipe, new UnGroup( new Fields( "num1", "num2", "char" ), new Fields( "num1", "num2" ), 1 ) );


    Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe );


    flow.complete();


    List<Tuple> tuples = asList( flow, sink );
    assertEquals( 10, tuples.size() );


    List<Object> values = new ArrayList<Object>();

View Full Code Here


    Filter filter = new RegexFilter( "^68.*" );


    pipe = new Each( pipe, new Fields( "line" ), filter );


    Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe );


    flow.complete();


    validateLength( flow, 3 );
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of cascading.flow.Flow

bixo.examples.crawl.DemoCrawlTool

bixo.examples.crawl.DemoCrawlWorkflow

bixo.examples.crawl.DemoCrawlWorkflowLRTest

bixo.examples.crawl.LatestUrlDatumBufferTest

bixo.examples.crawl.UrlImporter

bixo.examples.webmining.DemoWebMiningTool

bixo.examples.webmining.DemoWebMiningWorkflow

bixo.examples.webmining.DemoWebMiningWorkflowTest

bixo.fetcher.FetcherTest

bixo.pipes.AbstractFetchPipeTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.