Examples of cascading.flow.FlowDef

cascading.flow.FlowDef
Class FlowDef is a fluent interface for defining a {@link Flow}.
This allows for ad-hoc building of Flow data and meta-data, like tags.
Instead of calling one of the {@link FlowConnector} connect methods, {@link FlowConnector#connect(FlowDef)}can be called.

      pipe = new Discard( pipe, discardFields );


    Tap source = getPlatform().getDelimitedFile( "\t", "\"", planner.getFieldTypeResolver(), DATA_PATH + testModel + ".tsv", SinkMode.KEEP );
    Tap sink = getPlatform().getDelimitedFile( "\t", "\"", null, getResultPath(), SinkMode.REPLACE );


    FlowDef flowDef = FlowDef.flowDef()
      .addSource( "head", source )
      .addSink( "tail", sink )
      .addTail( pipe )
      .addAssemblyPlanner( planner );

View Full Code Here

    pipe = new Pipe( "tail", pipe );


    Tap source = getPlatform().getDelimitedFile( expectedFields.append( predictedFields ), true, ",", "\"", DATA_PATH + inputData, SinkMode.KEEP );
    Tap sink = getPlatform().getDelimitedFile( Fields.ALL, true, ",", "\"", getResultPath(), SinkMode.REPLACE );


    FlowDef flowDef = FlowDef.flowDef()
      .addSource( "head", source )
      .addSink( "tail", sink )
      .addTail( pipe );


    Flow flow = getPlatform().getFlowConnector().connect( flowDef );

View Full Code Here

    // NB: RHS is large given the sample data, but in practice the logs on the LHS could be much larger
    Pipe recoPipe = new Pipe( "reco", logsPipe );
    recoPipe = new CoGroup( recoPipe, new Fields( "gps_geohash" ), shadeCheck, new Fields( "tree_geohash" ), new InnerJoin() );


    // connect the taps, pipes, etc., into a flow
    FlowDef flowDef = FlowDef.flowDef()
     .setName( "copa" )
     .addSource( gisPipe, gisTap )
     .addTrap( gisPipe, trapTap )
     .addCheckpoint( tsvCheck, tsvTap )
     .addTailSink( parkPipe, parkTap )

View Full Code Here


   Properties properties = new Properties();
   AppProps.setApplicationJarClass( properties, Main.class );
   HadoopFlowConnector flowConnector = new HadoopFlowConnector( properties );


   FlowDef flowDef = createFlowDef(docPath, wcPath);


   // write a DOT file and run the flow
   Flow wcFlow = flowConnector.connect( flowDef );
   wcFlow.writeDOT( "dot/wc.dot" );
   wcFlow.complete();

View Full Code Here

    Pipe wcPipe = new Pipe( "wc", docPipe );
    wcPipe = new GroupBy( wcPipe, token );
    wcPipe = new Every( wcPipe, Fields.ALL, new Count(), Fields.ALL );


    // connect the taps, pipes, etc., into a flow
    FlowDef flowDef = FlowDef.flowDef()
     .setName( "wc" )
     .addSource( docPipe, docTap )
     .addTailSink( wcPipe, wcTap );


    return flowDef;

View Full Code Here

    pipe = new Each( pipe, new Identity() );


    Tap sink = getPlatform().getTabDelimitedFile( new Fields( "mangled" ), getOutputPath( "unusedpath" ), SinkMode.REPLACE );
    Tap checkpoint = getPlatform().getTabDelimitedFile( Fields.ALL, getOutputPath( path ), SinkMode.REPLACE );


    FlowDef flowDef = FlowDef.flowDef()
      .addSource( pipe, source )
      .addTailSink( pipe, sink )
      .addCheckpoint( "checkpoint", checkpoint );


    return getPlatform().getFlowConnector().connect( flowDef );

View Full Code Here


    Fields declared = lhsFields.append( Fields.mask( rhsFields, lhsFields ) );


    Pipe pipe = new CoGroup( lhsPipe, new Fields( "date" ), rhsPipe, new Fields( "date" ), declared );


    FlowDef flowDef = FlowDef.flowDef()
      .addSource( lhsPipe, inputLhs )
      .addSource( rhsPipe, inputRhs )
      .addTailSink( pipe, output );


    Flow flow = getPlatform().getFlowConnector().connect( flowDef );

View Full Code Here


    Tap sink = getPlatform().getTextFile( getOutputPath( "checkpoint/sink" ), SinkMode.REPLACE );


    Tap checkpoint = getPlatform().getDelimitedFile( Fields.ALL, true, getOutputPath( "checkpoint/tap" ), SinkMode.REPLACE );


    FlowDef flowDef = flowDef()
      .addSource( pipe, source )
      .addTailSink( pipe, sink )
      .addCheckpoint( "checkpoint", checkpoint );


    Flow flow = getPlatform().getFlowConnector().connect( flowDef );

View Full Code Here


    Tap sink = getPlatform().getTextFile( getOutputPath( "failcheckpoint/sink" ), SinkMode.REPLACE );


    Tap checkpoint = getPlatform().getDelimitedFile( Fields.ALL, true, getOutputPath( "failcheckpoint/tap" ), SinkMode.REPLACE );


    FlowDef flowDef = flowDef()
      .addSource( pipe, source )
      .addTailSink( pipe, sink )
      .addCheckpoint( "checkpointXXXXX", checkpoint );


    try

View Full Code Here


    Tap sink = getPlatform().getTextFile( getOutputPath( "failcheckpointevery/sink" ), SinkMode.REPLACE );


    Tap checkpoint = getPlatform().getDelimitedFile( Fields.ALL, true, getOutputPath( "failcheckpointevery/tap" ), SinkMode.REPLACE );


    FlowDef flowDef = flowDef()
      .addSource( pipe, source )
      .addTailSink( pipe, sink )
      .addCheckpoint( "checkpoint", checkpoint );


    try

View Full Code Here

0 1 2 3

TOP

Related Classes of cascading.flow.FlowDef

bixo.pipes.AbstractFetchPipeTest

cascading.cascade.CascadePlatformTest

cascading.JoinFieldedPipesPlatformTest

cascading.lingual.examples.foodmart.FlowExample

cascading.lingual.flow.SQLFlowPlatformTest

cascading.lingual.platform.LingualFlowFactory

cascading.pattern.ensemble.EnsemblePlatformTestCase

cascading.pattern.Main

cascading.pattern.pmml.iris.RegressionFlowExample

cascading.pattern.pmml.PMMLPlatformTestCase

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.