Examples of cascading.flow.FlowDef

cascading.flow.FlowDef
Class FlowDef is a fluent interface for defining a {@link Flow}.
This allows for ad-hoc building of Flow data and meta-data, like tags.
Instead of calling one of the {@link FlowConnector} connect methods, {@link FlowConnector#connect(FlowDef)}can be called.


    Tap sink = getPlatform().getTextFile( getOutputPath( "failcheckpointdeclared/sink" ), SinkMode.REPLACE );


    Tap checkpoint = getPlatform().getTextFile( getOutputPath( "failcheckpointdeclared/tap" ), SinkMode.REPLACE );


    FlowDef flowDef = flowDef()
      .addSource( pipe, source )
      .addTailSink( pipe, sink )
      .addCheckpoint( "checkpoint", checkpoint );


    try

View Full Code Here


    pipe = new Checkpoint( "checkpoint", pipe );


    Tap sink = getPlatform().getTextFile( getOutputPath( "duplicatecheckpoint" ), SinkMode.REPLACE );


    FlowDef flowDef = FlowDef.flowDef()
      .setName( "restartable" )
      .addSource( "test", source )
      .addTailSink( pipe, sink )
      .setRunID( "restartable" );

View Full Code Here


    pipe = new Each( pipe, new TestFunction( new Fields( "insert" ), new Tuple( "value" ), fail ? 2 : -1 ) );


    Tap sink = getPlatform().getTextFile( sinkPath, SinkMode.REPLACE );


    FlowDef flowDef = FlowDef.flowDef()
      .setName( "restartable" )
      .addSource( "test", source )
      .addTailSink( pipe, sink )
      .setRunID( "restartable" );

View Full Code Here

    Pipe counted = new Every( grouped, count );


    String testJoinMerge = "testJoinMergeGroupBy/" + ( ( joined instanceof CoGroup ) ? "cogroup" : "hashjoin" );
    Tap sink = getPlatform().getDelimitedFile( Fields.ALL, true, "\t", null, getOutputPath( testJoinMerge ), SinkMode.REPLACE );


    FlowDef flowDef = FlowDef.flowDef()
      .setName( "join-merge" )
      .addSource( rhs, rhsTap )
      .addSource( lhs, lhsTap )
      .addTailSink( counted, sink );

View Full Code Here

    Tap sink = getPlatform().getDelimitedFile( new Fields( "id", "name" ).applyTypes( int.class, String.class ), ",", getOutputPath( getTestName() ), SinkMode.REPLACE );
    Tap trap = getPlatform().getTextFile( getOutputPath( getTestName() + "_trap" ), SinkMode.REPLACE );


    Pipe pipe = new Pipe( "copy" );


    FlowDef flowDef = FlowDef.flowDef()
      .addSource( pipe, source )
      .addTailSink( pipe, sink )
      .addTrap( pipe, trap );


    Flow flow = getPlatform().getFlowConnector().connect( flowDef );

View Full Code Here

    TupleEntrySchemeIteratorProps.setPermittedExceptions( props, java.io.EOFException.class );


    Pipe pipe = new Pipe( "data" );
    pipe = new Each( pipe, new Identity() );


    FlowDef flowDef = FlowDef.flowDef().addSource( pipe, source ).addTailSink( pipe, sink );
    Flow flow = getPlatform().getFlowConnector( props ).connect( flowDef );
    flow.complete();
    validateLength( flow.openSink(), 307 );
    }

View Full Code Here

    Tap sink = getPlatform().getTextFile( getOutputPath( getTestName() ), SinkMode.REPLACE );


    Pipe pipe = new Pipe( "data" );
    pipe = new Each( pipe, new Identity() );


    FlowDef flowDef = FlowDef.flowDef().addSource( pipe, source ).addTailSink( pipe, sink );
    Flow flow = getPlatform().getFlowConnector().connect( flowDef );


    try
      {
      flow.complete();

View Full Code Here

    // make sure the input path exists
    inputTap.openForWrite(new HadoopFlowProcess()).close();
    // make sure the output path does not exist
    FileSystem.get(new Configuration()).delete(new Path(OUTPUT_PATH), true);


    FlowDef flowDef = new FlowDef()
        .addSource("input", inputTap)
        .addSink(p, outputTap)
        .addTail(p);


    new HadoopFlowConnector().connect(flowDef).complete();

View Full Code Here

  public Flow connect(String name, Map<String, Tap> sources, Map<String, Tap> sinks, Map<String, Tap> traps, Pipe... tails) {
    LoggingHadoopPlanner planner = new LoggingHadoopPlanner(flowStepStrategy, getProperties());
    planner.initialize(this);


    String flowName = name != null ? name : defaultFlowName;
    FlowDef definition = new FlowDef()
        .setName(flowName)
        .addTails(tails)
        .addSources(sources)
        .addSinks(sinks)
        .addTraps(traps);


    if(getProperties().containsKey(CascadingUtil.CASCADING_RUN_ID)){


      //  cascading checkpointing fails if the job is named creatively with special chars.  Be safe for now and only allow path-friendly chars
      if(!CHECKPOINT_SAFE_NAME.matcher(flowName).matches()){
        throw new RuntimeException("Flow name "+flowName+" not compatible with checkpointing! Remove special characters from name.");
      }


      definition.setRunID((String) getProperties().get(CascadingUtil.CASCADING_RUN_ID));
    }


    return planner.buildFlow(definition);
  }

View Full Code Here

    }


  @Override
  public Flow create()
    {
    FlowDef flowDef = FlowDef.flowDef()
      .setName( getName() )
      .addTails( tail )
      .setDebugLevel( platformBroker.getDebugLevel() );


    LOG.debug( "using log level: {}", platformBroker.getDebugLevel() );


    for( String jar : jars )
      {
      LOG.debug( "adding jar to classpath: {}", jar );
      flowDef.addToClassPath( jar );
      }


    flowDef.addDescription( FlowDescriptors.STATEMENTS, lingualConnection.getCurrentSQL() );


    Flow flow = createFlowFrom( flowDef, tail );


    flow.addListener( new LingualConnectionFlowListener( lingualConnection ) );

View Full Code Here

0 1 2 3

TOP

Related Classes of cascading.flow.FlowDef

bixo.pipes.AbstractFetchPipeTest

cascading.cascade.CascadePlatformTest

cascading.JoinFieldedPipesPlatformTest

cascading.lingual.examples.foodmart.FlowExample

cascading.lingual.flow.SQLFlowPlatformTest

cascading.lingual.platform.LingualFlowFactory

cascading.pattern.ensemble.EnsemblePlatformTestCase

cascading.pattern.Main

cascading.pattern.pmml.iris.RegressionFlowExample

cascading.pattern.pmml.PMMLPlatformTestCase

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.