Examples of cascading.flow.planner.BaseFlowStep

cascading.flow.planner.BaseFlowStep
Class FlowStep is an internal representation of a given Job to be executed on a remote cluster. During planning, pipe assemblies are broken down into "steps" and encapsulated in this class.
FlowSteps are submitted in order of dependency. If two or more steps do not share the same dependencies and all can be scheduled simultaneously, the {@link #getSubmitPriority()} value determines the order in whichall steps will be submitted for execution. The default submit priority is 5.
This class is for internal use, there are no stable public methods.

  public void convert() {
    //  Returns a set of the nodes contained in this graph.
    Set vetices = jobsGraph.vertexSet();


    for (Object flowStep : vetices) {
      BaseFlowStep baseFlowStep = (BaseFlowStep) flowStep;
      String nodeName = baseFlowStep.getName();
      String[] features = getNodeFeatures(baseFlowStep.getGraph());
      // create a new DAGNode of this flowStep
      CascadingJob job = new CascadingJob();
      job.setFeatures(features);
      DAGNode newNode = new DAGNode(nodeName, job);
      // Add the new node to the Map of <nodeName, DAGNodes>

View Full Code Here

    {
    }


  public void testListeners()
    {
    BaseFlowStep step = new BaseFlowStep( "step1", 0 )
    {


    @Override
    public Object createInitializedConfig( FlowProcess fp, Object config )
      {
      return null;
      }


    @Override
    public void clean( Object config )
      {
      }


    @Override
    protected FlowStepJob createFlowStepJob( ClientState clientState, FlowProcess fp, Object initializedStepConfig )
      {
      return null;
      }


    @Override
    public Set getTraps()
      {
      return null;
      }


    @Override
    public Tap getTrap( String string )
      {
      return null;
      }
    };


    FlowStepListener listener = new FlowStepListener()
    {
    public void onStepStarting( FlowStep flowStep )
      {
      }


    public void onStepStopping( FlowStep flowStep )
      {
      }


    public void onStepCompleted( FlowStep flowStep )
      {
      }


    public boolean onStepThrowable( FlowStep flowStep, Throwable throwable )
      {
      return false;
      }


    public void onStepRunning( FlowStep flowStep )
      {
      }
    };
    step.addListener( listener );


    assertTrue( "no listener found", step.hasListeners() );


    step.removeListener( listener );


    assertFalse( "listener found", step.hasListeners() );
    }

View Full Code Here


    List steps = getPlatform().getFlowConnector().connect( sources, sinks, pipe ).getFlowSteps();


    assertEquals( "wrong size", 1, steps.size() );


    BaseFlowStep step = (BaseFlowStep) steps.get( 0 );


    assertEquals( "not equal: step.sources.size()", 1, step.getSourceTaps().size() );
    assertNotNull( "null: step.groupBy", step.getGroup() );
    assertNotNull( "null: step.sink", step.getSink() );


    int mapDist = countDistance( step.getElementGraph(), (FlowElement) step.getSourceTaps().iterator().next(), step.getGroup() );
    assertEquals( "not equal: mapDist", 0, mapDist );


    int reduceDist = countDistance( step.getElementGraph(), step.getGroup(), step.getSink() );
    assertEquals( "not equal: reduceDist", 1, reduceDist );
    }

View Full Code Here


    List steps = getPlatform().getFlowConnector().connect( sources, sinks, pipe ).getFlowSteps();


    assertEquals( "wrong size", 1, steps.size() );


    BaseFlowStep step = (BaseFlowStep) steps.get( 0 );


    assertEquals( "not equal: step.sources.size()", 1, step.getSourceTaps().size() );
    assertNotNull( "null: step.groupBy", step.getGroup() );
    assertNotNull( "null: step.sink", step.getSink() );


    int mapDist = countDistance( step.getElementGraph(), (FlowElement) step.getSourceTaps().iterator().next(), step.getGroup() );
    assertEquals( "not equal: mapDist", 2, mapDist );


    int reduceDist = countDistance( step.getElementGraph(), step.getGroup(), step.getSink() );
    assertEquals( "not equal: reduceDist", 1, reduceDist );
    }

View Full Code Here


    List steps = getPlatform().getFlowConnector().connect( sources, sinks, splice ).getFlowSteps();


    assertEquals( "wrong size", 1, steps.size() );


    BaseFlowStep step = (BaseFlowStep) steps.get( 0 );


    assertEquals( "not equal: step.sources.size()", 2, step.getSourceTaps().size() );
    assertNotNull( "null: step.groupBy", step.getGroup() );
    assertNotNull( "null: step.sink", step.getSink() );


    Iterator<Tap> iterator = step.getSourceTaps().iterator();
    int mapDist = countDistance( step.getElementGraph(), iterator.next(), step.getGroup() );
    assertEquals( "not equal: mapDist", 0, mapDist );
    mapDist = countDistance( step.getElementGraph(), iterator.next(), step.getGroup() );
    assertEquals( "not equal: mapDist", 0, mapDist );


    int reduceDist = countDistance( step.getElementGraph(), step.getGroup(), step.getSink() );
    assertEquals( "not equal: reduceDist", 0, reduceDist );
    }

View Full Code Here


    List steps = getPlatform().getFlowConnector().connect( sources, sinks, cogroup ).getFlowSteps();


    assertEquals( "wrong size", 1, steps.size() );


    BaseFlowStep step = (BaseFlowStep) steps.get( 0 );


    assertEquals( "not equal: step.sources.size()", 2, step.getSourceTaps().size() );
    assertNotNull( "null: step.groupBy", step.getGroup() );
    assertNotNull( "null: step.sink", step.getSink() );


    int mapDist = countDistance( step.getElementGraph(), (FlowElement) step.getSourceTaps().iterator().next(), step.getGroup() );
    assertEquals( "not equal: mapDist", 0, mapDist );


    int reduceDist = countDistance( step.getElementGraph(), step.getGroup(), step.getSink() );
    assertEquals( "not equal: reduceDist", 1, reduceDist );
    }

View Full Code Here


    List steps = getPlatform().getFlowConnector().connect( sources, sinks, splice ).getFlowSteps();


    assertEquals( "wrong size", 1, steps.size() );


    BaseFlowStep step = (BaseFlowStep) steps.get( 0 );


    assertEquals( "not equal: step.sources.size()", 2, step.getSourceTaps().size() );
    assertNotNull( "null: step.groupBy", step.getGroup() );
    assertNotNull( "null: step.sink", step.getSink() );


    int mapDist = countDistance( step.getElementGraph(), (FlowElement) step.getSourceTaps().iterator().next(), step.getGroup() );
    assertEquals( "not equal: mapDist", 0, mapDist );


    int reduceDist = countDistance( step.getElementGraph(), step.getGroup(), step.getSink() );
    assertEquals( "not equal: reduceDist", 1, reduceDist );
    }

View Full Code Here


    List<FlowStep> steps = flow.getFlowSteps();


    assertEquals( "not equal: steps.size()", 3, steps.size() );


    BaseFlowStep step = (BaseFlowStep) steps.get( 0 );


    Scope nextScope = step.getNextScope( step.getGroup() );
    FlowElement operator = step.getNextFlowElement( nextScope );


    assertTrue( "not an Every", operator instanceof Every );


    nextScope = step.getNextScope( operator );
    operator = step.getNextFlowElement( nextScope );


    assertTrue( "not a Each", operator instanceof Each );


    nextScope = step.getNextScope( operator );
    operator = step.getNextFlowElement( nextScope );


    assertTrue( "not a TempHfs", operator instanceof TempHfs );
    }

View Full Code Here


    List<FlowStep> steps = flow.getFlowSteps();


    assertEquals( "not equal: steps.size()", 3, steps.size() );


    BaseFlowStep step = (BaseFlowStep) steps.get( 0 );


    Scope nextScope = step.getNextScope( step.getGroup() );
    FlowElement operator = step.getNextFlowElement( nextScope );


    assertTrue( "not an Every", operator instanceof Every );


    nextScope = step.getNextScope( operator );
    operator = step.getNextFlowElement( nextScope );


    assertTrue( "not a Each", operator instanceof Each );


    nextScope = step.getNextScope( operator );
    operator = step.getNextFlowElement( nextScope );


    assertTrue( "not a TempHfs", operator instanceof TempHfs );
    }

View Full Code Here

    assertEquals( getDebugCount( getPlatform().getFlowConnector( properties ).connect( source, sink, pipe ) ), 0 );
    }


  private int getDebugCount( Flow flow )
    {
    BaseFlowStep step = (BaseFlowStep) flow.getFlowSteps().get( 0 );


    Collection<Operation> operations = step.getAllOperations();
    int count = 0;


    for( Operation operation : operations )
      {
      if( operation instanceof Debug )

View Full Code Here

0 1

TOP

Related Classes of cascading.flow.planner.BaseFlowStep

cascading.flow.FlowElement

cascading.flow.FlowStepListenersTest

cascading.flow.hadoop.BuildJobsHadoopPlatformTest

cascading.flow.planner.process.FlowStepGraph

cascading.management.CascadingServices

cascading.management.state.ClientState

cascading.operation.debug.BuildDebugPlatformTest

com.twitter.ambrose.cascading.AmbroseCascadingGraphConverter

cascading.flow.FlowException

java.util.Date

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.