Examples of cascading.flow.FlowElement

cascading.flow.FlowElement
Interface FlowElement is a utility interface used internally to simplify DAG management. It is not intended for users to interact with these methods directly.

    return dag;
    }


  private EdgeProperty createEdgeProperty( TezConfiguration config, ProcessGraph.ProcessEdge processEdge )
    {
    FlowElement flowElement = processEdge.getFlowElement();


    EdgeValues edgeValues = new EdgeValues( new TezConfiguration( config ), processEdge );


    edgeValues.keyClassName = Tuple.class.getName(); // TEZ_RUNTIME_INTERMEDIATE_OUTPUT_KEY_CLASS
    edgeValues.valueClassName = Tuple.class.getName(); // TEZ_RUNTIME_INTERMEDIATE_OUTPUT_VALUE_CLASS
    edgeValues.keyComparatorClassName = TupleComparator.class.getName();
    edgeValues.keyPartitionerClassName = TuplePartitioner.class.getName();
    edgeValues.outputClassName = null;
    edgeValues.inputClassName = null;
    edgeValues.movementType = null;
    edgeValues.sourceType = null;
    edgeValues.schedulingType = null;


    if( flowElement instanceof Group )
      applyGroup( edgeValues );
    else if( ( flowElement instanceof Boundary || flowElement instanceof Merge ) && processEdge.getSourceAnnotations().contains( StreamMode.Accumulated ) )
      applyBoundaryMergeAccumulated( edgeValues );
    else if( flowElement instanceof Boundary || flowElement instanceof Merge )
      applyBoundaryMerge( edgeValues );
    else
      throw new IllegalStateException( "unsupported flow element: " + flowElement.getClass().getCanonicalName() );


    return createEdgeProperty( edgeValues );
    }

View Full Code Here

public class EquivalentTapsScopeExpression extends ScopeExpression
  {
  @Override
  public boolean applies( PlannerContext plannerContext, ElementGraph elementGraph, Scope scope )
    {
    FlowElement edgeSource = elementGraph.getEdgeSource( scope );
    FlowElement edgeTarget = elementGraph.getEdgeTarget( scope );


    if( !( edgeSource instanceof Hfs ) || !( edgeTarget instanceof Hfs ) )
      throw new IllegalStateException( "non Hfs Taps matched" );


    Hfs predecessor = (Hfs) edgeSource;

View Full Code Here

    // returns a Set ordered topologically by the matched graph. retains this first, this second ordering for simple cases
    TopologicalOrderIterator<FlowElement, Scope> iterator = ElementGraphs.getTopologicalIterator( getMatchedGraph() );


    while( iterator.hasNext() )
      {
      FlowElement next = iterator.next();
      ElementExpression elementExpression = reversed.get( next );


      // matchedGraph may be a super-set of what's in the mapping, so elementExpression may be null
      if( elementExpression == null )
        continue;

View Full Code Here

    // no evidence elementGraph.vertexSet().iterator(); is faster without modifying jgrapht
    Iterator<FlowElement> iterator = SearchOrder.getNodeIterator( matchExpression.getSearchOrder(), elementGraph );


    while( iterator.hasNext() )
      {
      FlowElement flowElement = iterator.next();


      if( exclusions.contains( flowElement ) )
        continue;


      if( expression.applies( plannerContext, elementGraph, flowElement ) )

View Full Code Here

    for( ScopeExpression scopeExpression : delegate.edgeSet() )
      {
      ElementExpression lhs = delegate.getEdgeSource( scopeExpression );
      ElementExpression rhs = delegate.getEdgeTarget( scopeExpression );


      FlowElement lhsElement = vertexMapping.get( lhs );
      FlowElement rhsElement = vertexMapping.get( rhs );


      Set<Scope> edges = elementGraph.getAllEdges( lhsElement, rhsElement );


      if( edges != null )
        edgeMapping.put( scopeExpression, edges );

View Full Code Here


    if( LOG.isDebugEnabled() && results != null )
      {
      for( Scope result : results )
        {
        FlowElement lhs = elementGraph.getDelegate().getEdgeSource( result );
        FlowElement rhs = elementGraph.getDelegate().getEdgeTarget( result );
        LOG.debug( "compatible edge: {} - {}", lhs, rhs );
        }
      }


    return results != null;

View Full Code Here

    }


  private boolean areCompatibleNodes( int node1, int node2 )
    {
    Expression expression = matchGraph.getVertex( node1 );
    FlowElement flowElement = elementGraph.getVertex( node2 );


    boolean result;


    if( ( (ElementExpression) expression ).getCapture() == ElementCapture.Primary &&
      !finderContext.getRequiredElements().isEmpty() )

View Full Code Here

      {
      TopologicalOrderIterator<FlowElement, Scope> iterator = ElementGraphs.getTopologicalIterator( elementGraph );


      while( iterator.hasNext() )
        {
        FlowElement element = iterator.next();


        while( element != null )
          {
          if( element.hasStepConfigDef() )
            element.getStepConfigDef().apply( mode, setter );


          if( element instanceof Pipe )
            element = ( (Pipe) element ).getParent();
          else
            element = null;

View Full Code Here

    Set<String> checkpointNames = new HashSet<String>();


    // need to verify that only Extent instances are origins in this graph. Otherwise a Tap was not properly connected
    TopologicalOrderIterator<FlowElement, Scope> iterator = getTopologicalIterator();


    FlowElement flowElement = null;


    while( iterator.hasNext() )
      {
      try
        {

View Full Code Here

    {
    DepthFirstIterator<FlowElement, Scope> iterator = getDepthFirstIterator();


    while( iterator.hasNext() )
      {
      FlowElement element = iterator.next();


      if( !( element instanceof Pipe ) )
        continue;


      Pipe pipe = (Pipe) element;

View Full Code Here

0 1 2 3

TOP

Related Classes of cascading.flow.FlowElement

cascading.flow.hadoop.BuildJobsHadoopPlatformTest

cascading.flow.hadoop.planner.rule.scopeexpression.EquivalentTapsScopeExpression

cascading.flow.planner.BaseFlowStep

cascading.flow.planner.ElementGraph

cascading.flow.planner.graph.ElementGraphs

cascading.flow.planner.graph.ElementMultiGraph

cascading.flow.planner.graph.FlowElementGraph

cascading.flow.planner.iso.finder.GraphFinder

cascading.flow.planner.iso.finder.Match

cascading.flow.planner.iso.finder.State

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.