Examples of cascading.flow.planner.rule.RuleRegistry

cascading.flow.planner.rule.RuleRegistry
RuleRegistry contains all planner rules for a given platform. Where a rule is of type {@link cascading.flow.planner.rule.Rule} and allows for either asserts, transforms, and partitioningof pipe assembly process graphs.
Process graphs are initially a standard Pipe assembly and connected Taps. And depending on the underlying platform are partitioned into units of execution along the process hierarchy of {@link cascading.flow.planner.rule.ProcessLevel#Assembly}, {@link cascading.flow.planner.rule.ProcessLevel#Step}, {@link cascading.flow.planner.rule.ProcessLevel#Node}, and {@link cascading.flow.planner.rule.ProcessLevel#Pipeline}.
Where the Assembly is the user created pipe assembly and taps. The Steps are physical jobs executed by a platform. Nodes are internal elements of work within a job (a Mapper or Reducer in the case of MapReduce). And Pipelines, which are non-blocking streamed paths within a node. These can be optional.
Rules rely on a 'language' for sub-graph pattern matching and can be applied against a given process type (Assembly, or Step, etc), to test its correctness, or make changes to the graph. Or they can be used to partition a large graph into a smaller graph, converting an Assembly into Steps.
Debugging rule sets can be done by enabling system properties, see {@link cascading.flow.planner.FlowPlanner}.
The {@link cascading.flow.planner.rule.RuleExec} class is responsible for executing on a given rule set.

    {
    FlowPlanner flowPlanner = createFlowPlanner();


    flowPlanner.initialize( this, properties );


    RuleRegistry ruleRegistry = getRuleRegistry();


    return flowPlanner.buildFlow( flowDef, ruleRegistry );
    }

View Full Code Here

    }


  @Test
  public void testSubGraphIterator()
    {
    RuleRegistry ruleRegistry = new RuleRegistry();


    ruleRegistry.addElementFactory( NonTapFactory.TEMP_TAP, new NonTapFactory() );


    PlannerContext plannerContext = new PlannerContext( ruleRegistry, null, null, null, true );


    ruleRegistry.addRule( new RuleInsertionTransformer( PlanPhase.PreResolveAssembly, new TestCheckpointExpression(), IntermediateTapElementFactory.TEMP_TAP ) );
//    ruleRegistry.addRule( new RuleContractedTransform( PlanPhase.PreResolve, new NoOpPipeExpression() ) );


    RuleResult ruleResult = new RuleExec( ruleRegistry ).executeRulePhase( PlanPhase.PreResolveAssembly, plannerContext, new RuleResult( new StandardElementGraph() ) );
    FlowElementGraph flowElementGraph = ruleResult.getAssemblyGraph();

View Full Code Here

    }


  @Test
  public void testSubGraphIterator2()
    {
    RuleRegistry ruleRegistry = new RuleRegistry();


    PlannerContext plannerContext = new PlannerContext( ruleRegistry );


    ruleRegistry.addRule( new RemoveNoOpPipeTransformer() );


    FlowElementGraph elementGraph = new HashJoinMergeIntoHashJoinStreamedStreamedMergeGraph();
//    FlowElementGraph elementGraph = new HashJoinAroundHashJoinLeftMostGraph();
    RuleResult ruleResult = new RuleExec( ruleRegistry ).executeRulePhase( PlanPhase.PreResolveAssembly, plannerContext, new RuleResult( elementGraph ) );
    FlowElementGraph flowElementGraph = ruleResult.getAssemblyGraph();

View Full Code Here

   * see https://issues.apache.org/jira/browse/TEZ-1190
   */
  @Test
  public void testSubGraphIteratorSelfCoGroup()
    {
    RuleRegistry ruleRegistry = new RuleRegistry();


    ruleRegistry.setResolveElementsEnabled( false );


    ruleRegistry.addRule( new RemoveNoOpPipeTransformer() );


    ruleRegistry.addRule( new WholeGraphStepPartitioner() );


    ruleRegistry.addRule(
      new ExpressionRulePartitioner( PlanPhase.PartitionNodes,
        new RuleExpression(
          new TestNoGroupTapExpressionGraph(),
          new MultiTapGroupExpressionGraph()
        )
      )
    );


    ruleRegistry.addRule(
      new UniquePathRulePartitioner( PlanPhase.PartitionNodes,
        RulePartitioner.PartitionSource.PartitionCurrent,
        new RuleExpression(
          new TestNoGroupTapExpressionGraph(),
          new TestBoundarySelJoinCoGroupExpressionGraph()

View Full Code Here

    runSubGraphIteratorRotate( new HashJoinMergeIntoHashJoinStreamedStreamedMergeGraph(), 2 );
    }


  private void runSubGraphIteratorRotate( FlowElementGraph elementGraph, int numSubGraphs )
    {
    RuleRegistry ruleRegistry = new RuleRegistry();


    PlannerContext plannerContext = new PlannerContext( ruleRegistry );


    ruleRegistry.addRule( new RemoveNoOpPipeTransformer() );


    RuleResult ruleResult = new RuleExec( ruleRegistry ).executeRulePhase( PlanPhase.PreResolveAssembly, plannerContext, new RuleResult( elementGraph ) );
    FlowElementGraph flowElementGraph = ruleResult.getAssemblyGraph();


    flowElementGraph.writeDOT( getPlanPath() + "/node.dot" );

View Full Code Here

    }


  @Test
  public void testRuleEngine()
    {
    RuleRegistry ruleRegistry = new RuleRegistry();


    ruleRegistry.addElementFactory( NonTapFactory.TEMP_TAP, new NonTapFactory() );


    PlannerContext plannerContext = new PlannerContext( ruleRegistry, null, null, null, true );


    ruleRegistry.addRule( new RuleAssert( PlanPhase.PreResolveAssembly, new LoneGroupExpression(), "lone group assertion" ) );
    ruleRegistry.addRule( new RuleInsertionTransformer( PlanPhase.PreResolveAssembly, new TestGroupGroupExpression(), IntermediateTapElementFactory.TEMP_TAP ) );


    try
      {
      new RuleExec( ruleRegistry ).executeRulePhase( PlanPhase.PreResolveAssembly, plannerContext, new RuleResult( new LoneGroupAssertionGraph() ) );
      fail();

View Full Code Here

    transform( new JoinAroundJoinRightMostGraphSwapped(), new BufferAfterEveryExpression() );
    }


  private Asserted assertion( FlowElementGraph flowElementGraph, RuleExpression ruleExpression )
    {
    RuleRegistry ruleRegistry = new RuleRegistry();


    ruleRegistry.addElementFactory( NonTapFactory.TEMP_TAP, new NonTapFactory() );


    PlannerContext plannerContext = new PlannerContext( ruleRegistry, null, null, null, true );


    Asserted asserted = new RuleAssert( PlanPhase.PreResolveAssembly, ruleExpression, "message" ).assertion( plannerContext, flowElementGraph );

View Full Code Here

    return asserted;
    }


  private ElementGraph transform( ElementGraph flowElementGraph, RuleExpression ruleExpression )
    {
    RuleRegistry ruleRegistry = new RuleRegistry();


    ruleRegistry.addElementFactory( NonTapFactory.TEMP_TAP, new NonTapFactory() );


    PlannerContext plannerContext = new PlannerContext( ruleRegistry, null, null, null, true );


    RuleInsertionTransformer ruleTempTapInsertionTransformer = new RuleInsertionTransformer( PlanPhase.PreResolveAssembly, ruleExpression, IntermediateTapElementFactory.TEMP_TAP );
    Transformed<ElementGraph> insertionTransformed = ruleTempTapInsertionTransformer.transform( plannerContext, flowElementGraph );

View Full Code Here

TOP

Related Classes of cascading.flow.planner.rule.RuleRegistry

cascading.flow.FlowConnector

cascading.flow.iso.IsomorphismTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.