Package org.apache.pig.impl.plan.optimizer

Examples of org.apache.pig.impl.plan.optimizer.RulePlan


        mRulesOff = turnOffRules;
        runOptimizations(plan, mode);
    }

    private void runOptimizations(LogicalPlan plan, ExecType mode) {
        RulePlan rulePlan;

        // List of rules for the logical optimizer
       
        boolean turnAllRulesOff = false;
        if (mRulesOff != null) {
            for (String rule : mRulesOff) {
                if ("all".equalsIgnoreCase(rule)) {
                    turnAllRulesOff = true;
                    break;
                }
            }
        }
       
        // This one has to be before the type cast inserter as it expects the
        // load to only have one output.
        // Find any places in the plan that have an implicit split and make
        // it explicit. Since the RuleMatcher doesn't handle trees properly,
        // we cheat and say that we match any node. Then we'll do the actual
        // test in the transformers check method.
       
        rulePlan = new RulePlan();
        RuleOperator anyLogicalOperator = new RuleOperator(LogicalOperator.class, RuleOperator.NodeType.ANY_NODE,
                new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
        rulePlan.add(anyLogicalOperator);
        mRules.add(new Rule<LogicalOperator, LogicalPlan>(rulePlan,
                new ImplicitSplitInserter(plan), "ImplicitSplitInserter"));

       
        // this one is ordered to be before other optimizations since  later
        // optimizations may move the LOFilter that is looks for just after a
        // LOLoad
        rulePlan = new RulePlan();
        RuleOperator loLoad = new RuleOperator(LOLoad.class,
                new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
        rulePlan.add(loLoad);
        mRules.add(new Rule<LogicalOperator, LogicalPlan>(rulePlan,
                new PartitionFilterOptimizer(plan), "LoadPartitionFilterOptimizer"));
       
        // Add type casting to plans where the schema has been declared (by
        // user, data, or data catalog).
        rulePlan = new RulePlan();
        loLoad = new RuleOperator(LOLoad.class,
                new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
        rulePlan.add(loLoad);
        mRules.add(new Rule<LogicalOperator, LogicalPlan>(rulePlan,
                new TypeCastInserter(plan, LOLoad.class.getName()), "LoadTypeCastInserter"));

        // Add type casting to plans where the schema has been declared by
        // user in a statement with stream operator.
       
        rulePlan = new RulePlan();
        RuleOperator loStream= new RuleOperator(LOStream.class,
                new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
        rulePlan.add(loStream);
        mRules.add(new Rule<LogicalOperator, LogicalPlan>(rulePlan, new TypeCastInserter(plan,
                LOStream.class.getName()), "StreamTypeCastInserter"));
       
        if(!turnAllRulesOff) {

            // Push up limit wherever possible.
            rulePlan = new RulePlan();
            RuleOperator loLimit = new RuleOperator(LOLimit.class,
          new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
      rulePlan.add(loLimit);
      Rule<LogicalOperator, LogicalPlan> rule = new Rule<LogicalOperator, LogicalPlan>(rulePlan,
          new OpLimitOptimizer(plan, mode), "LimitOptimizer");
            checkAndAddRule(rule);
           
            // Push filters up wherever possible
            rulePlan = new RulePlan();
            RuleOperator loFilter = new RuleOperator(LOFilter.class,
                    new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
            rulePlan.add(loFilter);
            rule = new Rule<LogicalOperator, LogicalPlan>(rulePlan,
                    new PushUpFilter(plan), "PushUpFilter");
            checkAndAddRule(rule);
           
            // Push foreach with flatten down wherever possible
            rulePlan = new RulePlan();
            RuleOperator loForeach = new RuleOperator(LOForEach.class,
                    new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
            rulePlan.add(loForeach);
            rule = new Rule<LogicalOperator, LogicalPlan>(rulePlan,
                    new PushDownForeachFlatten(plan), "PushDownForeachFlatten");
            checkAndAddRule(rule);
           
            // Prune column up wherever possible
            rulePlan = new RulePlan();
            RuleOperator rulePruneColumnsOperator = new RuleOperator(RelationalOperator.class, RuleOperator.NodeType.ANY_NODE,
                    new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
            rulePlan.add(rulePruneColumnsOperator);
            pruneRule = new Rule<LogicalOperator, LogicalPlan>(rulePlan,
                    new PruneColumns(plan), "PruneColumns", Rule.WalkerAlgo.ReverseDependencyOrderWalker);
        }
       
    }
View Full Code Here


    private static NodeIdGenerator nodeIdGen = NodeIdGenerator.getGenerator();
   
    public FunctionalLogicalOptimizer(LogicalPlan plan) {
        super(plan);

        RulePlan rulePlan;       

        // List of rules for the logical optimizer

        // This one has to be first, as the type cast inserter expects the
        // load to only have one output.
        // Find any places in the plan that have an implicit split and make
        // it explicit. Since the RuleMatcher doesn't handle trees properly,
        // we cheat and say that we match any node. Then we'll do the actual
        // test in the transformers check method.
       
        rulePlan = new RulePlan();
        RuleOperator anyLogicalOperator = new RuleOperator(LogicalOperator.class, RuleOperator.NodeType.ANY_NODE,
                new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
        rulePlan.add(anyLogicalOperator);
        mRules.add(new Rule<LogicalOperator, LogicalPlan>(rulePlan,
                new ImplicitSplitInserter(plan), "ImplicitSplitInserter"));


        // Add type casting to plans where the schema has been declared (by
        // user, data, or data catalog).
        rulePlan = new RulePlan();
        RuleOperator loLoad = new RuleOperator(LOLoad.class,
                new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
        rulePlan.add(loLoad);
        mRules.add(new Rule<LogicalOperator, LogicalPlan>(rulePlan,
                new TypeCastInserter(plan, LOLoad.class.getName()), "LoadTypeCastInserter"));

        // Add type casting to plans where the schema has been declared by
        // user in a statement with stream operator.
        rulePlan = new RulePlan();
        RuleOperator loStream= new RuleOperator(LOStream.class,
                new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
        rulePlan.add(loStream);
        mRules.add(new Rule<LogicalOperator, LogicalPlan>(rulePlan, new TypeCastInserter(plan,
                LOStream.class.getName()), "StreamTypeCastInserter"));
       
        // Add type casting to plans where the schema has been declared by
        // user in a statement with native operator.
        rulePlan = new RulePlan();
        RuleOperator loNative= new RuleOperator(LONative.class,
                new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
        rulePlan.add(loNative);
        mRules.add(new Rule<LogicalOperator, LogicalPlan>(rulePlan, new TypeCastInserter(plan,
                LONative.class.getName()), "NativeTypeCastInserter"));
       
        // Push up filters as far as we can
        rulePlan = new RulePlan();
        RuleOperator loFilter = new RuleOperator(LOFilter.class,
                new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
        rulePlan.add(loFilter);
        mRules.add(new Rule<LogicalOperator, LogicalPlan>(rulePlan,
                new TypeCastInserter(plan, LOFilter.class.getName()),
                "PushUpFilter"));

    }
View Full Code Here

        mRulesOff = turnOffRules;
        runOptimizations(plan, mode);
    }

    private void runOptimizations(LogicalPlan plan, ExecType mode) {
        RulePlan rulePlan;

        // List of rules for the logical optimizer
       
        boolean turnAllRulesOff = false;
        if (mRulesOff != null) {
            for (String rule : mRulesOff) {
                if ("all".equalsIgnoreCase(rule)) {
                    turnAllRulesOff = true;
                    break;
                }
            }
        }
       
        // This one has to be before the type cast inserter as it expects the
        // load to only have one output.
        // Find any places in the plan that have an implicit split and make
        // it explicit. Since the RuleMatcher doesn't handle trees properly,
        // we cheat and say that we match any node. Then we'll do the actual
        // test in the transformers check method.
       
        rulePlan = new RulePlan();
        RuleOperator anyLogicalOperator = new RuleOperator(LogicalOperator.class, RuleOperator.NodeType.ANY_NODE,
                new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
        rulePlan.add(anyLogicalOperator);
        mRules.add(new Rule<LogicalOperator, LogicalPlan>(rulePlan,
                new ImplicitSplitInserter(plan), "ImplicitSplitInserter"));

       
        // this one is ordered to be before other optimizations since  later
        // optimizations may move the LOFilter that is looks for just after a
        // LOLoad
        rulePlan = new RulePlan();
        RuleOperator loLoad = new RuleOperator(LOLoad.class,
                new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
        rulePlan.add(loLoad);
        mRules.add(new Rule<LogicalOperator, LogicalPlan>(rulePlan,
                new PartitionFilterOptimizer(plan), "PartitionFilterOptimizer"));


        // Add type casting to plans where the schema has been declared (by
        // user, data, or data catalog).
        rulePlan = new RulePlan();
        loLoad = new RuleOperator(LOLoad.class,
                new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
        rulePlan.add(loLoad);
        mRules.add(new Rule<LogicalOperator, LogicalPlan>(rulePlan,
                new TypeCastInserter(plan, LOLoad.class.getName()), "LoadTypeCastInserter"));

        // Add type casting to plans where the schema has been declared by
        // user in a statement with stream operator.
       
        rulePlan = new RulePlan();
        RuleOperator loStream= new RuleOperator(LOStream.class,
                new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
        rulePlan.add(loStream);
        mRules.add(new Rule<LogicalOperator, LogicalPlan>(rulePlan, new TypeCastInserter(plan,
                LOStream.class.getName()), "StreamTypeCastInserter"));

        if(!turnAllRulesOff) {

            // Push up limit wherever possible.
            rulePlan = new RulePlan();
            RuleOperator loLimit = new RuleOperator(LOLimit.class,
          new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
      rulePlan.add(loLimit);
      Rule<LogicalOperator, LogicalPlan> rule = new Rule<LogicalOperator, LogicalPlan>(rulePlan,
          new OpLimitOptimizer(plan, mode), "LimitOptimizer");
            checkAndAddRule(rule);
           
            // Push filters up wherever possible
            rulePlan = new RulePlan();
            RuleOperator loFilter = new RuleOperator(LOFilter.class,
                    new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
            rulePlan.add(loFilter);
            rule = new Rule<LogicalOperator, LogicalPlan>(rulePlan,
                    new PushUpFilter(plan), "PushUpFilter");
            checkAndAddRule(rule);
           
            // Push foreach with flatten down wherever possible
            rulePlan = new RulePlan();
            RuleOperator loForeach = new RuleOperator(LOForEach.class,
                    new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
            rulePlan.add(loForeach);
            rule = new Rule<LogicalOperator, LogicalPlan>(rulePlan,
                    new PushDownForeachFlatten(plan), "PushDownForeachFlatten");
            checkAndAddRule(rule);
           
            // Prune column up wherever possible
            rulePlan = new RulePlan();
            RuleOperator rulePruneColumnsOperator = new RuleOperator(RelationalOperator.class, RuleOperator.NodeType.ANY_NODE,
                    new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
            rulePlan.add(rulePruneColumnsOperator);
            pruneRule = new Rule<LogicalOperator, LogicalPlan>(rulePlan,
                    new PruneColumns(plan), "PruneColumns", Rule.WalkerAlgo.ReverseDependencyOrderWalker);
        }
       
    }
View Full Code Here

    private static NodeIdGenerator nodeIdGen = NodeIdGenerator.getGenerator();
   
    public FunctionalLogicalOptimizer(LogicalPlan plan) {
        super(plan);

        RulePlan rulePlan;       

        // List of rules for the logical optimizer

        // This one has to be first, as the type cast inserter expects the
        // load to only have one output.
        // Find any places in the plan that have an implicit split and make
        // it explicit. Since the RuleMatcher doesn't handle trees properly,
        // we cheat and say that we match any node. Then we'll do the actual
        // test in the transformers check method.
       
        rulePlan = new RulePlan();
        RuleOperator anyLogicalOperator = new RuleOperator(LogicalOperator.class, RuleOperator.NodeType.ANY_NODE,
                new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
        rulePlan.add(anyLogicalOperator);
        mRules.add(new Rule<LogicalOperator, LogicalPlan>(rulePlan,
                new ImplicitSplitInserter(plan), "ImplicitSplitInserter"));


        // Add type casting to plans where the schema has been declared (by
        // user, data, or data catalog).
        rulePlan = new RulePlan();
        RuleOperator loLoad = new RuleOperator(LOLoad.class,
                new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
        rulePlan.add(loLoad);
        mRules.add(new Rule<LogicalOperator, LogicalPlan>(rulePlan,
                new TypeCastInserter(plan, LOLoad.class.getName()), "LoadTypeCastInserter"));

        // Add type casting to plans where the schema has been declared by
        // user in a statement with stream operator.
        rulePlan = new RulePlan();
        RuleOperator loStream= new RuleOperator(LOStream.class,
                new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
        rulePlan.add(loStream);
        mRules.add(new Rule<LogicalOperator, LogicalPlan>(rulePlan, new TypeCastInserter(plan,
                LOStream.class.getName()), "StreamTypeCastInserter"));
       
        // Push up filters as far as we can
        rulePlan = new RulePlan();
        RuleOperator loFilter = new RuleOperator(LOFilter.class,
                new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
        rulePlan.add(loFilter);
        mRules.add(new Rule<LogicalOperator, LogicalPlan>(rulePlan,
                new TypeCastInserter(plan, LOFilter.class.getName()),
                "PushUpFilter"));

    }
View Full Code Here

    private static NodeIdGenerator nodeIdGen = NodeIdGenerator.getGenerator();
   
    public FunctionalLogicalOptimizer(LogicalPlan plan) {
        super(plan);

        RulePlan rulePlan;       

        // List of rules for the logical optimizer

        // This one has to be first, as the type cast inserter expects the
        // load to only have one output.
        // Find any places in the plan that have an implicit split and make
        // it explicit. Since the RuleMatcher doesn't handle trees properly,
        // we cheat and say that we match any node. Then we'll do the actual
        // test in the transformers check method.
       
        rulePlan = new RulePlan();
        RuleOperator anyLogicalOperator = new RuleOperator(LogicalOperator.class, RuleOperator.NodeType.ANY_NODE,
                new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
        rulePlan.add(anyLogicalOperator);
        mRules.add(new Rule<LogicalOperator, LogicalPlan>(rulePlan,
                new ImplicitSplitInserter(plan), "ImplicitSplitInserter"));


        // Add type casting to plans where the schema has been declared (by
        // user, data, or data catalog).
        rulePlan = new RulePlan();
        RuleOperator loLoad = new RuleOperator(LOLoad.class,
                new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
        rulePlan.add(loLoad);
        mRules.add(new Rule<LogicalOperator, LogicalPlan>(rulePlan,
                new TypeCastInserter(plan, LOLoad.class.getName()), "LoadTypeCastInserter"));

        // Add type casting to plans where the schema has been declared by
        // user in a statement with stream operator.
        rulePlan = new RulePlan();
        RuleOperator loStream= new RuleOperator(LOStream.class,
                new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
        rulePlan.add(loStream);
        mRules.add(new Rule<LogicalOperator, LogicalPlan>(rulePlan, new TypeCastInserter(plan,
                LOStream.class.getName()), "StreamTypeCastInserter"));
       
        // Push up filters as far as we can
        rulePlan = new RulePlan();
        RuleOperator loFilter = new RuleOperator(LOFilter.class,
                new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
        rulePlan.add(loFilter);
        mRules.add(new Rule<LogicalOperator, LogicalPlan>(rulePlan,
                new TypeCastInserter(plan, LOFilter.class.getName()),
                "PushUpFilter"));

    }
View Full Code Here

    private static NodeIdGenerator nodeIdGen = NodeIdGenerator.getGenerator();
   
    public FunctionalLogicalOptimizer(LogicalPlan plan) {
        super(plan);

        RulePlan rulePlan;       

        // List of rules for the logical optimizer

        // This one has to be first, as the type cast inserter expects the
        // load to only have one output.
        // Find any places in the plan that have an implicit split and make
        // it explicit. Since the RuleMatcher doesn't handle trees properly,
        // we cheat and say that we match any node. Then we'll do the actual
        // test in the transformers check method.
       
        rulePlan = new RulePlan();
        RuleOperator anyLogicalOperator = new RuleOperator(LogicalOperator.class, RuleOperator.NodeType.ANY_NODE,
                new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
        rulePlan.add(anyLogicalOperator);
        mRules.add(new Rule<LogicalOperator, LogicalPlan>(rulePlan,
                new ImplicitSplitInserter(plan), "ImplicitSplitInserter"));


        // Add type casting to plans where the schema has been declared (by
        // user, data, or data catalog).
        rulePlan = new RulePlan();
        RuleOperator loLoad = new RuleOperator(LOLoad.class,
                new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
        rulePlan.add(loLoad);
        mRules.add(new Rule<LogicalOperator, LogicalPlan>(rulePlan,
                new TypeCastInserter(plan, LOLoad.class.getName()), "LoadTypeCastInserter"));

        // Add type casting to plans where the schema has been declared by
        // user in a statement with stream operator.
        rulePlan = new RulePlan();
        RuleOperator loStream= new RuleOperator(LOStream.class,
                new OperatorKey(SCOPE, nodeIdGen.getNextNodeId(SCOPE)));
        rulePlan.add(loStream);
        mRules.add(new Rule<LogicalOperator, LogicalPlan>(rulePlan, new TypeCastInserter(plan,
                LOStream.class.getName()), "StreamTypeCastInserter"));

    }
View Full Code Here

TOP

Related Classes of org.apache.pig.impl.plan.optimizer.RulePlan

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.