Examples of org.apache.pig.newplan.optimizer.PlanOptimizer

Package org.apache.pig.newplan.optimizer

Examples of org.apache.pig.newplan.optimizer.PlanOptimizer

org.apache.pig.newplan.optimizer.PlanOptimizer
The core class of the optimizer. The basic design of this class is that it is provided a list of RuleSets. RuleSets represent all of the optimizer rules that can be run together. The rules in the RuleSet will be run repeatedly until either no rule in the RuleSet passes check and calls transform or until maxIter iterations (default 500) has been made over the RuleSet. Then the next RuleSet will be moved to. Once finished, a given RuleSet is never returned to. Each rule is has two parts: a pattern and and associated transformer. Transformers have two important functions: check(), and transform(). The pattern describes a pattern of node types that the optimizer will look to match. If that match is found anywhere in the plan, then check() will be called. check() allows the rule to look more in depth at the matched pattern and decide whether the rule should be run or not. For example, one might design a rule to push filters above join that would look for the pattern filter(join) (meaning a filter followed by a join). But only certain types of filters can be pushed. The check() function would need to decide whether the filter that it found was pushable or not. If check() returns true, the rule is said to have matched, and transform() is then called. This function is responsible for making changes in the logical plan. Once transform is complete PlanPatcher.patchUp will be called to do any necessary cleanup in the plan, such as resetting schemas, etc.

        // boolean constant elimination: AND
        String query = "b = filter (load 'd.txt' as (id:int, v1, v2)) by ((v1 is not null) AND (id == 1) AND (1 == 1));" +
                       "store b into 'empty';";
        LogicalPlan newLogicalPlan = Util.buildLp(pigServer, query);


        PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
        optimizer.optimize();


        query = "b = filter (load 'd.txt' as (id:int, v1, v2)) by ((v1 is not null) AND (id == 1));" +
                "store b into 'empty';";
        LogicalPlan expected = Util.buildLp(pigServer, query);


        assertTrue(expected.isEqual(newLogicalPlan));


        // boolean constant elimination: OR
        query = "b = filter (load 'd.txt' as (id:int, v1, v2)) by (((v1 is not null) AND (id == 1)) OR (1 == 0));" +
                "store b into 'empty';";
        newLogicalPlan = Util.buildLp(pigServer, query);


        optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
        optimizer.optimize();


        query = "b = filter (load 'd.txt' as (id:int, v1, v2)) by ((v1 is not null) AND (id == 1));" +
                "store b into 'empty';";
        expected = Util.buildLp(pigServer, query);


        assertTrue(expected.isEqual(newLogicalPlan));


        // the mirror case of the above
        query = "b = filter (load 'd.txt' as (id:int, v1, v2)) by ((1 == 0) OR ((v1 is not null) AND (id == 1)));" +
                "store b into 'empty';";
        newLogicalPlan = Util.buildLp(pigServer, query);


        optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
        optimizer.optimize();


        query = "b = filter (load 'd.txt' as (id:int, v1, v2)) by ((v1 is not null) AND (id == 1));" +
                "store b into 'empty';";
        expected = Util.buildLp(pigServer, query);

View Full Code Here

    public void test4() throws Exception {
        String query = "b = filter (load 'd.txt' as (a:chararray, b:long, c:map[], d:chararray, e:chararray)) by a == 'v' and b == 117L and c#'p1' == 'h' and c#'p2' == 'to' and ((d is not null and d != '') or (e is not null and e != ''));" +
                       "store b into 'empty';";
        LogicalPlan newLogicalPlan = Util.buildLp(pigServer, query);


        PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
        optimizer.optimize();


        query = "b = filter (load 'd.txt' as (a:chararray, b:long, c:map[], d:chararray, e:chararray)) by a == 'v' and b == 117L and c#'p1' == 'h' and c#'p2' == 'to' and ((d is not null and d != '') or (e is not null and e != ''));" +
                "store b into 'empty';";
        LogicalPlan expected = Util.buildLp(pigServer, query);


        assertTrue(expected.isEqual(newLogicalPlan));


        // mirror of the above
        query = "b = filter (load 'd.txt' as (a:chararray, b:long, c:map[], d:chararray, e:chararray)) by ((d is not null and d != '') or (e is not null and e != '')) and a == 'v' and b == 117L and c#'p1' == 'h' and c#'p2' == 'to';" +
                "store b into 'empty';";
        newLogicalPlan = Util.buildLp(pigServer, query);


        optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
        optimizer.optimize();


        query = "b = filter (load 'd.txt' as (a:chararray, b:long, c:map[], d:chararray, e:chararray)) by ((d is not null and d != '') or (e is not null and e != '')) and a == 'v' and b == 117L and c#'p1' == 'h' and c#'p2' == 'to';" +
                "store b into 'empty';";
        expected = Util.buildLp(pigServer, query);

View Full Code Here

        StringBuilder sb = new StringBuilder();
        sb.append("b = filter (load 'd.txt' as (a:int, b:int, c:int, d:int)) by (((a < 1) " + (b1 ? "and" : "or") + " (b < 2)) " + (b2 ? "and" : "or") + " ((c < 3) " + (b3 ? "and" : "or") + " (d < 4)));");
        String query = sb.toString() + "store b into 'empty';";
        LogicalPlan newLogicalPlan = Util.buildLp(pigServer, query);


        PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
        optimizer.optimize();


        LogicalPlan expected = Util.buildLp(pigServer, query);


        assertTrue(expected.isEqual(newLogicalPlan));
    }

View Full Code Here

        StringBuilder sb = new StringBuilder();
        sb.append("b = filter (load 'd.txt' as (a:int, b:int, c:int, d:int, e:int, f:int, g:int, h:int)) by ((((a < 1) " + (b1 ? "and" : "or") + " (b < 2)) " + (b2 ? "and" : "or") + " ((c < 3) " + (b3 ? "and" : "or") + " (d < 4))) " + (b4 ? "and" : "or") + " (((e < 5) " + (b5 ? "and" : "or") + " (f < 6)) " + (b6 ? "and" : "or") + " ((g < 7) " + (b7 ? "and" : "or") + " (h < 8))));");
        String query = sb.toString() + "store b into 'empty';";
        LogicalPlan newLogicalPlan = Util.buildLp(pigServer, query);


        PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
        optimizer.optimize();


        LogicalPlan expected = Util.buildLp(pigServer, query);


        assertTrue(expected.isEqual(newLogicalPlan));
    }

View Full Code Here

    public void test7() throws Exception {
        String query = "b = filter (load 'd.txt' as (k1, k2, k3, v1, v2, v3)) by k2#'f1'#'f' is not null and (v2#'f'#'f1' is not null or v2#'f'#'f2' is not null);" +
                       "store b into 'empty';";
        LogicalPlan newLogicalPlan = Util.buildLp(pigServer, query);


        PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
        optimizer.optimize();


        query = "b = filter (load 'd.txt' as (k1, k2, k3, v1, v2, v3)) by k2#'f1'#'f' is not null and (v2#'f'#'f1' is not null or v2#'f'#'f2' is not null);" +
                "store b into 'empty';";
        LogicalPlan expected = Util.buildLp(pigServer, query);


        assertTrue(expected.isEqual(newLogicalPlan));


        query = "b = filter (load 'd.txt' as (k1, k2, k3, v1, v2, v3)) by k2#'f1'#'f' is not null and (v2#'f1'#'f' is not null or v2#'f2'#'f' is not null);" +
                "store b into 'empty';";
        newLogicalPlan = Util.buildLp(pigServer, query);


        optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
        optimizer.optimize();


        query = "b = filter (load 'd.txt' as (k1, k2, k3, v1, v2, v3)) by k2#'f1'#'f' is not null and (v2#'f1'#'f' is not null or v2#'f2'#'f' is not null);" +
                "store b into 'empty';";
        expected = Util.buildLp(pigServer, query);

View Full Code Here

    public void test8() throws Exception {
        String query = "b = filter (load 'd.txt' as (a0, a1)) by (a0 is not null or a1 is not null) and IsEmpty(a0);" +
                       "store b into 'empty';";
        LogicalPlan newLogicalPlan = Util.buildLp(pigServer, query);


        PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
        optimizer.optimize();


        query = "b = filter (load 'd.txt' as (a0, a1)) by (a0 is not null or a1 is not null) and IsEmpty(a0);" +
                "store b into 'empty';";
        LogicalPlan expected = Util.buildLp(pigServer, query);

View Full Code Here

    public void testSimiplificationNonDeterministicUdf() throws Exception {
        String query = "b = filter (load 'd.txt' as (a0, a1)) by RANDOM() > 0.1 and RANDOM() > 0.1;" +
                       "store b into 'empty';";
        LogicalPlan newLogicalPlan = Util.buildLp(pigServer, query);


        PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
        optimizer.optimize();
        LOFilter optimizedFilt = (LOFilter) newLogicalPlan.getSuccessors(newLogicalPlan.getSources().get(0)).get(0);


        //expected query is same as original query, optimizer should not combine
        // conditions involving non deterministic udfs
        query = "b = filter (load 'd.txt' as (a0, a1)) by RANDOM() > 0.1 and RANDOM() > 0.1;" +

View Full Code Here

        String query = "b = filter (load 'd.txt' as (a0, a1)) by " +
        " NOT IsEmpty( " +  MyFilterFunc.class.getName() + "(a0,a1));" +
        "store b into 'empty';";
        LogicalPlan newLogicalPlan = Util.buildLp(pigServer, query);


        PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
        optimizer.optimize();


        //expected plan is same as original plan
        LogicalPlan expected = Util.buildLp(pigServer, query);


        assertTrue(expected.isEqual(newLogicalPlan));

View Full Code Here

    public void testEqualNotEqualWithSameValue() throws Exception {
        String query = "b = filter (load 'd.txt' as (a0:int, a1:int)) "
                       + "by ((a0 == 1) or (a0 != 1));"
                       + "store b into 'empty';";
        LogicalPlan newLogicalPlan = Util.buildLp(pigServer, query);
        PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
        optimizer.optimize();


        //expected plan is same as original plan
        LogicalPlan expected = Util.buildLp(pigServer, query);
        assertTrue(expected.isEqual(newLogicalPlan));


        //swapping == and !=
        query = "b = filter (load 'd.txt' as (a0:int, a1:int)) "
                       + "by ((a0 != 1) or (a0 == 1));"
                       + "store b into 'empty';";
        newLogicalPlan = Util.buildLp(pigServer, query);
        optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
        optimizer.optimize();


        //expected plan is same as original plan
        expected = Util.buildLp(pigServer, query);
        assertTrue(expected.isEqual(newLogicalPlan));


        //more realistic test case which created incorrect output
        query = "b = filter (load 'd.txt' as (a0:int, a1:int)) "
                       + "by ((a0 == 1 and a1 == 3) or (a0 != 1));"
                       + "store b into 'empty';";
        newLogicalPlan = Util.buildLp(pigServer, query);
        optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
        optimizer.optimize();


        //expected plan is same as original plan
        expected = Util.buildLp(pigServer, query);
        assertTrue(expected.isEqual(newLogicalPlan));
    }

View Full Code Here

        String query = "a = load 'd.txt' as (id, v1, v2);" +
        "b = filter a by v1==NULL;" +
        "store b into 'empty';";
        LogicalPlan newLogicalPlan = buildPlan(query);


        PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();


        query  = "a = load 'd.txt' as (id, v1, v2);" +
        "b = filter a by v1==NULL;"  +
        "store b into 'empty';";
        LogicalPlan expected = buildPlan(query);


        assertTrue(expected.isEqual(newLogicalPlan));


        // no schema
        query = "a = load 'd.txt';" +
        "b = foreach a generate $0, $1;" +
        "store b into 'empty';";
        newLogicalPlan = buildPlan(query);


        optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();


        query = "a = load 'd.txt';"+
        "b = foreach a generate $0, $1;"+
        "store b into 'empty';";
        expected = buildPlan(query);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.pig.newplan.optimizer.PlanOptimizer

org.apache.pig.newplan.OperatorPlan

org.apache.pig.test.TestConstantCalculator

org.apache.pig.test.TestFilterSimplification

org.apache.pig.test.TestMergeForEachOptimization

org.apache.pig.test.TestNewPartitionFilterPushDown

org.apache.pig.test.TestNewPlanColumnPrune

org.apache.pig.test.TestNewPlanFilterAboveForeach

org.apache.pig.test.TestNewPlanFilterRule

org.apache.pig.test.TestNewPlanPushDownForeachFlatten

org.apache.pig.test.TestNewPlanPushUpFilter

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.