Examples of LogicalPlan

org.apache.drill.common.logical.LogicalPlan
org.apache.pig.experimental.logical.relational.LogicalPlan
LogicalPlan is the logical view of relational operations Pig will execute for a given script. Note that it contains only relational operations. All expressions will be contained in LogicalExpressionPlans inside each relational operator.
org.apache.pig.impl.logicalLayer.LogicalPlan
org.apache.pig.newplan.logical.relational.LogicalPlan
LogicalPlan is the logical view of relational operations Pig will execute for a given script. Note that it contains only relational operations. All expressions will be contained in LogicalExpressionPlans inside each relational operator.
org.apache.tajo.engine.planner.LogicalPlan
This represents and keeps every information about a query plan for a query.

Examples of org.apache.pig.newplan.logical.relational.LogicalPlan

        LogicalPlanTester lpt = new LogicalPlanTester( pc );
        lpt.buildPlan( "A = LOAD 'file.txt' AS (a(u,v), b, c);" );
        lpt.buildPlan( "B = FOREACH A GENERATE $0, b, flatten(1);" );
        lpt.buildPlan( "C = FILTER B BY " + Identity.class.getName() +"(*) > 5;" );
        org.apache.pig.impl.logicalLayer.LogicalPlan plan = lpt.buildPlan( "STORE C INTO 'empty';" );  
        LogicalPlan newLogicalPlan = migrateAndOptimizePlan( plan );


        Operator load = newLogicalPlan.getSources().get( 0 );
        Assert.assertTrue( load instanceof LOLoad );
        Operator fe1 = newLogicalPlan.getSuccessors( load ).get( 0 );
        Assert.assertTrue( fe1 instanceof LOForEach );
        Operator fe2 = newLogicalPlan.getSuccessors( fe1 ).get( 0 );
        Assert.assertTrue( fe2 instanceof LOForEach );
        Operator filter = newLogicalPlan.getSuccessors( fe2 ).get( 0 );
        Assert.assertTrue( filter instanceof LOFilter );
    }

View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LogicalPlan

        planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
        planTester.buildPlan("B = foreach A generate $1, $2;");        
        planTester.buildPlan("C = filter B by $0 < 18;");
        org.apache.pig.impl.logicalLayer.LogicalPlan plan = planTester.buildPlan( "D = STORE C INTO 'empty';" ); 


        LogicalPlan newLogicalPlan = migrateAndOptimizePlan( plan );


        Operator load = newLogicalPlan.getSources().get( 0 );
        Assert.assertTrue( load instanceof LOLoad );
        Operator filter = newLogicalPlan.getSuccessors( load ).get( 0 );
        Assert.assertTrue( filter instanceof LOFilter );
        Operator fe = newLogicalPlan.getSuccessors( filter ).get( 0 );
        Assert.assertTrue( fe instanceof LOForEach );
        Operator store = newLogicalPlan.getSuccessors( fe ).get( 0 );
        Assert.assertTrue( store instanceof LOStore );
    }

View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LogicalPlan

        planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
        planTester.buildPlan("B = foreach A generate $1, $2, COUNT({(1)});");        
        planTester.buildPlan("C = filter B by $2 < 18;");
        org.apache.pig.impl.logicalLayer.LogicalPlan plan = planTester.buildPlan( "D = STORE C INTO 'empty';" ); 


        LogicalPlan newLogicalPlan = migrateAndOptimizePlan( plan );


        Operator load = newLogicalPlan.getSources().get( 0 );
        Assert.assertTrue( load instanceof LOLoad );
        Operator fe = newLogicalPlan.getSuccessors( load ).get( 0 );
        Assert.assertTrue( fe instanceof LOForEach );
        Operator filter = newLogicalPlan.getSuccessors( fe ).get( 0 );
        Assert.assertTrue( filter instanceof LOFilter );
        Operator store = newLogicalPlan.getSuccessors( filter ).get( 0 );
        Assert.assertTrue( store instanceof LOStore );
    }

View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LogicalPlan

        planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
        planTester.buildPlan("B = foreach A generate (int)$1, $2;");        
        planTester.buildPlan("C = filter B by $0 < 18;");
        org.apache.pig.impl.logicalLayer.LogicalPlan plan = planTester.buildPlan( "D = STORE C INTO 'empty';" ); 


        LogicalPlan newLogicalPlan = migrateAndOptimizePlan( plan );


        Operator load = newLogicalPlan.getSources().get( 0 );
        Assert.assertTrue( load instanceof LOLoad );
        Operator fe = newLogicalPlan.getSuccessors( load ).get( 0 );
        Assert.assertTrue( fe instanceof LOForEach );
        Operator filter = newLogicalPlan.getSuccessors( fe ).get( 0 );
        Assert.assertTrue( filter instanceof LOFilter );
        Operator store = newLogicalPlan.getSuccessors( filter ).get( 0 );
        Assert.assertTrue( store instanceof LOStore );
    }

View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LogicalPlan

        planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
        planTester.buildPlan("B = foreach A generate $1, $2;");        
        planTester.buildPlan("C = filter B by (int)$0 < 18;");
        org.apache.pig.impl.logicalLayer.LogicalPlan plan = planTester.buildPlan( "D = STORE C INTO 'empty';" ); 


        LogicalPlan newLogicalPlan = migrateAndOptimizePlan( plan );


        Operator load = newLogicalPlan.getSources().get( 0 );
        Assert.assertTrue( load instanceof LOLoad );
        Operator filter = newLogicalPlan.getSuccessors( load ).get( 0 );
        Assert.assertTrue( filter instanceof LOFilter );
        Operator fe = newLogicalPlan.getSuccessors( filter ).get( 0 );
        Assert.assertTrue( fe instanceof LOForEach );
        Operator store = newLogicalPlan.getSuccessors( fe ).get( 0 );
        Assert.assertTrue( store instanceof LOStore );
    }

View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LogicalPlan

        planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
        planTester.buildPlan("B = foreach A generate $1, $2;");        
        planTester.buildPlan("C = filter B by 1 == 1;");
        org.apache.pig.impl.logicalLayer.LogicalPlan plan = planTester.buildPlan( "D = STORE C INTO 'empty';" ); 


        LogicalPlan newLogicalPlan = migrateAndOptimizePlan( plan );


        Operator load = newLogicalPlan.getSources().get( 0 );
        Assert.assertTrue( load instanceof LOLoad );
        Operator filter = newLogicalPlan.getSuccessors( load ).get( 0 );
        Assert.assertTrue( filter instanceof LOFilter );
        Operator fe = newLogicalPlan.getSuccessors( filter ).get( 0 );
        Assert.assertTrue( fe instanceof LOForEach );
        Operator store = newLogicalPlan.getSuccessors( fe ).get( 0 );
        Assert.assertTrue( store instanceof LOStore );
    }

View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LogicalPlan

        // no foreach
        LogicalPlanTester lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd.txt' as (id, v1, v2);");
        lpt.buildPlan("b = filter a by v1==NULL;");        
        org.apache.pig.impl.logicalLayer.LogicalPlan plan = lpt.buildPlan("store b into 'empty';");  
        LogicalPlan newLogicalPlan = migratePlan(plan);
               
        PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();
        
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd.txt' as (id, v1, v2);");
        lpt.buildPlan("b = filter a by v1==NULL;");        
        plan = lpt.buildPlan("store b into 'empty';");  
        LogicalPlan expected = migratePlan(plan);
        
        assertTrue(expected.isEqual(newLogicalPlan));
        
        // no schema
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd.txt';");
        lpt.buildPlan("b = foreach a generate $0, $1;");
        plan = lpt.buildPlan("store b into 'empty';");  
        newLogicalPlan = migratePlan(plan);
               
        optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();
        
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd.txt';");
        lpt.buildPlan("b = foreach a generate $0, $1;");
        plan = lpt.buildPlan("store b into 'empty';");  
        expected = migratePlan(plan);
        assertTrue(expected.isEqual(newLogicalPlan));
    }

View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LogicalPlan

        // only foreach
        LogicalPlanTester lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd.txt' as (id, v1, v2);");
        lpt.buildPlan("b = foreach a generate id;");        
        org.apache.pig.impl.logicalLayer.LogicalPlan plan = lpt.buildPlan("store b into 'empty';");  
        LogicalPlan newLogicalPlan = migratePlan(plan);
               
        PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();
        
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd.txt' as (id);");
        lpt.buildPlan("b = foreach a generate id;");        
        plan = lpt.buildPlan("store b into 'empty';");  
        LogicalPlan expected = migratePlan(plan);
        
        assertTrue(expected.isEqual(newLogicalPlan));
        
        // with filter
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd.txt' as (id, v1, v5, v3, v4, v2);");
        lpt.buildPlan("b = filter a by v1 != NULL AND (v2+v3)<100;");
        lpt.buildPlan("c = foreach b generate id;");
        plan = lpt.buildPlan("store c into 'empty';");  
        newLogicalPlan = migratePlan(plan);
               
        optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();
        
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd.txt' as (id, v1, v3, v2);");
        lpt.buildPlan("b = filter a by v1 != NULL AND (v2+v3)<100;");
        lpt.buildPlan("c = foreach b generate id;");
        plan = lpt.buildPlan("store c into 'empty';"); 
        expected = migratePlan(plan);
        assertTrue(expected.isEqual(newLogicalPlan));
        
        // with 2 foreach
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd.txt' as (id, v1, v5, v3, v4, v2);");
        lpt.buildPlan("b = foreach a generate v2, v5, v4;");
        lpt.buildPlan("c = foreach b generate v5, v4;");
        plan = lpt.buildPlan("store c into 'empty';");  
        newLogicalPlan = migratePlan(plan);
               
        optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();
        
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd.txt' as (v5, v4);");
        lpt.buildPlan("b = foreach a generate v5, v4;");
        lpt.buildPlan("c = foreach b generate v5, v4;");
        plan = lpt.buildPlan("store c into 'empty';"); 
        expected = migratePlan(plan);
        assertTrue(expected.isEqual(newLogicalPlan));
        
        // with 2 foreach
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd.txt' as (id, v1, v5, v3, v4, v2);");
        lpt.buildPlan("b = foreach a generate id, v1, v5, v3, v4;");
        lpt.buildPlan("c = foreach b generate v5, v4;");
        plan = lpt.buildPlan("store c into 'empty';");  
        newLogicalPlan = migratePlan(plan);
               
        optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();
        
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd.txt' as (v5, v4);");
        lpt.buildPlan("b = foreach a generate v5, v4;");
        lpt.buildPlan("c = foreach b generate v5, v4;");
        plan = lpt.buildPlan("store c into 'empty';"); 
        expected = migratePlan(plan);
        assertTrue(expected.isEqual(newLogicalPlan));
        
        // with 2 foreach and filter in between
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd.txt' as (id, v1, v5, v3, v4, v2);");
        lpt.buildPlan("b = foreach a generate v2, v5, v4;");
        lpt.buildPlan("c = filter b by v2 != NULL;");
        lpt.buildPlan("d = foreach c generate v5, v4;");
        plan = lpt.buildPlan("store d into 'empty';");  
        newLogicalPlan = migratePlan(plan);
               
        optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();
        
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd.txt' as (v5, v4, v2);");
        lpt.buildPlan("b = foreach a generate v2, v5, v4;");
        lpt.buildPlan("c = filter b by v2 != NULL;");
        lpt.buildPlan("d = foreach c generate v5, v4;");
        plan = lpt.buildPlan("store d into 'empty';");  
        expected = migratePlan(plan);
        assertTrue(expected.isEqual(newLogicalPlan));
        
        // with 2 foreach after join
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd.txt' as (id, v1, v2, v3);");
        lpt.buildPlan("b = load 'c.txt' as (id, v4, v5, v6);");
        lpt.buildPlan("c = join a by id, b by id;");       
        lpt.buildPlan("d = foreach c generate a::id, v5, v3, v4;");
        plan = lpt.buildPlan("store d into 'empty';");  
        newLogicalPlan = migratePlan(plan);
               
        optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();
        
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd.txt' as (id, v3);");
        lpt.buildPlan("b = load 'c.txt' as (id, v4, v5);");
        lpt.buildPlan("c = join a by id, b by id;");  
        lpt.buildPlan("d = foreach c generate a::id, v5, v3, v4;");
        plan = lpt.buildPlan("store d into 'empty';");  
        expected = migratePlan(plan);
        assertTrue(expected.isEqual(newLogicalPlan));
        
        // with BinStorage, insert foreach after load
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd.txt' using BinStorage() as (id, v1, v5, v3, v4, v2);");        
        lpt.buildPlan("c = filter a by v2 != NULL;");
        lpt.buildPlan("d = foreach c generate v5, v4;");
        plan = lpt.buildPlan("store d into 'empty';");  
        newLogicalPlan = migratePlan(plan);
               
        optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();
        
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd.txt' using BinStorage() as (id, v1, v5, v3, v4, v2);");
        lpt.buildPlan("b = foreach a generate v5, v4, v2;");
        lpt.buildPlan("c = filter b by v2 != NULL;");
        lpt.buildPlan("d = foreach c generate v5, v4;");
        plan = lpt.buildPlan("store d into 'empty';");  
        expected = migratePlan(plan);
        assertTrue(expected.isEqual(newLogicalPlan));
        
       // with BinStorage, not to insert foreach after load if there is already one
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd.txt' using BinStorage() as (id, v1, v5, v3, v4, v2);");    
        lpt.buildPlan("b = foreach a generate v5, v4, v2;");
        lpt.buildPlan("c = filter b by v2 != NULL;");
        lpt.buildPlan("d = foreach c generate v5;");
        plan = lpt.buildPlan("store d into 'empty';");  
        newLogicalPlan = migratePlan(plan);
               
        optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();
        
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd.txt' using BinStorage() as (id, v1, v5, v3, v4, v2);");
        lpt.buildPlan("b = foreach a generate v5, v2;");
        lpt.buildPlan("c = filter b by v2 != NULL;");
        lpt.buildPlan("d = foreach c generate v5;");
        plan = lpt.buildPlan("store d into 'empty';");  
        expected = migratePlan(plan);
        assertTrue(expected.isEqual(newLogicalPlan));
        
       // with BinStorage, not to insert foreach after load if there is already one
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd.txt' using BinStorage() as (id, v1, v5, v3, v4, v2);");    
        lpt.buildPlan("b = foreach a generate v5, v4, v2, 10;");
        lpt.buildPlan("c = filter b by v2 != NULL;");
        lpt.buildPlan("d = foreach c generate v5;");
        plan = lpt.buildPlan("store d into 'empty';");  
        newLogicalPlan = migratePlan(plan);
               
        optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();
        
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd.txt' using BinStorage() as (id, v1, v5, v3, v4, v2);");
        lpt.buildPlan("b = foreach a generate v5, v2, 10;");
        lpt.buildPlan("c = filter b by v2 != NULL;");
        lpt.buildPlan("d = foreach c generate v5;");
        plan = lpt.buildPlan("store d into 'empty';");  
        expected = migratePlan(plan);
        assertTrue(expected.isEqual(newLogicalPlan));
    }

View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LogicalPlan

         // only foreach
        LogicalPlanTester lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd.txt' as (id, v1, m:map[]);");
        lpt.buildPlan("b = foreach a generate id, m#'path';");        
        org.apache.pig.impl.logicalLayer.LogicalPlan plan = lpt.buildPlan("store b into 'empty';");  
        LogicalPlan newLogicalPlan = migratePlan(plan);
               
        PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();
        
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd.txt' as (id, m:map[]);");
        lpt.buildPlan("b = foreach a generate id, m#'path';");        
        plan = lpt.buildPlan("store b into 'empty';");  
        LogicalPlan expected = migratePlan(plan);
        
        assertTrue(expected.isEqual(newLogicalPlan));
        
        LOLoad op = (LOLoad)newLogicalPlan.getSources().get(0);
        Map<Integer,Set<String>> annotation = 
                (Map<Integer, Set<String>>) op.getAnnotation(MapKeysPruneHelper.REQUIRED_MAPKEYS);
        assertEquals(annotation.size(), 1);
        Set<String> s = new HashSet<String>();
        s.add("path");
        assertEquals(annotation.get(2), s);
        
        // foreach with join
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd.txt' as (id, v1, m:map[]);");
        lpt.buildPlan("b = load 'd.txt' as (id, v1, m:map[]);");
        lpt.buildPlan("c = join a by id, b by id;");
        lpt.buildPlan("d = filter c by a::m#'path' != NULL;");
        lpt.buildPlan("e = foreach d generate a::id, b::id, b::m#'path', a::m;");        
        plan = lpt.buildPlan("store e into 'empty';");  
        newLogicalPlan = migratePlan(plan);
               
        optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();
        
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd.txt' as (id, m:map[]);");
        lpt.buildPlan("b = load 'd.txt' as (id, m:map[]);");
        lpt.buildPlan("c = join a by id, b by id;");
        lpt.buildPlan("d = filter c by a::m#'path' != NULL;");
        lpt.buildPlan("e = foreach d generate a::id, b::id, b::m#'path', a::m;");        
        plan = lpt.buildPlan("store e into 'empty';");  
        expected = migratePlan(plan);
        
        assertTrue(expected.isEqual(newLogicalPlan));
        
        List<Operator> ll = newLogicalPlan.getSources();
        assertEquals(ll.size(), 2);
        LOLoad loada = null;
        LOLoad loadb = null;

View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LogicalPlan

        lpt.buildPlan("a = load 'd.txt' as (id, v:bag{t:(s1,s2,s3)});");
        lpt.buildPlan("b = filter a by id>10;");
        lpt.buildPlan("c = foreach b generate id, FLATTEN(v);");    
        lpt.buildPlan("d = foreach c generate id, v::s2;");    
        org.apache.pig.impl.logicalLayer.LogicalPlan plan = lpt.buildPlan("store d into 'empty';");  
        LogicalPlan newLogicalPlan = migratePlan(plan);
               
        PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
        optimizer.optimize();
        
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd.txt' as (id, v:bag{t:(s1,s2,s3)});");
        lpt.buildPlan("b = filter a by id>10;");
        lpt.buildPlan("c = foreach b generate id, FLATTEN(v);");    
        lpt.buildPlan("d = foreach c generate id, v::s2;");    
        plan = lpt.buildPlan("store d into 'empty';");
        LogicalPlan expected = migratePlan(plan);
        
        assertTrue(expected.isEqual(newLogicalPlan));
    }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.