Examples of org.apache.pig.test.utils.LogicalPlanTester

org.apache.pig.test.utils.LogicalPlanTester
This class is used for logical plan testing


    }


    @Test
    public void test2() throws Exception {
        LogicalPlanTester lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("b = filter (load 'd.txt' as (name, age, gpa)) by age >= 50 or name > 'fred' and gpa <= 3.0 or name >= 'bob';");
        org.apache.pig.impl.logicalLayer.LogicalPlan plan = lpt.buildPlan("store b into 'empty';");
        LogicalPlan newLogicalPlan = migratePlan(plan);


        PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
        optimizer.optimize();


        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("b = filter (load 'd.txt' as (name, age, gpa)) by age >= 50 or name >= 'bob';");
        plan = lpt.buildPlan("store b into 'empty';");
        LogicalPlan expected = migratePlan(plan);


        assertTrue(expected.isEqual(newLogicalPlan));
        
        // Regex filtering
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("b = filter (load 'd.txt' as (name:chararray, age:int, registration, contributions:double)) by (name matches '^fred.*' and (chararray)registration matches '^dem.*');");
        plan = lpt.buildPlan("store b into 'empty';");
        newLogicalPlan = migratePlan(plan);


        optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
        optimizer.optimize();


        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("b = filter (load 'd.txt' as (name:chararray, age:int, registration, contributions:double)) by (name matches '^fred.*' and (chararray)registration matches '^dem.*');");
        plan = lpt.buildPlan("store b into 'empty';");
        expected = migratePlan(plan);


        assertTrue(expected.isEqual(newLogicalPlan));
        
        // NOT Regex filtering 
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("b = filter (load 'd.txt') by (not $0 matches '^fred.*');");
        plan = lpt.buildPlan("store b into 'empty';");
        newLogicalPlan = migratePlan(plan);


        optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
        optimizer.optimize();


        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("b = filter (load 'd.txt') by (not $0 matches '^fred.*');");
        plan = lpt.buildPlan("store b into 'empty';");
        expected = migratePlan(plan);


        assertTrue(expected.isEqual(newLogicalPlan));   
        
        // naiive filtering
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("b = filter (load 'd.txt') by 1==1;");
        plan = lpt.buildPlan("store b into 'empty';");
        newLogicalPlan = migratePlan(plan);


        optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
        optimizer.optimize();


        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("b = load 'd.txt';");
        plan = lpt.buildPlan("store b into 'empty';");
        expected = migratePlan(plan);


        assertTrue(expected.isEqual(newLogicalPlan));
    }

View Full Code Here

    }


    @Test
    public void test3() throws Exception {
        // boolean constant elimination: AND
        LogicalPlanTester lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("b = filter (load 'd.txt' as (id:int, v1, v2)) by ((v1 is not null) AND (id == 1) AND (1 == 1));");
        org.apache.pig.impl.logicalLayer.LogicalPlan plan = lpt.buildPlan("store b into 'empty';");
        LogicalPlan newLogicalPlan = migratePlan(plan);


        PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
        optimizer.optimize();


        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("b = filter (load 'd.txt' as (id:int, v1, v2)) by ((v1 is not null) AND (id == 1));");
        plan = lpt.buildPlan("store b into 'empty';");
        LogicalPlan expected = migratePlan(plan);


        assertTrue(expected.isEqual(newLogicalPlan));


        // boolean constant elimination: OR
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("b = filter (load 'd.txt' as (id:int, v1, v2)) by (((v1 is not null) AND (id == 1)) OR (1 == 0));");
        plan = lpt.buildPlan("store b into 'empty';");
        newLogicalPlan = migratePlan(plan);


        optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
        optimizer.optimize();


        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("b = filter (load 'd.txt' as (id:int, v1, v2)) by ((v1 is not null) AND (id == 1));");
        plan = lpt.buildPlan("store b into 'empty';");
        expected = migratePlan(plan);


        assertTrue(expected.isEqual(newLogicalPlan));
        
        // the mirror case of the above
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("b = filter (load 'd.txt' as (id:int, v1, v2)) by ((1 == 0) OR ((v1 is not null) AND (id == 1)));");
        plan = lpt.buildPlan("store b into 'empty';");
        newLogicalPlan = migratePlan(plan);


        optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
        optimizer.optimize();


        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("b = filter (load 'd.txt' as (id:int, v1, v2)) by ((v1 is not null) AND (id == 1));");
        plan = lpt.buildPlan("store b into 'empty';");
        expected = migratePlan(plan);


        assertTrue(expected.isEqual(newLogicalPlan));
    }

View Full Code Here

        assertTrue(expected.isEqual(newLogicalPlan));
    }


    @Test
    public void test4() throws Exception {
        LogicalPlanTester lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("b = filter (load 'd.txt' as (a:chararray, b:long, c:map[], d:chararray, e:chararray)) by a == 'v' and b == 117L and c#'p1' == 'h' and c#'p2' == 'to' and ((d is not null and d != '') or (e is not null and e != ''));"); 


        org.apache.pig.impl.logicalLayer.LogicalPlan plan = lpt.buildPlan("store b into 'empty';");
        LogicalPlan newLogicalPlan = migratePlan(plan);


        PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
        optimizer.optimize();


        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("b = filter (load 'd.txt' as (a:chararray, b:long, c:map[], d:chararray, e:chararray)) by a == 'v' and b == 117L and c#'p1' == 'h' and c#'p2' == 'to' and ((d is not null and d != '') or (e is not null and e != ''));"); 
        plan = lpt.buildPlan("store b into 'empty';");
        LogicalPlan expected = migratePlan(plan);


        assertTrue(expected.isEqual(newLogicalPlan));


        // mirror of the above
        lpt.buildPlan("b = filter (load 'd.txt' as (a:chararray, b:long, c:map[], d:chararray, e:chararray)) by ((d is not null and d != '') or (e is not null and e != '')) and a == 'v' and b == 117L and c#'p1' == 'h' and c#'p2' == 'to';"); 


        plan = lpt.buildPlan("store b into 'empty';");
        newLogicalPlan = migratePlan(plan);


        optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
        optimizer.optimize();


        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("b = filter (load 'd.txt' as (a:chararray, b:long, c:map[], d:chararray, e:chararray)) by ((d is not null and d != '') or (e is not null and e != '')) and a == 'v' and b == 117L and c#'p1' == 'h' and c#'p2' == 'to';"); 
        plan = lpt.buildPlan("store b into 'empty';");
        expected = migratePlan(plan);


        assertTrue(expected.isEqual(newLogicalPlan));


    }

View Full Code Here

    private void comboRunner2(boolean b1, boolean b2, boolean b3) throws Exception {
        StringBuilder sb = new StringBuilder();
        sb.append("b = filter (load 'd.txt' as (a:int, b:int, c:int, d:int)) by (((a < 1) " + (b1 ? "and" : "or") + " (b < 2)) " + (b2 ? "and" : "or") + " ((c < 3) " + (b3 ? "and" : "or") + " (d < 4)));");  
        String query = sb.toString();


        LogicalPlanTester lpt = new LogicalPlanTester(pc);
        lpt.buildPlan(query); 


        org.apache.pig.impl.logicalLayer.LogicalPlan plan = lpt.buildPlan("store b into 'empty';");
        LogicalPlan newLogicalPlan = migratePlan(plan);


        PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
        optimizer.optimize();


        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan(query); 
        plan = lpt.buildPlan("store b into 'empty';");
        LogicalPlan expected = migratePlan(plan);


        assertTrue(expected.isEqual(newLogicalPlan));
    }

View Full Code Here

    private void comboRunner3(boolean b1, boolean b2, boolean b3, boolean b4, boolean b5, boolean b6, boolean b7) throws Exception {
        StringBuilder sb = new StringBuilder();
        sb.append("b = filter (load 'd.txt' as (a:int, b:int, c:int, d:int, e:int, f:int, g:int, h:int)) by ((((a < 1) " + (b1 ? "and" : "or") + " (b < 2)) " + (b2 ? "and" : "or") + " ((c < 3) " + (b3 ? "and" : "or") + " (d < 4))) " + (b4 ? "and" : "or") + " (((e < 5) " + (b5 ? "and" : "or") + " (f < 6)) " + (b6 ? "and" : "or") + " ((g < 7) " + (b7 ? "and" : "or") + " (h < 8))));");  
        String query = sb.toString();


        LogicalPlanTester lpt = new LogicalPlanTester(pc);
        lpt.buildPlan(query); 


        org.apache.pig.impl.logicalLayer.LogicalPlan plan = lpt.buildPlan("store b into 'empty';");
        LogicalPlan newLogicalPlan = migratePlan(plan);


        PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
        optimizer.optimize();


        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan(query); 
        plan = lpt.buildPlan("store b into 'empty';");
        LogicalPlan expected = migratePlan(plan);


        assertTrue(expected.isEqual(newLogicalPlan));
    }

View Full Code Here


public class TestLogicalPlanMigrationVisitor extends TestCase {


    PigContext pc = new PigContext(ExecType.LOCAL, new Properties());
    public void testSimplePlan() throws Exception {
        LogicalPlanTester lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd.txt';");
        lpt.buildPlan("b = filter a by $0==NULL;");        
        LogicalPlan plan = lpt.buildPlan("store b into 'empty';");
        
        // check basics
        org.apache.pig.newplan.logical.relational.LogicalPlan newPlan = migratePlan(plan);
        assertEquals(3, newPlan.size());
        assertEquals(newPlan.getSources().size(), 1);

View Full Code Here

        op = (LogicalRelationalOperator)newPlan.getSuccessors(op).get(0);
        assertEquals(op.getClass(), org.apache.pig.newplan.logical.relational.LOStore.class);
    }
    
    public void testPlanWithCast() throws Exception {
        LogicalPlanTester lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd.txt' as (id, c);");
        lpt.buildPlan("b = filter a by (int)id==10;");        
        LogicalPlan plan = lpt.buildPlan("store b into 'empty';");
        
        // check basics
        org.apache.pig.newplan.logical.relational.LogicalPlan newPlan = migratePlan(plan);
        assertEquals(3, newPlan.size());
        assertEquals(newPlan.getSources().size(), 1);

View Full Code Here

        op = (LogicalRelationalOperator)newPlan.getSuccessors(op).get(0);
        assertEquals(op.getClass(), org.apache.pig.newplan.logical.relational.LOStore.class);
    }
    
    public void testJoinPlan() throws Exception {
        LogicalPlanTester lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load 'd1.txt' as (id, c);");
        lpt.buildPlan("b = load 'd2.txt'as (id, c);");
        lpt.buildPlan("c = join a by id, b by c;");
        lpt.buildPlan("d = filter c by a::id==NULL AND b::c==NULL;");        
        LogicalPlan plan = lpt.buildPlan("store d into 'empty';");
      
        // check basics
        org.apache.pig.newplan.logical.relational.LogicalPlan newPlan = migratePlan(plan);
        assertEquals(5, newPlan.size());
        assertEquals(newPlan.getSources().size(), 2);

View Full Code Here

        op = (LogicalRelationalOperator)newPlan.getSuccessors(op).get(0);
        assertEquals(op.getClass(), org.apache.pig.newplan.logical.relational.LOStore.class); 
    }
    
    public void testForeachPlan() throws Exception {
        LogicalPlanTester lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load '/test/d.txt' as (id, d);");
        lpt.buildPlan("b = foreach a generate id, FLATTEN(d);");        
        LogicalPlan plan = lpt.buildPlan("store b into '/test/empty';");
        
        // check basics
        org.apache.pig.newplan.logical.relational.LogicalPlan newPlan = migratePlan(plan);
        
        org.apache.pig.newplan.logical.relational.LogicalPlan expected =

View Full Code Here

        assertTrue(schema.isEqual(aschema));
    }


    public void testForeachSchema() throws Exception {
        // test flatten
        LogicalPlanTester lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load '/test/d.txt' as (id, d:tuple(v, s));");
        LogicalPlan plan = lpt.buildPlan("b = foreach a generate id, FLATTEN(d);");  
                
        org.apache.pig.newplan.logical.relational.LogicalPlan newPlan = migratePlan(plan);
        LogicalRelationalOperator op = (LogicalRelationalOperator)newPlan.getSinks().get(0);
        
        LogicalSchema s2 = new LogicalSchema();
        s2.addField(new LogicalSchema.LogicalFieldSchema("id", null, DataType.BYTEARRAY));
        s2.addField(new LogicalSchema.LogicalFieldSchema("v", null, DataType.BYTEARRAY));
        s2.addField(new LogicalSchema.LogicalFieldSchema("s", null, DataType.BYTEARRAY));
        assertTrue(s2.isEqual(op.getSchema()));
        
        // test no flatten
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load '/test/d.txt' as (id, d:bag{t:(v, s)});");
        plan = lpt.buildPlan("b = foreach a generate id, d;");  
                
        newPlan = migratePlan(plan);
        op = (LogicalRelationalOperator)newPlan.getSinks().get(0);
        
        LogicalSchema aschema = new LogicalSchema();      
        aschema.addField(new LogicalSchema.LogicalFieldSchema("id", null, DataType.BYTEARRAY));
        LogicalSchema aschema2 = new LogicalSchema();
        LogicalSchema aschema3 = new LogicalSchema();
        aschema3.addField(new LogicalSchema.LogicalFieldSchema("v", null, DataType.BYTEARRAY));
        aschema3.addField(new LogicalSchema.LogicalFieldSchema("s", null, DataType.BYTEARRAY));
        aschema2.addField(new LogicalSchema.LogicalFieldSchema("t", aschema3, DataType.TUPLE));
        aschema.addField(new LogicalSchema.LogicalFieldSchema("d", aschema2, DataType.BAG));  
        
        assertTrue(aschema.isEqual(op.getSchema()));
        
        // check with defined data type
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load '/test/d.txt' as (id, d:bag{t:(v:int, s)});");
        lpt.buildPlan("b = foreach a generate id, FLATTEN(d);");        
        plan = lpt.buildPlan("store b into '/test/empty';");
                
        newPlan = migratePlan(plan);
        op = (LogicalRelationalOperator)newPlan.getSinks().get(0);
        op = (LogicalRelationalOperator)newPlan.getPredecessors(op).get(0);
        LogicalSchema schema = op.getSchema();
        
        aschema = new LogicalSchema();
        aschema.addField(new LogicalSchema.LogicalFieldSchema("id", null, DataType.BYTEARRAY));
        aschema.addField(new LogicalSchema.LogicalFieldSchema("v", null, DataType.INTEGER));
        aschema.addField(new LogicalSchema.LogicalFieldSchema("s", null, DataType.BYTEARRAY));
        assertTrue(schema.isEqual(aschema));
      
    
        // test with add
        lpt = new LogicalPlanTester(pc);
        lpt.buildPlan("a = load '/test/d.txt' as (id, v:int, s:int);");
        lpt.buildPlan("b = foreach a generate id, v+s;");        
        plan = lpt.buildPlan("store b into '/test/empty';");
        
        newPlan = migratePlan(plan);
        op = (LogicalRelationalOperator)newPlan.getSinks().get(0);
        op = (LogicalRelationalOperator)newPlan.getPredecessors(op).get(0);
        schema = op.getSchema();

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.pig.test.utils.LogicalPlanTester

org.apache.pig.impl.logicalLayer.optimizer.LogicalOptimizer

org.apache.pig.impl.logicalLayer.optimizer.SchemaCalculator

org.apache.pig.impl.logicalLayer.optimizer.SchemaRemover

org.apache.pig.impl.logicalLayer.validators.TypeCheckingValidator

org.apache.pig.impl.PigContext

org.apache.pig.impl.plan.CompilationMessageCollector

org.apache.pig.test.TestCollectedGroup

org.apache.pig.test.TestCombiner

org.apache.pig.test.TestExperimentalColumnPrune

org.apache.pig.test.TestExperimentalLogToPhyTranslationVisitor

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.