Examples of org.apache.pig.newplan.logical.relational.LOCogroup

Package org.apache.pig.newplan.logical.relational

Examples of org.apache.pig.newplan.logical.relational.LOCogroup

org.apache.pig.newplan.logical.relational.LOCogroup

                       "c = foreach b {d = order a by $1; generate flatten(d), MAX(a.age) as max_age;};" +
                       "store c into 'output';";
        LogicalPlan lp = buildPlan(query);
        Operator store = lp.getSinks().get(0);
        LOForEach foreach = (LOForEach)lp.getPredecessors(store).get(0);
        LOCogroup cogroup = (LOCogroup) lp.getPredecessors(foreach).get(0);
        String s = cogroup.getSchema().toString(false);
        Assert.assertTrue( s.equals("group:bytearray,a:bag{:tuple(name:bytearray,age:bytearray,gpa:bytearray)}"));
        s = foreach.getSchema().toString(false);
        Assert.assertTrue( s.equals("d::name:bytearray,d::age:bytearray,d::gpa:bytearray,max_age:double"));
    }

View Full Code Here

        String query = "a = load 'one' as (name, age, gpa);" + 
        "b = group a by *;" +
        "store b into 'output';";
        LogicalPlan lp = buildPlan(query);
        Operator store = lp.getSinks().get(0);
        LOCogroup cogroup = (LOCogroup)lp.getPredecessors(store).get(0);
        String s = cogroup.getSchema().toString(false);
        Assert.assertTrue(s.equals("group:tuple(name:bytearray,age:bytearray,gpa:bytearray),a:bag{:tuple(name:bytearray,age:bytearray,gpa:bytearray)}"));
    }

View Full Code Here

                       "b = load 'two' as (first_name, enrol_age, high_school_gpa);" +
                       "c = group a by *, b by *;" +
                       "store c into 'output';";
        LogicalPlan lp = buildPlan(query);
        Operator store = lp.getSinks().get(0);
        LOCogroup cogroup = (LOCogroup)lp.getPredecessors(store).get(0);
        String s = cogroup.getSchema().toString(false);
        Assert.assertTrue(s.equals("group:tuple(name:bytearray,age:bytearray,gpa:bytearray),a:bag{:tuple(name:bytearray,age:bytearray,gpa:bytearray)},b:bag{:tuple(first_name:bytearray,enrol_age:bytearray,high_school_gpa:bytearray)}"));
    }

View Full Code Here

            // set schemas
            load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1))) ;
            load2.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema2))) ;
            load2.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema2))) ;
    
            LOCogroup cogroup1 = new LOCogroup(plan);
            
            // Create expression inner plan #1 of input #1
            LogicalExpressionPlan innerPlan11 = new LogicalExpressionPlan() ;
            ProjectExpression project111 = new ProjectExpression(innerPlan11, 0, 0, cogroup1) ;
            ConstantExpression const111 = new ConstantExpression(innerPlan11, 26F) ;
            SubtractExpression subtract111 = new SubtractExpression(innerPlan11, project111, const111) ;
    


    
            // Create expression inner plan #2 of input #1
            LogicalExpressionPlan innerPlan21 = new LogicalExpressionPlan() ;
            ProjectExpression project211 = new ProjectExpression(innerPlan21, 0, 0, cogroup1) ;
            ProjectExpression project212 = new ProjectExpression(innerPlan21, 0, 1, cogroup1) ;
    
            AddExpression add211 = new AddExpression(innerPlan21, project211, project212) ;


   
            // Create expression inner plan #1 of input #2
            LogicalExpressionPlan innerPlan12 = new LogicalExpressionPlan() ;
            ProjectExpression project121 = new ProjectExpression(innerPlan12, 1, 0, cogroup1) ;
            ConstantExpression const121 = new ConstantExpression(innerPlan12, 26) ;
            SubtractExpression subtract121 = new SubtractExpression(innerPlan12, project121, const121) ;
    


            // Create expression inner plan #2 of input #2
            LogicalExpressionPlan innerPlan22 = new LogicalExpressionPlan() ;
            ConstantExpression const122 = new ConstantExpression(innerPlan22, 26) ;
//            innerPlan22.add(const122) ;
    
            // Create Cogroup
            ArrayList<LogicalRelationalOperator> inputs = new ArrayList<LogicalRelationalOperator>() ;
            inputs.add(load1) ;
            inputs.add(load2) ;
    
            MultiMap<Integer, LogicalExpressionPlan> maps
                                = new MultiMap<Integer, LogicalExpressionPlan>() ;
            maps.put(0, innerPlan11);
            maps.put(0, innerPlan21);
            maps.put(1, innerPlan12);
            maps.put(1, innerPlan22);
    
            boolean[] isInner = new boolean[inputs.size()] ;
            for (int i=0; i < isInner.length ; i++) {
                isInner[i] = false ;
            }
    
            cogroup1.setInnerFlags(isInner);
            cogroup1.setExpressionPlans(maps);
    
            // construct the main plan
            plan.add(load1) ;
            plan.add(load2) ;
            plan.add(cogroup1) ;
    
            plan.connect(load1, cogroup1);
            plan.connect(load2, cogroup1);
    
            CompilationMessageCollector collector = new CompilationMessageCollector() ;
            TypeCheckingRelVisitor typeChecker = new TypeCheckingRelVisitor(plan, collector);
            typeChecker.visit();  
            printMessageCollector(collector) ;
            //printTypeGraph(plan) ;
    
            if (collector.hasError()) {
                throw new AssertionError("Expect no error") ;
            }
            
            // check outer schema
            LogicalSchema endResultSchema = cogroup1.getSchema() ;
    
            // Tuple group column
            assertEquals(endResultSchema.getField(0).type, DataType.TUPLE) ;
            assertEquals(endResultSchema.getField(0).schema.getField(0).type, DataType.DOUBLE) ;
            assertEquals(endResultSchema.getField(0).schema.getField(1).type, DataType.LONG);

View Full Code Here

    
            // set schemas
            load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1))) ; ;
            load2.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema2))) ;
    
            LOCogroup cogroup1 = new LOCogroup(plan);
                    
            // Create expression inner plan #1 of input #1
            LogicalExpressionPlan innerPlan11 = new LogicalExpressionPlan() ;
            ProjectExpression project111 = new ProjectExpression(innerPlan11, 0, 0, cogroup1) ;
            ConstantExpression const111 = new ConstantExpression(innerPlan11, 26F) ;
            SubtractExpression subtract111 = new SubtractExpression(innerPlan11, project111, const111) ;


    
            // Create expression inner plan #1 of input #2
            LogicalExpressionPlan innerPlan12 = new LogicalExpressionPlan() ;
            ProjectExpression project121 = new ProjectExpression(innerPlan12, 1, 0, cogroup1) ;
            ConstantExpression const121 = new ConstantExpression(innerPlan12, 26) ;
            SubtractExpression subtract121 = new SubtractExpression(innerPlan12, project121, const121) ;
    


    
            // Create Cogroup
            ArrayList<LogicalRelationalOperator> inputs = new ArrayList<LogicalRelationalOperator>() ;
            inputs.add(load1) ;
            inputs.add(load2) ;
    
            MultiMap<Integer, LogicalExpressionPlan> maps
                                = new MultiMap<Integer, LogicalExpressionPlan>() ;
            maps.put(0, innerPlan11);
            maps.put(1, innerPlan12);
    
            boolean[] isInner = new boolean[inputs.size()] ;
            for (int i=0; i < isInner.length ; i++) {
                isInner[i] = false ;
            }
    
            
            cogroup1.setInnerFlags(isInner);
            cogroup1.setExpressionPlans(maps);
    
            // construct the main plan
            plan.add(load1) ;
            plan.add(load2) ;
            plan.add(cogroup1) ;
    
            plan.connect(load1, cogroup1);
            plan.connect(load2, cogroup1);
    
            CompilationMessageCollector collector = new CompilationMessageCollector() ;
            TypeCheckingRelVisitor typeChecker = new TypeCheckingRelVisitor(plan, collector);
            typeChecker.visit();  
            printMessageCollector(collector) ;
            //printTypeGraph(plan) ;
    
            if (collector.hasError()) {
                throw new AssertionError("Expect no error") ;
            }
    
            // check outer schema
            LogicalSchema endResultSchema = cogroup1.getSchema() ;
    
            // Tuple group column
            assertEquals(endResultSchema.getField(0).type, DataType.DOUBLE) ;
    
            assertEquals(endResultSchema.getField(1).type, DataType.BAG) ;

View Full Code Here

    
            // set schemas
            load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1))) ;
            load2.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema2))) ;
    
            LOCogroup cogroup1 = new LOCogroup(plan);
            // Create expression inner plan #1
            LogicalExpressionPlan innerPlan11 = new LogicalExpressionPlan() ;
            ProjectExpression project111 = new ProjectExpression(innerPlan11, 0, 0, cogroup1) ;
            ConstantExpression const111 = new ConstantExpression(innerPlan11, 26F) ;
            SubtractExpression subtract111 = new SubtractExpression(innerPlan11, project111, const111) ;


    
            // Create expression inner plan #2
            LogicalExpressionPlan innerPlan12 = new LogicalExpressionPlan() ;
            ConstantExpression const121 = new ConstantExpression(innerPlan12, 26) ;
//            innerPlan12.add(const121) ;
    
            // Create Cogroup
            ArrayList<LogicalRelationalOperator> inputs = new ArrayList<LogicalRelationalOperator>() ;
            inputs.add(load1) ;
            inputs.add(load2) ;
    
            MultiMap<Integer, LogicalExpressionPlan> maps
                                = new MultiMap<Integer, LogicalExpressionPlan>() ;
            maps.put(0, innerPlan11);
            maps.put(1, innerPlan12);
    
            boolean[] isInner = new boolean[inputs.size()] ;
            for (int i=0; i < isInner.length ; i++) {
                isInner[i] = false ;
            }
    
            
            cogroup1.setInnerFlags(isInner);
            cogroup1.setExpressionPlans(maps);
    
            // construct the main plan
            plan.add(load1) ;
            plan.add(load2) ;
            plan.add(cogroup1) ;
    
            plan.connect(load1, cogroup1);
            plan.connect(load2, cogroup1);
    
            CompilationMessageCollector collector = new CompilationMessageCollector() ;
            TypeCheckingRelVisitor typeChecker = new TypeCheckingRelVisitor(plan, collector);
            typeChecker.visit();  
            printMessageCollector(collector) ;
            //printTypeGraph(plan) ;
    
            if (collector.hasError()) {
                throw new AssertionError("Expect no error") ;
            }
    
            // check outer schema
            LogicalSchema endResultSchema = cogroup1.getSchema() ;
    
            // Tuple group column
            assertEquals(endResultSchema.getField(0).type, DataType.FLOAT) ;
    
            assertEquals(endResultSchema.getField(1).type, DataType.BAG) ;

View Full Code Here

        LogicalPlan lp;
        String query = "a = load 'myfile' as (name, age, gpa);" + 
                   "b = group a by (name, age);";
        lp = buildPlan( query + "store b into 'output';");
        Operator store = lp.getSinks().get(0);
        LOCogroup cogroup = (LOCogroup) lp.getPredecessors(store).get(0);
        
        LogicalSchema actual = cogroup.getSchema();
        System.out.println( actual.toString( false ) );


        Assert.assertTrue(  actual.toString( false ).equals( "group:tuple(name:bytearray,age:bytearray),a:bag{:tuple(name:bytearray,age:bytearray,gpa:bytearray)}" ) );


        lp = buildPlan(query +

View Full Code Here

        String query = "a = load 'myfile' as (name:Chararray, age:Int, gpa:Float);" +
                       "b = group a by (name, age);" +
                       "store b into 'output';";
        lp = buildPlan( query );
        Operator store = lp.getSinks().get(0);
        LOCogroup cogroup = (LOCogroup) lp.getPredecessors(store).get(0);


        Schema.FieldSchema nameFs = new Schema.FieldSchema("name", DataType.CHARARRAY);
        Schema.FieldSchema ageFs = new Schema.FieldSchema("age", DataType.INTEGER);
        Schema.FieldSchema gpaFs = new Schema.FieldSchema("gpa", DataType.FLOAT);


        Schema groupSchema = new Schema(nameFs);
        groupSchema.add(ageFs);
        Schema.FieldSchema groupFs = new Schema.FieldSchema("group", groupSchema, DataType.TUPLE);


        Schema loadSchema = new Schema(nameFs);
        loadSchema.add(ageFs);
        loadSchema.add(gpaFs);


        Schema.FieldSchema bagFs = new Schema.FieldSchema("a", loadSchema, DataType.BAG);


        Schema cogroupExpectedSchema = new Schema(groupFs);
        cogroupExpectedSchema.add(bagFs);
        Assert.assertTrue(cogroup.getSchema().toString(false).equals("group:tuple(name:chararray,age:int),a:bag{:tuple(name:chararray,age:int,gpa:float)}"));
    }

View Full Code Here

                       "c = foreach b {d = order a by $1; generate flatten(d), MAX(a.age) as max_age;};" +
                       "store c into 'output';";
        LogicalPlan lp = buildPlan(query);
        Operator store = lp.getSinks().get(0);
        LOForEach foreach = (LOForEach)lp.getPredecessors(store).get(0);
        LOCogroup cogroup = (LOCogroup) lp.getPredecessors(foreach).get(0);
        String s = cogroup.getSchema().toString(false);
        Assert.assertTrue( s.equals("group:bytearray,a:bag{:tuple(name:bytearray,age:bytearray,gpa:bytearray)}"));
        s = foreach.getSchema().toString(false);
        Assert.assertTrue( s.equals("d::name:bytearray,d::age:bytearray,d::gpa:bytearray,max_age:double"));
    }

View Full Code Here

        String query = "a = load 'one' as (name, age, gpa);" + 
        "b = group a by *;" +
        "store b into 'output';";
        LogicalPlan lp = buildPlan(query);
        Operator store = lp.getSinks().get(0);
        LOCogroup cogroup = (LOCogroup)lp.getPredecessors(store).get(0);
        String s = cogroup.getSchema().toString(false);
        Assert.assertTrue(s.equals("group:tuple(name:bytearray,age:bytearray,gpa:bytearray),a:bag{:tuple(name:bytearray,age:bytearray,gpa:bytearray)}"));
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.pig.newplan.logical.relational.LOCogroup

org.apache.pig.newplan.logical.expression.LogicalExpression

org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema

org.apache.pig.newplan.logical.rules.GroupByConstParallelSetter

org.apache.pig.newplan.logical.rules.GroupByConstParallelSetter$GroupAllParallelSetterTransformer

org.apache.pig.newplan.logical.rules.PushUpFilter$PushUpFilterTransformer

org.apache.pig.parser.LogicalPlanBuilder

org.apache.pig.pen.LineageTrimmingVisitor

org.apache.pig.test.TestCollectedGroup

org.apache.pig.test.TestLogicalPlanBuilder

org.apache.pig.test.TestLogicalPlanMigrationVisitor

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.