Examples of LOCogroup

Examples of org.apache.pig.newplan.logical.relational.LOCogroup


    public class GroupAllParallelSetterTransformer extends Transformer {


        @Override
        public boolean check(OperatorPlan matched) throws FrontendException {
            LOCogroup group = (LOCogroup)matched.getSources().get(0);
            MultiMap<Integer, LogicalExpressionPlan> explPlans =
                group.getExpressionPlans();
            //check if the expression plan consist of just a ConstantExpression
            for(LogicalExpressionPlan ep : explPlans.values()){
                Iterator<Operator> op_iter = ep.getOperators();
                if(op_iter.hasNext()){
                    //return false if the ExpressionOperator is not

View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOCogroup

        public void transform(OperatorPlan plan) throws FrontendException {
            Iterator<Operator> iter = plan.getOperators();
            while (iter.hasNext()) {
                Operator op = iter.next();
                if (op instanceof LOCogroup) {
                    LOCogroup group = (LOCogroup)op;
                    if(group.getRequestedParallelism() > 1){
                        log.warn("Resetting parallism to 1 for the group/cogroup " +
                                group.getAlias() +
                        " as the group by expressions returns a constant");
                    }
                    ((LOCogroup) op).setRequestedParallelism(1);
                }
            }

View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOCogroup

    private String convertCubeToFGPlan(SourceLocation loc, LOCube op, String inputAlias,
        List<String> operations, MultiMap<Integer, LogicalExpressionPlan> expressionPlans)
        throws FrontendException {


        LOForEach foreach = new LOForEach(plan);
        LOCogroup groupby = new LOCogroup(plan);
        LogicalPlan innerPlan = new LogicalPlan();
        LogicalRelationalOperator gen = new LOGenerate(innerPlan);


        injectForeachOperator(loc, op, foreach);

View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOCogroup

            }
        }
    }


    LOCogroup createGroupOp() {
        return new LOCogroup( plan );
    }

View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOCogroup

        return expectedSch;
    }


    private LOCogroup checkNumExpressionPlansForGroup(LogicalPlan lp, int numPlans) {
        Class<?> groupClass = org.apache.pig.newplan.logical.relational.LOCogroup.class;
        LOCogroup grp = (LOCogroup) NewLogicalPlanUtil.getRelOpFromPlan(lp, groupClass);


        for( int inp : grp.getExpressionPlans().keySet()){
            List<LogicalExpressionPlan> plans = grp.getExpressionPlans().get(inp);
            assertEquals("number of group-by plans", numPlans, plans.size());
        }


        return grp;
    }

View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOCogroup

      String query = "A = LOAD '" + INPUT_FILE + "' as (id, name, grade);" + 
                     "B = group A by id using 'collected';" +
                     "Store B into 'y';";
        LogicalPlan lp = Util.buildLp(pigServer, query );
        LOStore store = (LOStore)lp.getSinks().get(0);
        LOCogroup grp = (LOCogroup)lp.getPredecessors( store ).get(0);
        Assert.assertEquals( LOCogroup.GROUPTYPE.COLLECTED, grp.getGroupType() );
    }

View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOCogroup

        String query = "A = LOAD '" + INPUT_FILE + "' as (id, name, grade);" +
                       "B = group A all using 'regular';" +
                       "Store B into 'y';";
        LogicalPlan lp = Util.buildLp(pigServer, query );
        LOStore store = (LOStore)lp.getSinks().get(0);
        LOCogroup grp = (LOCogroup)lp.getPredecessors( store ).get(0);
        Assert.assertEquals(LOCogroup.GROUPTYPE.REGULAR, grp.getGroupType());
    }

View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOCogroup

        LogicalPlan lp;
        String query = "a = load 'myfile' as (name, age, gpa);" +
                   "b = group a by (name, age);";
        lp = buildPlan( query + "store b into 'output';");
        Operator store = lp.getSinks().get(0);
        LOCogroup cogroup = (LOCogroup) lp.getPredecessors(store).get(0);


        LogicalSchema actual = cogroup.getSchema();
        System.out.println( actual.toString( false ) );


        Assert.assertTrue(  actual.toString( false ).equals( "group:tuple(name:bytearray,age:bytearray),a:bag{:tuple(name:bytearray,age:bytearray,gpa:bytearray)}" ) );


        lp = buildPlan(query +

View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOCogroup

        String query = "a = load 'myfile' as (name:Chararray, age:Int, gpa:Float);" +
                       "b = group a by (name, age);" +
                       "store b into 'output';";
        lp = buildPlan( query );
        Operator store = lp.getSinks().get(0);
        LOCogroup cogroup = (LOCogroup) lp.getPredecessors(store).get(0);


        Schema.FieldSchema nameFs = new Schema.FieldSchema("name", DataType.CHARARRAY);
        Schema.FieldSchema ageFs = new Schema.FieldSchema("age", DataType.INTEGER);
        Schema.FieldSchema gpaFs = new Schema.FieldSchema("gpa", DataType.FLOAT);


        Schema groupSchema = new Schema(nameFs);
        groupSchema.add(ageFs);
        Schema.FieldSchema groupFs = new Schema.FieldSchema("group", groupSchema, DataType.TUPLE);


        Schema loadSchema = new Schema(nameFs);
        loadSchema.add(ageFs);
        loadSchema.add(gpaFs);


        Schema.FieldSchema bagFs = new Schema.FieldSchema("a", loadSchema, DataType.BAG);


        Schema cogroupExpectedSchema = new Schema(groupFs);
        cogroupExpectedSchema.add(bagFs);
        Assert.assertTrue(cogroup.getSchema().toString(false).equals("group:tuple(name:chararray,age:int),a:bag{:tuple(name:chararray,age:int,gpa:float)}"));
    }

View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOCogroup

                       "c = foreach b {d = order a by $1; generate flatten(d), MAX(a.age) as max_age;};" +
                       "store c into 'output';";
        LogicalPlan lp = buildPlan(query);
        Operator store = lp.getSinks().get(0);
        LOForEach foreach = (LOForEach)lp.getPredecessors(store).get(0);
        LOCogroup cogroup = (LOCogroup) lp.getPredecessors(foreach).get(0);
        String s = cogroup.getSchema().toString(false);
        Assert.assertTrue( s.equals("group:bytearray,a:bag{:tuple(name:bytearray,age:bytearray,gpa:bytearray)}"));
        s = foreach.getSchema().toString(false);
        Assert.assertTrue( s.equals("d::name:bytearray,d::age:bytearray,d::gpa:bytearray,max_age:double"));
    }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.