Examples of org.apache.pig.experimental.logical.relational.LogicalPlan

Package org.apache.pig.experimental.logical.relational

Examples of org.apache.pig.experimental.logical.relational.LogicalPlan

org.apache.pig.experimental.logical.relational.LogicalPlan
LogicalPlan is the logical view of relational operations Pig will execute for a given script. Note that it contains only relational operations. All expressions will be contained in LogicalExpressionPlans inside each relational operator.

        
        // Plan here is 
        // Load (name, cuisines{t:(name)}) -> foreach gen name,cuisines 
        // -> filter name == 'joe2' --> filter name == 'joe' --> stor
        
        LogicalPlan plan = null;
        LOLoad load = null;
        LOForEach foreach = null;
        LOFilter filter = null;
        LOStore stor = null;
        
        plan = new LogicalPlan();
        
        LogicalSchema schema = new LogicalSchema();
        schema.addField(new LogicalSchema.LogicalFieldSchema("name", null, DataType.CHARARRAY));
        LogicalSchema bagSchema = new LogicalSchema();
        LogicalSchema bagTupleSchema = new LogicalSchema();
        bagTupleSchema.addField( new LogicalSchema.LogicalFieldSchema("name", null, DataType.CHARARRAY) );
        bagSchema.addField( new LogicalSchema.LogicalFieldSchema( "t", bagTupleSchema, DataType.TUPLE ) );
        schema.addField(new LogicalSchema.LogicalFieldSchema("cuisines", bagSchema, DataType.BAG));
        
        load = new LOLoad(null, schema, plan);
        load.setAlias("A");
        plan.add(load);
        
        foreach = new LOForEach(plan);
        
        LogicalPlan innerPlan = new LogicalPlan();
        LOInnerLoad innerLoad = new LOInnerLoad(innerPlan, foreach, 0);
        innerPlan.add(innerLoad);
        
        LOInnerLoad innerLoad2 = new LOInnerLoad(innerPlan, foreach, 1);
        innerPlan.add(innerLoad2);
        
        LogicalExpressionPlan namePrj = new LogicalExpressionPlan();        
        ProjectExpression prjName = new ProjectExpression(namePrj, DataType.CHARARRAY, 0, 0);
        namePrj.add(prjName);
        
        LogicalExpressionPlan cuisinesPrj = new LogicalExpressionPlan();
        ProjectExpression prjCuisines = new ProjectExpression(cuisinesPrj, DataType.BAG, 1, 0);
        cuisinesPrj.add(prjCuisines);
        
        List<LogicalExpressionPlan> expPlans = new ArrayList<LogicalExpressionPlan>();
        expPlans.add(namePrj);
        expPlans.add(cuisinesPrj);
        
        boolean flatten[] = new boolean[2];
        flatten[0] = false;
        flatten[1] = true;
        
        LOGenerate generate = new LOGenerate(innerPlan, expPlans, flatten);        
        innerPlan.add(generate);
        innerPlan.connect(innerLoad, generate);
        innerPlan.connect(innerLoad2, generate);
        
        foreach.setInnerPlan(innerPlan);
        foreach.setAlias("B");
        plan.add(foreach);

View Full Code Here

        
        // Plan here is 
        // Load (name, cuisines{t:(name)}) -> foreach gen name,cuisines 
        // -> filter $1 == 'joe2' --> filter $1 == 'joe' --> stor
        
        LogicalPlan plan = null;
        LOLoad load = null;
        LOForEach foreach = null;
        LOFilter filter = null;
        LOStore stor = null;
        
        plan = new LogicalPlan();
        
        LogicalSchema schema = new LogicalSchema();
        schema.addField(new LogicalSchema.LogicalFieldSchema("name", null, DataType.CHARARRAY));
        LogicalSchema bagSchema = new LogicalSchema();
        LogicalSchema bagTupleSchema = new LogicalSchema();
        bagTupleSchema.addField( new LogicalSchema.LogicalFieldSchema("name", null, DataType.CHARARRAY) );
        bagSchema.addField( new LogicalSchema.LogicalFieldSchema( "t", bagTupleSchema, DataType.TUPLE ) );
        schema.addField(new LogicalSchema.LogicalFieldSchema("cuisines", bagSchema, DataType.BAG));
        
        load = new LOLoad(null, schema, plan);
        load.setAlias("A");
        plan.add(load);
        
        foreach = new LOForEach(plan);
        
        LogicalPlan innerPlan = new LogicalPlan();
        LOInnerLoad innerLoad = new LOInnerLoad(innerPlan, foreach, 0);
        innerPlan.add(innerLoad);
        
        LOInnerLoad innerLoad2 = new LOInnerLoad(innerPlan, foreach, 1);
        innerPlan.add(innerLoad2);
        
        LogicalExpressionPlan namePrj = new LogicalExpressionPlan();        
        ProjectExpression prjName = new ProjectExpression(namePrj, DataType.CHARARRAY, 0, 0);
        namePrj.add(prjName);
        
        LogicalExpressionPlan cuisinesPrj = new LogicalExpressionPlan();
        ProjectExpression prjCuisines = new ProjectExpression(cuisinesPrj, DataType.BAG, 1, 0);
        cuisinesPrj.add(prjCuisines);
        
        List<LogicalExpressionPlan> expPlans = new ArrayList<LogicalExpressionPlan>();
        expPlans.add(namePrj);
        expPlans.add(cuisinesPrj);
        
        boolean flatten[] = new boolean[2];
        flatten[0] = false;
        flatten[1] = true;
        
        LOGenerate generate = new LOGenerate(innerPlan, expPlans, flatten);        
        innerPlan.add(generate);
        innerPlan.connect(innerLoad, generate);
        innerPlan.connect(innerLoad2, generate);
        
        foreach.setInnerPlan(innerPlan);
        foreach.setAlias("B");
        plan.add(foreach);

View Full Code Here

    public void testNotPossibleFilter() throws Exception {
        // Plan here is 
        // Load (name, cuisines{t:(name)}) -> foreach gen name,flatten(cuisines) 
        // -> filter $1 == 'joe' --> stor


        LogicalPlan plan = null;
        LOLoad load = null;
        LOForEach foreach = null;
        LOFilter filter = null;
        LOStore stor = null;


        plan = new LogicalPlan();


        LogicalSchema schema = new LogicalSchema();
        schema.addField(new LogicalSchema.LogicalFieldSchema("name", null, DataType.CHARARRAY));
        LogicalSchema bagSchema = new LogicalSchema();
        LogicalSchema bagTupleSchema = new LogicalSchema();
        bagTupleSchema.addField( new LogicalSchema.LogicalFieldSchema("name", null, DataType.CHARARRAY) );
        bagSchema.addField( new LogicalSchema.LogicalFieldSchema( "t", bagTupleSchema, DataType.TUPLE ) );
        schema.addField(new LogicalSchema.LogicalFieldSchema("cuisines", bagSchema, DataType.BAG));
        
        load = new LOLoad(null, schema, plan);
        load.setAlias("A");
        plan.add(load);
        
        foreach = new LOForEach(plan);
        
        LogicalPlan innerPlan = new LogicalPlan();
        LOInnerLoad innerLoad = new LOInnerLoad(innerPlan, foreach, 0);
        innerPlan.add(innerLoad);
        
        LOInnerLoad innerLoad2 = new LOInnerLoad(innerPlan, foreach, 1);
        innerPlan.add(innerLoad2);
        
        LogicalExpressionPlan namePrj = new LogicalExpressionPlan();        
        ProjectExpression prjName = new ProjectExpression(namePrj, DataType.CHARARRAY, 0, 0);
        namePrj.add(prjName);
        
        LogicalExpressionPlan cuisinesPrj = new LogicalExpressionPlan();
        ProjectExpression prjCuisines = new ProjectExpression(cuisinesPrj, DataType.BAG, 1, 0);
        cuisinesPrj.add(prjCuisines);
        
        List<LogicalExpressionPlan> expPlans = new ArrayList<LogicalExpressionPlan>();
        expPlans.add(namePrj);
        expPlans.add(cuisinesPrj);
        
        boolean flatten[] = new boolean[2];
        flatten[0] = false;
        flatten[1] = true;
        
        LOGenerate generate = new LOGenerate(innerPlan, expPlans, flatten);        
        innerPlan.add(generate);
        innerPlan.connect(innerLoad, generate);
        innerPlan.connect(innerLoad2, generate);
        
        foreach.setInnerPlan(innerPlan);
        foreach.setAlias("B");
        plan.add(foreach);

View Full Code Here

        
        // Plan here is 
        // Load (name, cuisines{t:(name)}) -> foreach gen name,cuisines 
        // -> filter name == 'joe' --> stor
        
        LogicalPlan plan = null;
        LOLoad load = null;
        LOForEach foreach = null;
        LOFilter filter = null;
        LOStore stor = null;
        
        plan = new LogicalPlan();
        
        LogicalSchema schema = new LogicalSchema();
        schema.addField(new LogicalSchema.LogicalFieldSchema("name", null, DataType.CHARARRAY));
        LogicalSchema bagSchema = new LogicalSchema();
        LogicalSchema bagTupleSchema = new LogicalSchema();
        bagTupleSchema.addField( new LogicalSchema.LogicalFieldSchema("name", null, DataType.CHARARRAY) );
        bagSchema.addField( new LogicalSchema.LogicalFieldSchema( "t", bagTupleSchema, DataType.TUPLE ) );
        schema.addField(new LogicalSchema.LogicalFieldSchema("cuisines", bagSchema, DataType.BAG));
        
        load = new LOLoad(null, schema, plan);
        load.setAlias("A");
        plan.add(load);
        
        foreach = new LOForEach(plan);
        
        LogicalPlan innerPlan = new LogicalPlan();
        LOInnerLoad innerLoad = new LOInnerLoad(innerPlan, foreach, 0);
        innerPlan.add(innerLoad);
        
        LOInnerLoad innerLoad2 = new LOInnerLoad(innerPlan, foreach, 1);
        innerPlan.add(innerLoad2);
        
        LogicalExpressionPlan namePrj = new LogicalExpressionPlan();        
        ProjectExpression prjName = new ProjectExpression(namePrj, DataType.CHARARRAY, 0, 0);
        namePrj.add(prjName);
        
        LogicalExpressionPlan cuisinesPrj = new LogicalExpressionPlan();
        ProjectExpression prjCuisines = new ProjectExpression(cuisinesPrj, DataType.BAG, 1, 0);
        cuisinesPrj.add(prjCuisines);
        
        List<LogicalExpressionPlan> expPlans = new ArrayList<LogicalExpressionPlan>();
        expPlans.add(namePrj);
        expPlans.add(cuisinesPrj);
        
        boolean flatten[] = new boolean[2];
        flatten[0] = false;
        flatten[1] = false;
        
        LOGenerate generate = new LOGenerate(innerPlan, expPlans, flatten);        
        innerPlan.add(generate);
        innerPlan.connect(innerLoad, generate);
        innerPlan.connect(innerLoad2, generate);
        
        foreach.setInnerPlan(innerPlan);
        foreach.setAlias("B");
        plan.add(foreach);

View Full Code Here

                }                      
                
                // Loader does not support column pruning, insert foreach      
                if (columnPrune) {
                    if (response==null || !response.getRequiredFieldResponse()) {
                        LogicalPlan p = (LogicalPlan)load.getPlan();                        
                        Operator next = p.getSuccessors(load).get(0); 
                        // if there is already a LOForEach after load, we don't need to 
                        // add another LOForEach
                        if (next instanceof LOForEach) {
                            return;
                        }
                        
                        LOForEach foreach = new LOForEach(load.getPlan());
                        
                        // add foreach to the base plan                       
                        p.add(foreach);
                                       
                        Pair<Integer,Integer> disconnectedPos = p.disconnect(load, next);
                        p.connect(load, disconnectedPos.first.intValue(), foreach, 0 );
                        p.connect(foreach, 0, next, disconnectedPos.second.intValue());
                        
                        // add foreach to the subplan
                        subPlan.add(foreach);
                        
                        LogicalPlan innerPlan = new LogicalPlan();
                        foreach.setInnerPlan(innerPlan);
                        
                        // build foreach inner plan
                        List<LogicalExpressionPlan> exps = new ArrayList<LogicalExpressionPlan>();              
                        LOGenerate gen = new LOGenerate(innerPlan, exps, new boolean[requiredFields.getFields().size()]);
                        innerPlan.add(gen);
                        
                        for (int i=0; i<requiredFields.getFields().size(); i++) {
                            LoadPushDown.RequiredField rf = requiredFields.getFields().get(i);
                            LOInnerLoad innerLoad = new LOInnerLoad(innerPlan, foreach, rf.getIndex());
                            innerLoad.getProjection().setUid(foreach);                    
                            innerPlan.add(innerLoad);          
                            innerPlan.connect(innerLoad, gen);
                            
                            LogicalExpressionPlan exp = new LogicalExpressionPlan();
                            ProjectExpression prj = new ProjectExpression(exp, rf.getType(), i, 0);
                            prj.setUid(gen);
                            exp.add(prj);

View Full Code Here

                // get column numbers from input uids
                Set<Long> input = (Set<Long>)foreach.getAnnotation(ColumnPruneHelper.INPUTUIDS);
                LogicalRelationalOperator op = (LogicalRelationalOperator)foreach.getPlan().getPredecessors(foreach).get(0);
                Set<Integer> cols = columnHelper.getColumns(op.getSchema(), input);
                
                LogicalPlan innerPlan = foreach.getInnerPlan();
                LOGenerate gen = (LOGenerate)innerPlan.getSinks().get(0);
                            
                // clean up the predecessors of LOGenerate
                List<Operator> ll = innerPlan.getPredecessors(gen);
                List<Operator> toRemove = new ArrayList<Operator>();
                for(int i=0; i<ll.size(); i++) {
                    
                    // if the LOInnerLoads for this subplan are all in the column set, 
                    // no change required, keep going

View Full Code Here

                return true;
            }             
            
            // remove all the operators starting from an operator
            protected void removeSubTree(LogicalRelationalOperator op) throws IOException {
                LogicalPlan p = (LogicalPlan)op.getPlan();
                List<Operator> ll = p.getPredecessors(op);
                if (ll != null) {
                    for(Operator pred: ll) {          
                        removeSubTree((LogicalRelationalOperator)pred);
                    }
                }
                        
                if (p.getSuccessors(op) != null) {
                    Operator[] succs = p.getSuccessors(op).toArray(new Operator[0]);
                    
                    for(Operator s: succs) {
                        p.disconnect(op, s);
                    }
                }
                
                p.remove(op);
            }

View Full Code Here


    @Override
    protected OperatorPlan buildPattern() {
        // the pattern that this rule looks for
        // is foreach -> filter
        LogicalPlan plan = new LogicalPlan();
        LogicalRelationalOperator foreach = new LOForEach(plan);
        LogicalRelationalOperator filter = new LOFilter(plan);
        
        plan.add(foreach);
        plan.add(filter);
        plan.connect(foreach, filter);


        return plan;
    }

View Full Code Here


    @Override
    protected OperatorPlan buildPattern() {        
        // the pattern that this rule looks for
        // is filter
        LogicalPlan plan = new LogicalPlan();      
        LogicalRelationalOperator op2 = new LOFilter(plan);
        plan.add(op2);
        
        return plan;
    }

View Full Code Here

        
        @SuppressWarnings("unchecked")
        public void visitLOForEach(LOForEach foreach) throws IOException {
            Set<Long> output = setOutputUids(foreach);
            
            LogicalPlan innerPlan = foreach.getInnerPlan();
            LOGenerate gen = (LOGenerate)innerPlan.getSinks().get(0);
            gen.annotate(OUTPUTUIDS, output);
            
            ColumnDependencyVisitor v = new ColumnDependencyVisitor(innerPlan);            
            v.visit();
            
            Set<Long> input = new HashSet<Long>();
            List<Operator> sources = innerPlan.getSources();
            for(Operator s: sources) {
                Set<Long> in = (Set<Long>)s.getAnnotation(INPUTUIDS);
                if (in != null) {
                    input.addAll(in);
                }

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.pig.experimental.logical.relational.LogicalPlan

org.apache.pig.experimental.logical.rules.AddForEach$AddForEachTransformer

org.apache.pig.experimental.logical.rules.ColumnMapKeyPrune$ColumnMapKeyPruneTransformer$ColumnPruneVisitor

org.apache.pig.experimental.logical.rules.ColumnPruneHelper$ColumnDependencyVisitor

org.apache.pig.experimental.logical.rules.FilterAboveForeach

org.apache.pig.experimental.logical.rules.MergeFilter

org.apache.pig.experimental.logical.rules.PushUpFilter

org.apache.pig.experimental.logical.rules.SplitFilter

org.apache.pig.test.TestExperimentalColumnPrune

org.apache.pig.test.TestExperimentalFilterAboveForeach

org.apache.pig.test.TestExperimentalFilterRule

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.