Package org.apache.pig.experimental.logical.expression

Examples of org.apache.pig.experimental.logical.expression.LogicalExpressionPlan


             Iterator<Long> iter = output.iterator();
             while(iter.hasNext()) {
                 long uid = iter.next();
                 boolean found = false;
                 for(int i=0; i<ll.size(); i++) {
                     LogicalExpressionPlan exp = ll.get(i);
                     LogicalExpression op = (LogicalExpression)exp.getSources().get(0);
                    
                     if (op.getUid() == uid) {
                         collectUids(gen, exp, input);                        
                         found = true;
                       
                     } else if (op instanceof ProjectExpression && ((ProjectExpression)op).isProjectStar()) {
                         int inputNum = ((ProjectExpression)op).getInputNum();                             
                         LogicalRelationalOperator pred = (LogicalRelationalOperator)gen.getPlan().getPredecessors(gen).get(inputNum);
                        
                         if (pred.getSchema() == null) {
                             throw new SchemaNotDefinedException("Schema for " + pred.getName() + " is not defined.");
                         }
                         for(LogicalFieldSchema f: pred.getSchema().getFields()) {
                             if (f.uid == uid) {
                                 input.add(uid);
                                 found = true;
                             }
                         }
                        
                     } else if (gen.getFlattenFlags()[i]) {
                         // if uid equal to the expression, get all uids of original projections
                         List<Operator> ss = exp.getSinks();
                         for(Operator s: ss) {
                             if (s instanceof ProjectExpression) {
                                 int inputNum = ((ProjectExpression)s).getInputNum();                             
                                 LogicalRelationalOperator pred = (LogicalRelationalOperator)gen.getPlan().getPredecessors(gen).get(inputNum);
                                
                                 if (pred.getSchema() == null) {
                                     throw new SchemaNotDefinedException("Schema for " + pred.getName() + " is not defined.");
                                 }
                                 if (pred.getSchema().findField(uid) != -1) {                                   
                                     input.add(uid);                                   
                                     found = true;
                                 }
                             }
                         }
                     }
                    
                     if (found) {
                         break;
                     }
                 }
                
                 if (!found) {                   
                     throw new IOException("uid " + uid +" is not in the schema of LOForEach");                   
                 }
             }
             
             // for the flatten bag, we need to make sure at least one field is in the input
             for(int i=0; i<ll.size(); i++) {
                 if (!gen.getFlattenFlags()[i]) {
                     continue;
                 }
                
                 LogicalExpressionPlan exp = ll.get(i);
                 Operator s = exp.getSinks().get(0);
                
                 if (s instanceof ProjectExpression) {
                     int inputNum = ((ProjectExpression)s).getInputNum();      
                     int colNum = ((ProjectExpression)s).getColNum();      
                     LogicalRelationalOperator pred = (LogicalRelationalOperator)gen.getPlan().getPredecessors(gen).get(inputNum);
View Full Code Here


                LOInnerLoad innerLoad = new LOInnerLoad(innerPlan, foreach, i);
                innerLoad.getProjection().setUid(foreach);                   
                innerPlan.add(innerLoad);         
                innerPlan.connect(innerLoad, gen);
               
                LogicalExpressionPlan exp = new LogicalExpressionPlan();
                ProjectExpression prj = new ProjectExpression(exp, schema.getField(i).type, j++, 0);
                prj.setUid(gen);
                exp.add(prj);
                exps.add(exp);               
            }               
          
        }
View Full Code Here

        }
       
        // combine the condition of two filters. The condition of second filter
        // is added into the condition of first filter with an AND operator.
        private void combineFilterCond(LOFilter f1, LOFilter f2) throws IOException {
            LogicalExpressionPlan p1 = f1.getFilterPlan();
            LogicalExpressionPlan p2 = f2.getFilterPlan();
            LogicalExpressionPlan andPlan = new LogicalExpressionPlan();
           
            // add existing operators         
            Iterator<Operator> iter = p1.getOperators();
            while(iter.hasNext()) {
                andPlan.add(iter.next());
            }
           
            iter = p2.getOperators();
            while(iter.hasNext()) {
                andPlan.add(iter.next());
            }
           
            // add all connections
            iter = p1.getOperators();
            while(iter.hasNext()) {
                Operator n = iter.next();
                List<Operator> l = p1.getPredecessors(n);
                if (l != null) {
                    for(Operator op: l) {
                        andPlan.connect(op, n);
                    }
                }
            }
           
            iter = p2.getOperators();
            while(iter.hasNext()) {
                Operator n = iter.next();
                List<Operator> l = p2.getPredecessors(n);
                if (l != null) {
                    for(Operator op: l) {
                        andPlan.connect(op, n);
                    }
                }
            }         
           
            // create an AND
View Full Code Here

            }
          
            Operator next = matched.getSinks().get(0);
            while(next != null && next instanceof LOFilter) {
                LOFilter filter = (LOFilter)next;           
                LogicalExpressionPlan filterPlan = filter.getFilterPlan();
               
                // collect all uids used in the filter plan
                Set<Long> uids = new HashSet<Long>();
                Iterator<Operator> iter = filterPlan.getOperators();
                while(iter.hasNext()) {
                    Operator op = iter.next();
                    if (op instanceof ProjectExpression) {
                        long uid = ((ProjectExpression)op).getUid();
                        uids.add(uid);
View Full Code Here

            Operator next = matched.getSinks().get(0);
            while(next != null && next instanceof LOFilter) {
                LOFilter filter = (LOFilter)next;               
                subPlan.add(filter);
               
                LogicalExpressionPlan filterPlan = filter.getFilterPlan();
               
                // collect all uids used in the filter plan
                Set<Long> uids = new HashSet<Long>();
                Iterator<Operator> iter = filterPlan.getOperators();
                while(iter.hasNext()) {
                    Operator op = iter.next();
                    if (op instanceof ProjectExpression) {
                        long uid = ((ProjectExpression)op).getUid();
                        uids.add(uid);
View Full Code Here

                            LOInnerLoad innerLoad = new LOInnerLoad(innerPlan, foreach, rf.getIndex());
                            innerLoad.getProjection().setUid(foreach);                   
                            innerPlan.add(innerLoad);         
                            innerPlan.connect(innerLoad, gen);
                           
                            LogicalExpressionPlan exp = new LogicalExpressionPlan();
                            ProjectExpression prj = new ProjectExpression(exp, rf.getType(), i, 0);
                            prj.setUid(gen);
                            exp.add(prj);
                            exps.add(exp);
                        }               
                      
                    } else {
                        // columns are pruned, reset schema for LOLoader
View Full Code Here

                    // clean up and adjust expression plans
                    Iterator<LogicalExpressionPlan> iter = gen.getOutputPlans().iterator();
                    int j=-1;
                    while(iter.hasNext()) {
                        j++;
                        LogicalExpressionPlan exp = iter.next();
                        List<Operator> sinks = exp.getSinks();
                        for(Operator s: sinks) {
                            if (s instanceof ProjectExpression) {
                                int inputNo = ((ProjectExpression)s).getInputNum();
                                if (inputNo + toRemove.size() == i) {
                                    // if this expression has this input that is to be removed,
View Full Code Here

TOP

Related Classes of org.apache.pig.experimental.logical.expression.LogicalExpressionPlan

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.