Examples of org.apache.pig.experimental.logical.relational.LogicalSchema

Package org.apache.pig.experimental.logical.relational

Examples of org.apache.pig.experimental.logical.relational.LogicalSchema

org.apache.pig.experimental.logical.relational.LogicalSchema
Schema, from a logical perspective.

            // build foreach inner plan
            List<LogicalExpressionPlan> exps = new ArrayList<LogicalExpressionPlan>();              
            LOGenerate gen = new LOGenerate(innerPlan, exps, new boolean[cols.size()]);
            innerPlan.add(gen);
            
            LogicalSchema schema = op.getSchema();
            for (int i=0, j=0; i<schema.size(); i++) {   
                if (!cols.contains(i)) {
                    continue;
                }
                               
                LOInnerLoad innerLoad = new LOInnerLoad(innerPlan, foreach, i);
                innerLoad.getProjection().setUid(foreach);                    
                innerPlan.add(innerLoad);          
                innerPlan.connect(innerLoad, gen);
                
                LogicalExpressionPlan exp = new LogicalExpressionPlan();
                ProjectExpression prj = new ProjectExpression(exp, schema.getField(i).type, j++, 0);
                prj.setUid(gen);
                exp.add(prj);
                exps.add(exp);                
            }

View Full Code Here

    
    @Override
    public void setUid(LogicalRelationalOperator currentOp) throws IOException {
        LogicalRelationalOperator referent = findReferent(currentOp);
        
        LogicalSchema schema = referent.getSchema();
        if (schema != null && !isProjectStar()) {
            uid = schema.getField(col).uid;
        } else {
            // If the schema of referent is null, or this is to project the whole tuple,
            // we kindof create a uid so we 
            // can track it in remaining plan
            uid = getNextUid();

View Full Code Here

        public void visitLOStore(LOStore store) throws IOException {
            Set<Long> output = setOutputUids(store);            
            
            if (output.isEmpty()) {
                // to deal with load-store-load-store case
                LogicalSchema s = store.getSchema();
                if (s == null) {
                    throw new SchemaNotDefinedException("Schema for " + store.getName() + " is not defined.");
                }
                                
                for(int i=0; i<s.size(); i++) {
                    output.add(s.getField(i).uid);
                }                                                
            }        
            
            // for store, input uids are same as output uids
            store.annotate(INPUTUIDS, output);

View Full Code Here

                 if (s instanceof ProjectExpression) {
                     int inputNum = ((ProjectExpression)s).getInputNum();       
                     int colNum = ((ProjectExpression)s).getColNum();       
                     LogicalRelationalOperator pred = (LogicalRelationalOperator)gen.getPlan().getPredecessors(gen).get(inputNum);
                     
                     LogicalSchema predSchema = pred.getSchema();
                     if (predSchema == null) {
                         throw new SchemaNotDefinedException("Schema for " + pred.getName() + " is not defined.");
                     }
                     
                     if (predSchema.getField(colNum).type == DataType.BAG) {
                         long fuid = predSchema.getField(colNum).uid;
                         LogicalSchema fschema = predSchema.getField(colNum).schema;
                         if (input.contains(fuid)) {
                             continue;
                         }
                         
                         if (fschema == null) {
                             input.add(fuid);
                         }


                         boolean found = false;
                         for(long uid: input) {
                             if (fschema.findField(uid) != -1) {
                                 found = true;
                                 break;
                             }
                         }
                         
                         // if the input uids doesn't contain any field from this bag, then add the first field
                         if (!found) {
                             input.add(fschema.getField(0).uid);
                         }
                     }
                 }                                    
             }

View Full Code Here

                    if (!((ProjectExpression)op).isProjectStar()) {
                        long uid = ((ProjectExpression)op).getUid();
                        uids.add(uid);
                    } else {
                        LogicalRelationalOperator ref = ((ProjectExpression)op).findReferent(currentOp);
                        LogicalSchema s = ref.getSchema();
                        if (s == null) {
                            throw new SchemaNotDefinedException("Schema not defined for " + ref.getAlias());
                        }
                        for(LogicalFieldSchema f: s.getFields()) {
                            uids.add(f.uid);
                        }
                    }
                }
            }

View Full Code Here

        private Set<Long> setOutputUids(LogicalRelationalOperator op) throws IOException {
            
            List<Operator> ll = plan.getSuccessors(op);
            Set<Long> uids = new HashSet<Long>();
            
            LogicalSchema s = op.getSchema();
            if (s == null) {
                throw new SchemaNotDefinedException("Schema for " + op.getName() + " is not defined.");
            }
                            
            if (ll != null) {
                // if this is not sink, the output uids are union of input uids of its successors
                for(Operator succ: ll) {
                    Set<Long> inputUids = (Set<Long>)succ.getAnnotation(INPUTUIDS);
                    if (inputUids != null) {
                        Iterator<Long> iter = inputUids.iterator();
                        while(iter.hasNext()) {
                            long uid = iter.next();
                            
                            if (s.findField(uid) != -1) {
                                uids.add(uid);
                            }
                        }
                    }
                }
            } else {
                // if  it's leaf, set to its schema                
                for(int i=0; i<s.size(); i++) {
                    uids.add(s.getField(i).uid);
                }                                
            } 
            
            op.annotate(OUTPUTUIDS, uids);
            return uids;

View Full Code Here

            }
        }
        
        // check if a relational operator contains all of the specified uids
        private boolean hasAll(LogicalRelationalOperator op, Set<Long> uids) {
            LogicalSchema schema = op.getSchema();
            for(long uid: uids) {
                if (schema.findField(uid) == -1) {
                    return false;
                }
            }
            
            return true;

View Full Code Here

        // store D into 'whatever';
        
        // A = load
        LogicalPlan lp = new LogicalPlan();
        {
          LogicalSchema aschema = new LogicalSchema();
          aschema.addField(new LogicalSchema.LogicalFieldSchema(
              "x", null, DataType.INTEGER));
          aschema.addField(new LogicalSchema.LogicalFieldSchema(
              "y", null, DataType.INTEGER));
          aschema.getField(0).uid = 1;
          aschema.getField(1).uid = 2;
          LOLoad A = new LOLoad(new FileSpec("bla", new FuncSpec("PigStorage", "\t")), aschema, lp);
          lp.add(A);
          
          // B = load
          LogicalSchema bschema = new LogicalSchema();
          bschema.addField(new LogicalSchema.LogicalFieldSchema(
              "a", null, DataType.INTEGER));
          bschema.addField(new LogicalSchema.LogicalFieldSchema(
              "b", null, DataType.INTEGER));
          bschema.getField(0).uid = 3;
          bschema.getField(1).uid = 4;
          LOLoad B = new LOLoad(null, bschema, lp);
          lp.add(B);
          
          // C = join
          LogicalSchema cschema = new LogicalSchema();
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "x", null, DataType.INTEGER));
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "y", null, DataType.INTEGER));
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "a", null, DataType.INTEGER));
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "b", null, DataType.INTEGER));
          cschema.getField(0).uid = 1;
          cschema.getField(1).uid = 2;
          cschema.getField(2).uid = 3;
          cschema.getField(3).uid = 4;
          LogicalExpressionPlan aprojplan = new LogicalExpressionPlan();
          ProjectExpression x = new ProjectExpression(aprojplan, DataType.INTEGER, 0, 0);
          x.neverUseForRealSetUid(1);
          LogicalExpressionPlan bprojplan = new LogicalExpressionPlan();
          ProjectExpression y = new ProjectExpression(bprojplan, DataType.INTEGER, 1, 0);
          y.neverUseForRealSetUid(3);
          MultiMap<Integer, LogicalExpressionPlan> mm = 
              new MultiMap<Integer, LogicalExpressionPlan>();
          mm.put(0, aprojplan);
          mm.put(1, bprojplan);
          LOJoin C = new LOJoin(lp, mm, JOINTYPE.HASH, new boolean[] {true, true});
          C.neverUseForRealSetSchema(cschema);
          lp.add(C);
          lp.connect(A, C);
          lp.connect(B, C);
        
          // D = filter
          LogicalExpressionPlan filterPlan = new LogicalExpressionPlan();
          ProjectExpression fx = new ProjectExpression(filterPlan, DataType.INTEGER, 0, 0);
          fx.neverUseForRealSetUid(1);
          ConstantExpression fc0 = new ConstantExpression(filterPlan, DataType.INTEGER, new Integer(0));
          EqualExpression eq1 = new EqualExpression(filterPlan, fx, fc0);
          ProjectExpression fanotherx = new ProjectExpression(filterPlan, DataType.INTEGER, 0, 0);
          fanotherx.neverUseForRealSetUid(1);
          ProjectExpression fa = new ProjectExpression(filterPlan, DataType.INTEGER, 0, 2);
          fa.neverUseForRealSetUid(3);
          EqualExpression eq2 = new EqualExpression(filterPlan, fanotherx, fa);
          AndExpression and1 = new AndExpression(filterPlan, eq1, eq2);
          ProjectExpression fb = new ProjectExpression(filterPlan, DataType.INTEGER, 0, 3);
          fb.neverUseForRealSetUid(4);
          ConstantExpression fc1 = new ConstantExpression(filterPlan, DataType.INTEGER, new Integer(1));
          EqualExpression eq3 = new EqualExpression(filterPlan, fb, fc1);
          AndExpression and2 = new AndExpression(filterPlan, and1, eq3);
          ProjectExpression fanotherb = new ProjectExpression(filterPlan, DataType.INTEGER, 0, 3);
          fanotherb.neverUseForRealSetUid(4);
          ProjectExpression fy = new ProjectExpression(filterPlan, DataType.INTEGER, 0, 1);
          fy.neverUseForRealSetUid(2);
          EqualExpression eq4 = new EqualExpression(filterPlan, fy, fanotherb);
          new AndExpression(filterPlan, and2, eq4);
        
          LOFilter D = new LOFilter(lp, filterPlan);
          D.neverUseForRealSetSchema(cschema);
          // Connect D to B, since the transform has happened.
          lp.add(D);
          lp.connect(C, D);
        }
        
        LogicalPlanOptimizer optimizer = new LogicalPlanOptimizer(lp, 500);
        optimizer.optimize();
        
        LogicalPlan expected = new LogicalPlan();
        {
            // A = load
          LogicalSchema aschema = new LogicalSchema();
          aschema.addField(new LogicalSchema.LogicalFieldSchema(
              "x", null, DataType.INTEGER));
          aschema.addField(new LogicalSchema.LogicalFieldSchema(
              "y", null, DataType.INTEGER));
          aschema.getField(0).uid = 1;
          aschema.getField(1).uid = 2;
          LOLoad A = new LOLoad(new FileSpec("bla", new FuncSpec("PigStorage", "\t")), aschema, expected);
          expected.add(A);
          
          // DA = filter
          LogicalExpressionPlan DAfilterPlan = new LogicalExpressionPlan();
          ProjectExpression fx = new ProjectExpression(DAfilterPlan, DataType.INTEGER, 0, 0);
          fx.neverUseForRealSetUid(1);
          ConstantExpression fc0 = new ConstantExpression(DAfilterPlan, DataType.INTEGER, new Integer(0));
          new EqualExpression(DAfilterPlan, fx, fc0);
          
          LOFilter DA = new LOFilter(expected, DAfilterPlan);
          DA.neverUseForRealSetSchema(aschema);
          expected.add(DA);
          expected.connect(A, DA);
          
          // B = load
          LogicalSchema bschema = new LogicalSchema();
          bschema.addField(new LogicalSchema.LogicalFieldSchema(
              "a", null, DataType.INTEGER));
          bschema.addField(new LogicalSchema.LogicalFieldSchema(
              "b", null, DataType.INTEGER));
          bschema.getField(0).uid = 3;
          bschema.getField(1).uid = 4;
          LOLoad B = new LOLoad(null, bschema, expected);
          expected.add(B);
          
          // DB = filter
          LogicalExpressionPlan DBfilterPlan = new LogicalExpressionPlan();
          ProjectExpression fb = new ProjectExpression(DBfilterPlan, DataType.INTEGER, 0, 1);
          fb.neverUseForRealSetUid(4);
          ConstantExpression fc1 = new ConstantExpression(DBfilterPlan, DataType.INTEGER, new Integer(1));
          new EqualExpression(DBfilterPlan, fb, fc1);
          
          LOFilter DB = new LOFilter(expected, DBfilterPlan);
          DB.neverUseForRealSetSchema(bschema);
          expected.add(DB);
          expected.connect(B, DB);
          
          // C = join
          LogicalSchema cschema = new LogicalSchema();
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "x", null, DataType.INTEGER));
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "y", null, DataType.INTEGER));
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "a", null, DataType.INTEGER));
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "b", null, DataType.INTEGER));
          cschema.getField(0).uid = 1;
          cschema.getField(1).uid = 2;
          cschema.getField(2).uid = 3;
          cschema.getField(3).uid = 4;
          LogicalExpressionPlan aprojplan = new LogicalExpressionPlan();
          ProjectExpression x = new ProjectExpression(aprojplan, DataType.INTEGER, 0, 0);
          x.neverUseForRealSetUid(1);
          LogicalExpressionPlan bprojplan = new LogicalExpressionPlan();
          ProjectExpression y = new ProjectExpression(bprojplan, DataType.INTEGER, 1, 0);

View Full Code Here

0 1 2 3 4 5

TOP

Related Classes of org.apache.pig.experimental.logical.relational.LogicalSchema

org.apache.pig.experimental.logical.expression.ProjectExpression

org.apache.pig.experimental.logical.LogicalPlanMigrationVistor

org.apache.pig.experimental.logical.optimizer.ProjectionPatcher$ProjectionRewriter

org.apache.pig.experimental.logical.optimizer.UidStamper

org.apache.pig.experimental.logical.rules.AddForEach$AddForEachTransformer

org.apache.pig.experimental.logical.rules.ColumnMapKeyPrune$ColumnMapKeyPruneTransformer$ColumnPruneVisitor

org.apache.pig.experimental.logical.rules.ColumnPruneHelper

org.apache.pig.experimental.logical.rules.ColumnPruneHelper$ColumnDependencyVisitor

org.apache.pig.experimental.logical.rules.MapKeysPruneHelper

org.apache.pig.experimental.logical.rules.PushUpFilter$PushUpFilterTransformer

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.