Examples of org.apache.pig.impl.logicalLayer.LOProject

Package org.apache.pig.impl.logicalLayer

Examples of org.apache.pig.impl.logicalLayer.LOProject

org.apache.pig.impl.logicalLayer.LOProject
LOProject is designed like a singly linked list; A few examples will illustrate the point about the linked list nature of the design; a = load 'input1' as (name, age); b = group a by name; foreach b generate a, a.name; The project operator occurs in two places in the above script: generate a(here) and a.name(here) In the first occurrence, we are trying to project the elements of the bag a; In order to retrieve the bag, we need to project the the second column ($1) or column number 1 (using the zero based index) from the input (the relation or bag b) In the second occurence, we are trying to project the first column ($0) or column number 0 from the bag a which in turn is the column number 1 in the relation b; As you can see, the nested structure or the singly linked list nature is clearly visible; Given that it's a singly linked list, the null pointer or the sentinel is marked explictly using the boolean variable mSentinel; The sentinel is marked true only when the input is a relational operator; This occurs when we create the innermost operator

                LogicalPlan p = new LogicalPlan();
                genPlans.add(p);
                flattens.add(false);
                List<Integer> toProject = new ArrayList<Integer>(1);
                toProject.add(i);
                LOProject proj = new LOProject(p, OperatorKey.genOpKey(scope),
                    lo, toProject);
                p.add(proj);
                Schema.FieldSchema fs = s.getField(i);
                if (fs.type != DataType.BYTEARRAY) {
                    if(determinedSchema == null || (fs.type != determinedSchema.getField(i).type)) {

View Full Code Here

                
                ArrayList<LogicalPlan> foreachInnerPlans = new ArrayList<LogicalPlan>();
                
                for(int i = 0; i < successorSchema.size(); ++i) {
                    LogicalPlan innerPlan = new LogicalPlan();
                    LOProject project = new LOProject(innerPlan, OperatorKey
                            .genOpKey(foreach.getOperatorKey().scope),
                            successor, i);
                    innerPlan.add(project);
                    foreachInnerPlans.add(innerPlan);
                    flattenList.add(false);

View Full Code Here

        } catch (VisitorException ve) {
            throw new OptimizerException();
        }
        if (projectFinder.getProjectSet()!=null && projectFinder.getProjectSet().size()==1)
        {
            LOProject project = projectFinder.getProjectSet().iterator().next();
            if (lp.getPredecessors(project)==null)
            {
                LogicalOperator pred = project;
                while (lp.getSuccessors(pred)!=null)
                {

View Full Code Here

        newForEachInnerPlan.add(generate);
        
        for (int i=0;i<foreach.getForEachPlans().size();i++) {
            LogicalPlan innerPlan = foreach.getForEachPlans().get(i);
            // Assume only one project is allowed in this level of foreach
            LOProject project = (LOProject)innerPlan.iterator().next();


            LOInnerLoad innerLoad = new LOInnerLoad(newForEachInnerPlan, 
                    newForEach, project.isStar()?-1:project.getCol());
            newForEachInnerPlan.add(innerLoad);
            newForEachInnerPlan.connect(innerLoad, generate);
            LogicalExpressionPlan expPlan = new LogicalExpressionPlan();
            expPlans.add(expPlan);
            ProjectExpression pe = new ProjectExpression(expPlan, i, -1, generate);

View Full Code Here

                            {
                                // Foreach plan is flattened, check if there is only one input for this foreach plan 
                                // and input schema for that input is not map, if so, it is a dummy flatten 
                                if (forEachPlan.getRoots().size()==1 && forEachPlan.getRoots().get(0) instanceof LOProject)
                                {
                                    LOProject loProj = (LOProject)forEachPlan.getRoots().get(0);
                                    if (loProj.getExpression().getSchema()!=null &&
                                            loProj.getExpression().getSchema().getField(loProj.getCol()).type!=DataType.BAG)
                                        nonflatten = true;
                                }
                            }
                            if (nonflatten && outputMapKeysInfo!=null && isSimpleProjectCast(forEachPlan))
                            {

View Full Code Here

            throw new OptimizerException();
        }
        boolean relayingMapKeys = false;
        if (projectFinder.getProjectSet()!=null && projectFinder.getProjectSet().size()==1)
        {
            LOProject project = projectFinder.getProjectSet().iterator().next();
            if (innerPlan.getPredecessors(project)==null)
            {
                relayingMapKeys = true;
                LogicalOperator pred = project;
                while (innerPlan.getSuccessors(pred)!=null)

View Full Code Here

                LogicalPlan p = new LogicalPlan();
                genPlans.add(p);
                flattens.add(false);
                List<Integer> toProject = new ArrayList<Integer>(1);
                toProject.add(i);
                LOProject proj = new LOProject(p, OperatorKey.genOpKey(scope),
                    lo, toProject);
                p.add(proj);
                Schema.FieldSchema fs = s.getField(i);
                if (fs.type != DataType.BYTEARRAY) {
                    if(determinedSchema == null || (fs.type != determinedSchema.getField(i).type)) {

View Full Code Here

        } catch (VisitorException ve) {
            throw new OptimizerException();
        }
        if (projectFinder.getProjectSet()!=null && projectFinder.getProjectSet().size()==1)
        {
            LOProject project = projectFinder.getProjectSet().iterator().next();
            if (lp.getPredecessors(project)==null)
            {
                LogicalOperator pred = project;
                while (lp.getSuccessors(pred)!=null)
                {

View Full Code Here

        newForEachInnerPlan.add(generate);
        
        for (int i=0;i<foreach.getForEachPlans().size();i++) {
            LogicalPlan innerPlan = foreach.getForEachPlans().get(i);
            // Assume only one project is allowed in this level of foreach
            LOProject project = (LOProject)innerPlan.iterator().next();


            LOInnerLoad innerLoad = new LOInnerLoad(newForEachInnerPlan, 
                    newForEach, project.isStar()?-1:project.getCol());
            newForEachInnerPlan.add(innerLoad);
            newForEachInnerPlan.connect(innerLoad, generate);
            LogicalExpressionPlan expPlan = new LogicalExpressionPlan();
            expPlans.add(expPlan);
            ProjectExpression pe = new ProjectExpression(expPlan, i, -1, generate);

View Full Code Here

                
                ArrayList<LogicalPlan> foreachInnerPlans = new ArrayList<LogicalPlan>();
                
                for(int i = 0; i < successorSchema.size(); ++i) {
                    LogicalPlan innerPlan = new LogicalPlan();
                    LOProject project = new LOProject(innerPlan, OperatorKey
                            .genOpKey(foreach.getOperatorKey().scope),
                            successor, i);
                    innerPlan.add(project);
                    foreachInnerPlans.add(innerPlan);
                    flattenList.add(false);

View Full Code Here

0 1

TOP

Related Classes of org.apache.pig.impl.logicalLayer.LOProject

org.apache.pig.impl.logicalLayer.optimizer.PruneColumns

org.apache.pig.impl.logicalLayer.optimizer.PushDownForeachFlatten

org.apache.pig.impl.logicalLayer.optimizer.PushUpFilter

org.apache.pig.impl.logicalLayer.optimizer.TypeCastInserter

org.apache.pig.impl.logicalLayer.schema.Schema

org.apache.pig.newplan.logical.ForeachInnerPlanVisitor

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.