Package org.apache.pig.newplan.logical.relational

Examples of org.apache.pig.newplan.logical.relational.LogicalRelationalOperator


        @Override
        public void visit(ScalarExpression scalarExp) throws FrontendException{
            //track lineage to input load function
           
           
            LogicalRelationalOperator inputRel = (LogicalRelationalOperator)scalarExp.getAttachedLogicalOperator();
            LogicalPlan relPlan = (LogicalPlan) inputRel.getPlan();
            List<Operator> softPreds = relPlan.getSoftLinkPredecessors(inputRel);
           
            //1st argument is the column number in input, 2nd arg is the input file name
            Integer inputColNum = (Integer)((ConstantExpression) scalarExp.getArguments().get(0)).getValue();
            String inputFile = (String)((ConstantExpression) scalarExp.getArguments().get(1)).getValue();
View Full Code Here


        for(int i=0; i<loGenPreds.size(); i++){
            oldPos2Rel.put(i, (LogicalRelationalOperator) loGenPreds.get(i));
        }
       
        //get schema of predecessor, project-star expansion needs a schema
        LogicalRelationalOperator pred =
            (LogicalRelationalOperator) foreach.getPlan().getPredecessors(foreach).get(0);
        LogicalSchema inpSch = pred.getSchema();
        //store mapping between the projection in inner plans of
        // of LOGenerate to the input relation object
        Map<ProjectExpression, LogicalRelationalOperator> proj2InpRel =
            new HashMap<ProjectExpression, LogicalRelationalOperator>();
       
       
        for(int i=0; i<expPlans.size(); i++){
            LogicalExpressionPlan expPlan = expPlans.get(i);
            ProjectExpression projStar = getProjectLonelyStar(expPlan, oldPos2Rel);

            boolean foundExpandableProject = false;
            if(projStar != null){             
                //there is a project-star to be expanded

                LogicalSchema userStarSch = null;
                if(userSchema != null && userSchema.get(i) != null){
                    userStarSch = userSchema.get(i);
                }


                //the range values are set in the project in LOInnerLoad
                ProjectExpression loInnerProj = ((LOInnerLoad)oldPos2Rel.get(projStar.getInputNum())).getProjection();

                int firstProjCol = 0;
                int lastProjCol = 0;
               
                if(loInnerProj.isRangeProject()){
                    loInnerProj.setColumnNumberFromAlias();
                    firstProjCol = loInnerProj.getStartCol();
                    lastProjCol = loInnerProj.getEndCol();
                }

               
                boolean isProjectToEnd = loInnerProj.isProjectStar() ||
                    (loInnerProj.isRangeProject() && lastProjCol == -1);
               
                //can't expand if there is no input schema, and this is
                // as project star or project-range-to-end
                if( !(inpSch == null && isProjectToEnd) ){
                   
                    foundExpandableProject = true;

                    if(isProjectToEnd)
                        lastProjCol = inpSch.size() - 1;

                    //replacing the existing project star with new ones
                    expPlan.remove(projStar);

                    //remove the LOInnerLoad with star
                    LOInnerLoad oldLOInnerLoad = (LOInnerLoad)oldPos2Rel.get(projStar.getInputNum());
                    innerPlan.disconnect(oldLOInnerLoad, gen);
                    innerPlan.remove(oldLOInnerLoad);


                    //generate new exp plan, inner load for each field in schema
                    for(int j = firstProjCol; j <= lastProjCol; j++){

                        //add new LOInnerLoad
                        LOInnerLoad newInLoad = new LOInnerLoad(innerPlan, foreach, j);
                        innerPlan.add(newInLoad);
                        innerPlan.connect(newInLoad, gen);


                        // new expression plan and proj
                        LogicalExpressionPlan newExpPlan = new LogicalExpressionPlan();
                        newExpPlans.add(newExpPlan);

                        ProjectExpression newProj =
                            new ProjectExpression(newExpPlan, -2, -1, gen);

                        proj2InpRel.put(newProj, newInLoad);

                        newFlattens.add(flattens[i]);
                        if(newUserSchema != null ){
                            //index into user specified schema
                            int schIdx = j - firstProjCol;
                            if(userStarSch != null
                                    && userStarSch.getFields().size() > schIdx
                                    && userStarSch.getField(schIdx) != null){

                                //if the project-star field has user specified schema, use the
                                // j'th field for this column
                                LogicalSchema sch = new LogicalSchema();
                                sch.addField(new LogicalFieldSchema(userStarSch.getField(schIdx)));
                                newUserSchema.add(sch);
                            }
                            else{
                                newUserSchema.add(null);
                            }
                        }
                    }
                }
            }

            if(!foundExpandableProject){ //no project-star that could be expanded

                //get all projects in here
                FindProjects findProjs = new FindProjects(expPlan);
                findProjs.visit();
                List<ProjectExpression> projs = findProjs.getProjs();

                //create a mapping of project expression to their inputs
                for(ProjectExpression proj : projs){
                    proj2InpRel.put(proj, oldPos2Rel.get(proj.getInputNum()));
                }

                newExpPlans.add(expPlan);

                newFlattens.add(flattens[i]);
                if(newUserSchema != null)
                    newUserSchema.add(userSchema.get(i));

            }
        }

        //get mapping of LoGenerate input relation to current position
        Map<LogicalRelationalOperator, Integer> rel2pos = new HashMap<LogicalRelationalOperator, Integer>();
        List<Operator> newGenPreds = innerPlan.getPredecessors(gen);
        int numNewGenPreds = 0;
        if(newGenPreds != null)
            numNewGenPreds = newGenPreds.size();
           
        for(int i=0; i<numNewGenPreds; i++){
            rel2pos.put((LogicalRelationalOperator) newGenPreds.get(i),i);
        }
       
        //correct the input num for projects
        for(Entry<ProjectExpression, LogicalRelationalOperator> projAndInp : proj2InpRel.entrySet()){
           ProjectExpression proj = projAndInp.getKey();
           LogicalRelationalOperator rel = projAndInp.getValue();
           proj.setInputNum(rel2pos.get(rel));
        }
       
        // set the new lists
        gen.setOutputPlans(newExpPlans);
View Full Code Here

            ProjectExpression proj = (ProjectExpression)outputOp;
            //check if ProjectExpression is projectStar
            if(proj.isProjectStar()){
                //now check if its input is a LOInnerLoad and it is projectStar
                // or range project
                LogicalRelationalOperator inputRel = oldPos2Rel.get(proj.getInputNum());
                if(! (inputRel  instanceof LOInnerLoad)){
                    return null;
                }

                ProjectExpression innerProj = ((LOInnerLoad) inputRel).getProjection();
View Full Code Here

        //expand from firstProjCol to lastProjCol
        int firstProjCol = startAndEndProjs.first;
        int lastProjCol = startAndEndProjs.second;

        LogicalRelationalOperator relOp = proj.getAttachedRelationalOp();
        for(int i = firstProjCol; i <= lastProjCol; i++){
            newPlans.add(createExpPlanWithProj(relOp, inputNum, i));
        }

        return newPlans;
View Full Code Here

            LogicalExpressionPlan expPlan, ProjectExpression proj)
            throws FrontendException {
       
        // get the input schema first
       
        LogicalRelationalOperator relOp = proj.getAttachedRelationalOp();

        // list of inputs of attached relation
        List<Operator> inputRels = relOp.getPlan().getPredecessors(relOp);

        //the relation that is input to this project
        LogicalRelationalOperator inputRel =
            (LogicalRelationalOperator) inputRels.get(proj.getInputNum());

        LogicalSchema inputSchema = inputRel.getSchema();
       
       
        if(inputSchema == null &&
                (proj.isProjectStar() || (proj.isRangeProject() && proj.getEndCol() == -1))
        ){
View Full Code Here

        // test case pass.
        if( implicitReferencedOperator != null ) {
          List<Operator> args = plan.getSuccessors(this);
          if(args != null && args.size() > 0 ){
              int pos = (Integer)((ConstantExpression)args.get(0)).getValue();
              LogicalRelationalOperator inp = (LogicalRelationalOperator)implicitReferencedOperator;
 
              if( inp.getSchema() != null){
                  LogicalFieldSchema inpFs = inp.getSchema().getField(pos);
                  fieldSchema = new LogicalFieldSchema(inpFs);
                  //  fieldSchema.alias = "ReadScalars_" + fieldSchema.alias;
              }else{
                  fieldSchema = new LogicalFieldSchema(null, null, DataType.BYTEARRAY);
              }
View Full Code Here

    }

    private int findColNum(String alias) throws FrontendException {
        LogicalPlan lp = (LogicalPlan)attachedRelationalOp.getPlan();
        List<Operator> inputs = lp.getPredecessors( attachedRelationalOp );
        LogicalRelationalOperator input = (LogicalRelationalOperator)inputs.get( getInputNum() );
        LogicalSchema inputSchema = input.getSchema();

        if( alias != null ) {
            int colNum = inputSchema == null ? -1 : inputSchema.getFieldPosition( alias );
            if( colNum == -1 ) {
              String msg = "Invalid field projection. Projected field [" + alias + "] does not exist";
View Full Code Here

    @Override
    public LogicalSchema.LogicalFieldSchema getFieldSchema() throws FrontendException {
        if (fieldSchema!=null)
            return fieldSchema;
        LogicalRelationalOperator referent = findReferent();

        LogicalSchema schema = null;
        if (referent.getSchema()!=null) {
            schema = referent.getSchema().deepCopy();
        }

        if (attachedRelationalOp instanceof LOGenerate && plan.getSuccessors(this)==null) {
            if (!(findReferent() instanceof LOInnerLoad)||
                    ((LOInnerLoad)findReferent()).sourceIsBag()) {
View Full Code Here

        preds = attachedRelationalOp.getPlan().getPredecessors(attachedRelationalOp);
        if (preds == null || input >= preds.size()) {
            throw new FrontendException("Projection with nothing to reference!", 2225);
        }

        LogicalRelationalOperator pred =
            (LogicalRelationalOperator)preds.get(input);
        if (pred == null) {
            throw new FrontendException("Cannot find reference for " + this, 2226);
        }
        return pred;
View Full Code Here

            LogicalExpressionPlan expPlan = genPlans.get(i);
            List<Operator> expSources = expPlan.getSinks();

            for (Operator expSrc : expSources) {
                if (expSrc instanceof ProjectExpression) {
                    LogicalRelationalOperator reference = ((ProjectExpression)expSrc).findReferent();
                    if (branchHeadToRemove.contains(reference)) {
                        genPlansToRemove.add(expPlan);
                    }
                }
            }
View Full Code Here

TOP

Related Classes of org.apache.pig.newplan.logical.relational.LogicalRelationalOperator

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.