* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

    private LogicalPlan plan = new LogicalPlan();

    private String lastRel = null;

    private Map<String, Operator> operators = new HashMap<String, Operator>() {

        @Override

        public Operator put(String k, Operator v) {

            lastRel = k;

            return super.put(k, v);

        }

    };

    Map<String, String> fileNameMap;

    private PigContext pigContext = null;

    private String scope = null;

    private IntStream intStream;

    private int storeIndex = 0;

    private int loadIndex = 0;

    private final BlackAndWhitelistFilter filter;

    private static NodeIdGenerator nodeIdGen = NodeIdGenerator.getGenerator();

    public static long getNextId(String scope) {

        return nodeIdGen.getNextNodeId( scope );

    }

    LogicalPlanBuilder(PigContext pigContext, String scope, Map<String, String> fileNameMap,

            IntStream input) {

        this.pigContext = pigContext;

        this.scope = scope;

        this.fileNameMap = fileNameMap;

        this.intStream = input;

        this.filter = new BlackAndWhitelistFilter(this.pigContext);

    }

    LogicalPlanBuilder(IntStream input) throws ExecException {

        pigContext = new PigContext( ExecType.LOCAL, new Properties() );

        pigContext.connect();

        this.scope = "test";

        this.fileNameMap = new HashMap<String, String>();

        this.intStream = input;

        this.filter = new BlackAndWhitelistFilter(this.pigContext);

    }

    Operator lookupOperator(String alias) {

        return operators.get( alias );

    }

    FuncSpec lookupFunction(String alias) {

        return pigContext.getFuncSpecFromAlias( alias );

    }

    StreamingCommand lookupCommand(String alias) {

        return pigContext.getCommandForAlias( alias );

    }

    void defineCommand(String alias, StreamingCommand command) {

        try {

            filter.validate(PigCommandFilter.Command.DEFINE);

        } catch (FrontendException e) {

            throw new RuntimeException(e.getMessage());

        }

        pigContext.registerStreamCmd( alias, command );

    }

    void defineFunction(String alias, FuncSpec fs) {

        try {

            filter.validate(PigCommandFilter.Command.DEFINE);

        } catch (FrontendException e) {

            throw new RuntimeException(e);

        }

        pigContext.registerFunction( alias, fs );

    }

    LogicalPlan getPlan() {

        return plan;

    }

    Map<String, Operator> getOperators() {

        return operators;

    }

    LOFilter createFilterOp() {

        return new LOFilter( plan );

    }

    LOLimit createLimitOp() {

        return new LOLimit( plan );

    }

    LOFilter createSampleOp() {

        return new LOFilter( plan, true );

    }

    String buildFilterOp(SourceLocation loc, LOFilter op, String alias,

            String inputAlias, LogicalExpressionPlan expr)

                    throws ParserValidationException {

        op.setFilterPlan( expr );

        alias = buildOp( loc, op, alias, inputAlias, null ); // it should actually return same alias

        try {

            (new ProjStarInUdfExpander(op.getPlan())).visit(op);

            new SchemaResetter(op.getPlan(), true).visit(op);

        } catch (FrontendException e) {

            throw new ParserValidationException( intStream, loc, e );

        }

        return alias;

    }

    String buildDistinctOp(SourceLocation loc, String alias, String inputAlias, String partitioner) throws ParserValidationException {

        LODistinct op = new LODistinct( plan );

        return buildOp( loc, op, alias, inputAlias, partitioner );

    }

    String buildLimitOp(SourceLocation loc, String alias, String inputAlias, long limit) throws ParserValidationException {

        LOLimit op = new LOLimit( plan, limit );

        return buildOp( loc, op, alias, inputAlias, null );

    }

    String buildLimitOp(SourceLocation loc, LOLimit op, String alias, String inputAlias, LogicalExpressionPlan expr) throws ParserValidationException {

        op.setLimitPlan(expr);

        return buildOp(loc, op, alias, inputAlias, null);

    }

    String buildSampleOp(SourceLocation loc, String alias, String inputAlias, double value,

            SourceLocation valLoc)

                    throws ParserValidationException {

        LogicalExpressionPlan filterPlan = new LogicalExpressionPlan();

        //  Generate a filter condition.

        LogicalExpression konst = new ConstantExpression( filterPlan, value);

        konst.setLocation( valLoc );

        UserFuncExpression udf = new UserFuncExpression( filterPlan, new FuncSpec( RANDOM.class.getName() ) );

        new LessThanExpression( filterPlan, udf, konst );

        LOFilter filter = new LOFilter( plan, true );

        return buildFilterOp( loc, filter, alias, inputAlias, filterPlan );

    }

    String buildSampleOp(SourceLocation loc, LOFilter filter, String alias, String inputAlias,

            LogicalExpressionPlan samplePlan, LogicalExpression expr)

                    throws ParserValidationException {

        UserFuncExpression udf = new UserFuncExpression( samplePlan, new FuncSpec( RANDOM.class.getName() ) );

        new LessThanExpression( samplePlan, udf, expr );

        return buildFilterOp( loc, filter, alias, inputAlias, samplePlan );

    }

    String buildUnionOp(SourceLocation loc, String alias, List<String> inputAliases, boolean onSchema) throws ParserValidationException {

        checkDuplicateAliases(inputAliases, loc, "UNION");

        LOUnion op = new LOUnion( plan, onSchema );

        return buildOp( loc, op, alias, inputAliases, null );

    }

    String buildSplitOp(SourceLocation loc, String inputAlias) throws ParserValidationException {

        LOSplit op = new LOSplit( plan );

        return buildOp( loc, op, null, inputAlias, null );

    }

    LOSplitOutput createSplitOutputOp() {

        return  new LOSplitOutput( plan );

    }

    String buildSplitOutputOp(SourceLocation loc, LOSplitOutput op, String alias, String inputAlias,

            LogicalExpressionPlan filterPlan) throws ParserValidationException {

        op.setFilterPlan( filterPlan );

        return buildOp ( loc, op, alias, inputAlias, null );

    }

    String buildSplitOtherwiseOp(SourceLocation loc, LOSplitOutput op, String alias, String inputAlias)

            throws ParserValidationException, PlanGenerationFailureException {

        LogicalExpressionPlan splitPlan = new LogicalExpressionPlan();

        Operator losplit = lookupOperator(inputAlias);

        LogicalExpression currentExpr = null;

        for (Operator losplitoutput : plan.getSuccessors(losplit)) {

            // take all the LOSplitOutput and negate their filter plans

            LogicalExpressionPlan fragment = ((LOSplitOutput) losplitoutput)

                    .getFilterPlan();

            try {

                if (OptimizerUtils.planHasNonDeterministicUdf(fragment))

                    throw new ParserValidationException(

                            intStream, loc, new FrontendException(op,

                                    "Can not use Otherwise in Split with an expression containing a @Nondeterministic UDF", 1131));

            } catch (FrontendException e) {

                e.printStackTrace();

                throw new PlanGenerationFailureException(intStream, loc, e);

            }

            LogicalExpression root = null;

            try {

                // get the root expression of the filter plan in LOSplitOutput and copy it

                root = ((LogicalExpression) fragment.getSources().get(0))

                        .deepCopy(splitPlan);

            } catch (FrontendException e) {

                e.printStackTrace();

                throw new PlanGenerationFailureException(intStream, loc, e);

            }

            if (root == null)

                throw new PlanGenerationFailureException(intStream, loc,

                        new FrontendException(op,

                                "Could not retrieve LogicalExpression for LOSplitOutput " + losplitoutput, 2048));

            if (currentExpr == null)

                currentExpr = root;

            else

                currentExpr = new OrExpression(splitPlan, currentExpr, root);

        }

        // using De Morgan's law (!A && !B) == !(A || B)

        currentExpr = new NotExpression(splitPlan, currentExpr);

        try {

            // Going through all the ProjectExpressions that were cloned

            // and updating the attached operators from its original

            // LOSplitOutput to to the "otherwise" LOSplitOutput

            // (PIG-3641)

            new ResetProjectionAttachedRelationalOpVisitor(splitPlan, op).visit();

        } catch (FrontendException e) {

            e.printStackTrace();

            throw new PlanGenerationFailureException(intStream, loc, e);

        }

        op.setFilterPlan(splitPlan);

        return buildOp(loc, op, alias, inputAlias, null);

    }

    String buildCrossOp(SourceLocation loc, String alias, List<String> inputAliases, String partitioner) throws ParserValidationException {

        LOCross op = new LOCross( plan );

        return buildOp ( loc, op, alias, inputAliases, partitioner );

    }

    LOSort createSortOp() {

        return new LOSort( plan );

    }

    String buildSortOp(SourceLocation loc, LOSort sort, String alias, String inputAlias, List<LogicalExpressionPlan> plans,

            List<Boolean> ascFlags, FuncSpec fs) throws ParserValidationException {

        sort.setSortColPlans( plans );

        sort.setUserFunc( fs );

        if (ascFlags.isEmpty()) {

            for (int i=0;i<plans.size();i++)

                ascFlags.add(true);

        }

        sort.setAscendingCols( ascFlags );

        alias = buildOp( loc, sort, alias, inputAlias, null );

        try {

            (new ProjectStarExpander(sort.getPlan())).visit(sort);

            (new ProjStarInUdfExpander(sort.getPlan())).visit(sort);

            new SchemaResetter(sort.getPlan(), true).visit(sort);

        } catch (FrontendException e ) {

            throw new ParserValidationException( intStream, loc, e );

        }

        return alias;

    }

    LORank createRankOp() {

        return new LORank( plan );

    }

    String buildRankOp(SourceLocation loc, LORank rank, String alias, String inputAlias, List<LogicalExpressionPlan> plans,

            List<Boolean> ascFlags) throws ParserValidationException {

        //Rank

        rank.setRankColPlan(plans);

        if (ascFlags.isEmpty()) {

            for (int i=0;i<plans.size();i++)

                ascFlags.add(true);

        }

        rank.setAscendingCol(ascFlags);

        buildOp( loc, rank, alias, inputAlias, null );

        try {

            (new ProjectStarExpander(rank.getPlan())).visit(rank);

            (new ProjStarInUdfExpander(rank.getPlan())).visit(rank);

            new SchemaResetter(rank.getPlan(), true).visit(rank);

        } catch (FrontendException e ) {

            throw new ParserValidationException( intStream, loc, e );

        }

        return alias;

    }

    LOJoin createJoinOp() {

        return new LOJoin( plan );

    }

    String buildJoinOp(SourceLocation loc, LOJoin op, String alias, List<String> inputAliases,

            MultiMap<Integer, LogicalExpressionPlan> joinPlans,

            JOINTYPE jt, List<Boolean> innerFlags, String partitioner)

    throws ParserValidationException {

        checkDuplicateAliases(inputAliases, loc, "JOIN");

        if (jt==null)

            jt = JOINTYPE.HASH;

        else {

            op.pinOption(LOJoin.OPTION_JOIN);

        }

        int inputCount = inputAliases.size();

        if( jt == JOINTYPE.SKEWED ) {

            if( partitioner != null ) {

                throw new ParserValidationException( intStream, loc,

                        "Custom Partitioner is not supported for skewed join" );

            }

            if( inputCount != 2 ) {

                throw new ParserValidationException( intStream, loc,

                        "Skewed join can only be applied for 2-way joins" );

            }

        } else if( (jt == JOINTYPE.MERGE || jt == JOINTYPE.MERGESPARSE) && inputCount != 2 ) {

            throw new ParserValidationException( intStream, loc,

                    "Merge join can only be applied for 2-way joins" );

        } else if( jt == JOINTYPE.REPLICATED ) {

            if( innerFlags.size() == 2 && innerFlags.get( 0 ) == false ) {

                throw new ParserValidationException( intStream, loc,

                        "Replicated join does not support (right|full) outer joins" );

            }

        }

        boolean[] flags = new boolean[joinPlans.size()];

        if (innerFlags.size()!=0) {

            for( int i = 0; i < joinPlans.size(); i++ ) {

                flags[i] = innerFlags.get( i );

            }

        }

        else {

            for( int i = 0; i < joinPlans.size(); i++ ) {

                flags[i] = true;

            }

        }

        op.setJoinType( jt );

        op.setInnerFlags( flags );

        op.setJoinPlans( joinPlans );

        alias = buildOp( loc, op, alias, inputAliases, partitioner );

        try {

            (new ProjectStarExpander(op.getPlan())).visit(op);

            (new ProjStarInUdfExpander(op.getPlan())).visit(op);

            new SchemaResetter(op.getPlan(), true).visit(op);

        } catch (FrontendException e ) {

            throw new ParserValidationException( intStream, loc, e );

        }

        return alias;

    }

    LOCube createCubeOp() {

        return new LOCube(plan);

    }

    String buildCubeOp(SourceLocation loc, LOCube op, String alias, String inputAlias,

        List<String> operations, MultiMap<Integer, LogicalExpressionPlan> expressionPlans)

        throws ParserValidationException {

        // check if continuously occurring cube operations be combined

        combineCubeOperations((ArrayList<String>) operations, expressionPlans);

        // set the expression plans for cube operator and build cube operator

        op.setExpressionPlans(expressionPlans);

        op.setOperations(operations);

        buildOp(loc, op, alias, inputAlias, null);

        try {

            (new ProjectStarExpander(op.getPlan())).visit(op);

            (new ProjStarInUdfExpander(op.getPlan())).visit(op);

            new SchemaResetter(op.getPlan(), true).visit(op);

        } catch (FrontendException e ) {

            throw new ParserValidationException( intStream, loc, e );

        }

        try {

            alias = convertCubeToFGPlan(loc, op, inputAlias, operations, expressionPlans);

        } catch (FrontendException e) {

            throw new ParserValidationException(intStream, loc, e);

        }

        return alias;

    }

    // if multiple CUBE operations occur continuously then it can be combined

    // together CUBE rel BY CUBE(a,b), CUBE(c,d); => CUBE rel BY CUBE(a,b,c,d)

    private void combineCubeOperations(ArrayList<String> operations,

        MultiMap<Integer, LogicalExpressionPlan> expressionPlans) {

        int startIdx = -1;

        int endIdx = -1;

        int i = 0;

        boolean isMerged = false;

        // scan and perform merge of column projections

        for (i = 0; i < operations.size(); i++) {

            if ((startIdx == -1) && (operations.get(i).equals("CUBE") == true)) {

                startIdx = i;

            } else {

                if (operations.get(i).equals("CUBE") == true) {

                    endIdx = i;

                } else {

                    if (endIdx > startIdx) {

                        mergeAndMarkForDelete(operations, expressionPlans, startIdx, endIdx);

                        isMerged = true;

                        startIdx = -1;

                        endIdx = -1;

                    } else {

                        startIdx = -1;

                        endIdx = -1;

                    }

                }

            }

        }

        // this check is required for the case when the sequence of CUBE

        // operations occurs at the end, like (CUBE, ROLLUP, CUBE, CUBE)

        // in which case endIdx will be greater than startIdx

        if (endIdx > startIdx) {

            isMerged = true;

            mergeAndMarkForDelete(operations, expressionPlans, startIdx, endIdx);

        }

        // if merged then remove the column projections that were marked for

        // deletion

        if (isMerged) {

            performDeletion(expressionPlans, operations);

        }

    }

    private void performDeletion(MultiMap<Integer, LogicalExpressionPlan> expressionPlans,

        ArrayList<String> operations) {

        MultiMap<Integer, LogicalExpressionPlan> ep = new MultiMap<Integer, LogicalExpressionPlan>();

        List<String> op = new ArrayList<String>();

        int idx = 0;

        // rearranging indices

        for (int i = 0; i < operations.size(); i++) {

            if (operations.get(i) != null) {

                op.add(idx, operations.get(i));

            }

            if (expressionPlans.get(i) != null) {

                ep.put(idx, expressionPlans.get(i));

                idx++;

            }

        }

        // performing deletions

        operations.clear();

        operations.addAll(op);

        expressionPlans.clear();

        for (Integer i : ep.keySet()) {

            expressionPlans.put(i, ep.get(i));

        }

    }

    // performs merging of dimensions of merged cube operation

    // Ex: CUBE(a,b), CUBE(c,d) ==> CUBE(a,b,c,d)

    // in the above example CUBE operator and dimensions are merged

    private void mergeAndMarkForDelete(ArrayList<String> operations,

        MultiMap<Integer, LogicalExpressionPlan> expressionPlans, int startIdx, int endIdx) {

        // mark for delete

        for (int i = startIdx + 1; i <= endIdx; i++) {

            expressionPlans.put(startIdx, expressionPlans.get(i));

            expressionPlans.removeKey(i);

            operations.remove(i);

            operations.add(i, null);

        }

    }

    // This function creates logical plan for foreach and groupby operators.

    // It connects the predecessors of cube operator with foreach plan and

    // disconnects cube operator from the logical plan. It also connects foreach

    // plan with groupby plan.

    private String convertCubeToFGPlan(SourceLocation loc, LOCube op, String inputAlias,

        List<String> operations, MultiMap<Integer, LogicalExpressionPlan> expressionPlans)

        throws FrontendException {

        LOForEach foreach = new LOForEach(plan);

        LOCogroup groupby = new LOCogroup(plan);

        LogicalPlan innerPlan = new LogicalPlan();

        LogicalRelationalOperator gen = new LOGenerate(innerPlan);

        injectForeachOperator(loc, op, foreach);

        // Get all column attributes from the input relation.

        // Create ProjectExpression for all columns. Based on the

        // dimensions specified by the user, specified columns will be attached

        // to CubeDimension/RollupDimension UDF and rest will be pushed down

        List<Operator> inpOpers = foreach.getPlan().getPredecessors(foreach);

        List<LogicalExpressionPlan> allExprPlan = new ArrayList<LogicalExpressionPlan>();

        for (Operator oper : inpOpers) {

            LogicalSchema schema = new LogicalSchema();

            schema = ((LogicalRelationalOperator) oper).getSchema();

            if (schema != null) {

                ArrayList<LogicalFieldSchema> fields = (ArrayList<LogicalFieldSchema>) schema

                        .getFields();

                for (int i = 0; i < fields.size(); i++) {

                    LogicalExpressionPlan lEplan = new LogicalExpressionPlan();

                    new ProjectExpression(lEplan, i, fields.get(i).alias, null, gen);

                    allExprPlan.add(lEplan);

                }

            }

        }

        // iterate over all operations and generate corresponding UDFs

        for (int operIdx = 0; operIdx < operations.size(); operIdx++) {

            List<LogicalExpressionPlan> lexpPlanList = new ArrayList<LogicalExpressionPlan>();

            List<LogicalExpression> lexpList = new ArrayList<LogicalExpression>();

            lexpPlanList.addAll(expressionPlans.get(operIdx));

            // If duplicates exists in the dimension list then exception is

            // thrown

            checkDuplicateProject(lexpPlanList);

            // Construct ProjectExpression from the LogicalExpressionPlans

            lexpList = getProjectExpList(lexpPlanList, gen);

            for (int i = 0; i < lexpList.size(); i++) {

                // Retain the columns that needs to be pushed down.

                // Remove the dimension columns from the input column list

                // as it will be attached to CubeDimension UDF

                for (int j = 0; j < allExprPlan.size(); j++) {

                    LogicalExpression lexp = (LogicalExpression) allExprPlan.get(j).getSources()

                        .get(0);

                    String colAlias = ((ProjectExpression) lexpList.get(i)).getColAlias();

                    if (colAlias == null) {

                        colAlias = ((ProjectExpression) lexpList.get(i)).getFieldSchema().alias;

                    }

                    String projExpAlias = null;

                    try {

                        projExpAlias = ((ProjectExpression) lexp).getColAlias();

                    } catch (ClassCastException e) {

                        // if it is not projection then it should be

                        // UserFuncExpr.

                        // ignore and continue till next ProjExpr is encountered

                        continue;

                    }

                    if (colAlias.equals(projExpAlias) == true) {

                        allExprPlan.remove(j);

                    } else {

                        // if projected exp alias is a namespaced alias

                        if (projExpAlias.lastIndexOf(":") != -1) {

                            projExpAlias = projExpAlias.substring(

                                projExpAlias.lastIndexOf(":") + 1, projExpAlias.length());

                            if (colAlias.equals(projExpAlias) == true) {

                                allExprPlan.remove(j);

                            }

                        }

                    }

                }

            }

            // Create UDF with user specified dimensions

            LogicalExpressionPlan uexpPlan = new LogicalExpressionPlan();

            if (operations.get(operIdx).equals("CUBE")) {

                new UserFuncExpression(uexpPlan, new FuncSpec(CubeDimensions.class.getName()),

                        lexpList);

            } else {

                new UserFuncExpression(uexpPlan, new FuncSpec(RollupDimensions.class.getName()),

                        lexpList);

            }

            for (LogicalExpressionPlan lexp : lexpPlanList) {

                Iterator<Operator> it = lexp.getOperators();

                while (it.hasNext()) {

                    uexpPlan.add(it.next());

                }

            }

            // Add the UDF to logical expression plan that contains dependent

            // attributes (pushed down from input columns)

            allExprPlan.add(operIdx, uexpPlan);

        }

        // If the operator is a UserFuncExpression then set the flatten flags.

        List<Boolean> flattenFlags = new ArrayList<Boolean>();

        for (int idx = 0; idx < allExprPlan.size(); idx++) {

            List<Operator> opers = allExprPlan.get(idx).getSources();

            for (Operator oper : opers) {

                if (oper instanceof ProjectExpression) {

                    flattenFlags.add(false);

                } else if (oper instanceof UserFuncExpression) {

                    flattenFlags.add(true);

                }

            }

        }

        // Generate and Foreach operator creation

        String falias = null;

        try {

            buildGenerateOp(loc, (LOForEach) foreach, (LOGenerate) gen, allExprPlan,

                flattenFlags, getUserDefinedSchema(allExprPlan));

            falias = buildForeachOp(loc, (LOForEach) foreach, "cube", inputAlias, innerPlan);

        } catch (ParserValidationException pve) {

            throw new FrontendException(pve);

        }

        List<Boolean> innerFlags = new ArrayList<Boolean>();

        List<String> inpAliases = new ArrayList<String>();

        inpAliases.add(falias);

        innerFlags.add(false);

        // Get the output schema of foreach operator and reconstruct the

        // LogicalExpressionPlan for each dimensional attributes

        MultiMap<Integer, LogicalExpressionPlan> exprPlansCopy = new MultiMap<Integer, LogicalExpressionPlan>();

        for (LogicalExpressionPlan exp : expressionPlans.values()) {

            LogicalExpression lexp = (LogicalExpression) exp.getSources().get(0);

            LogicalExpressionPlan epGrp = new LogicalExpressionPlan();

            new ProjectExpression(epGrp, 0, lexp.getFieldSchema().alias, null, groupby);

            exprPlansCopy.put(0, epGrp);

        }

        // build group by operator

        try {

            return buildGroupOp(loc, (LOCogroup) groupby, op.getAlias(), inpAliases, exprPlansCopy,

                GROUPTYPE.REGULAR, innerFlags, null);

        } catch (ParserValidationException pve) {

            throw new FrontendException(pve);

        }

    }

    // User defined schema for generate operator. If not specified output schema

    // of UDF will be used which will prefix "dimensions" namespace to all fields

    private List<LogicalSchema> getUserDefinedSchema(List<LogicalExpressionPlan> allExprPlan)

        throws FrontendException {

        List<LogicalSchema> genOutputSchema = new ArrayList<LogicalSchema>();

        for (int i = 0; i < allExprPlan.size(); i++) {

            List<Operator> opers = allExprPlan.get(i).getSources();

            for (Operator oper : opers) {

                // add a logical schema for dimensions that are pushed from

                // predecessor of cube/rollup

                if (oper instanceof ProjectExpression) {

                    LogicalSchema output = new LogicalSchema();

                    output.addField(new LogicalFieldSchema(

                        ((ProjectExpression) oper).getColAlias(), null, DataType.NULL));

                    genOutputSchema.add(output);

                } else if (oper instanceof UserFuncExpression) {

                    // add logical schema for dimensions specified in

                    // cube/rollup operator

                    LogicalSchema output = new LogicalSchema();

                    for (Operator op : ((UserFuncExpression) oper).getPlan().getSinks()) {

                        output.addField(new LogicalFieldSchema(((ProjectExpression) op)

                                .getFieldSchema()));

                    }

                    genOutputSchema.add(output);

                }

            }

        }

        return genOutputSchema;

    }

    private List<LogicalExpression> getProjectExpList(List<LogicalExpressionPlan> lexpPlanList,

        LogicalRelationalOperator lro) throws FrontendException {

        List<LogicalExpression> leList = new ArrayList<LogicalExpression>();

        for (int i = 0; i < lexpPlanList.size(); i++) {

            LogicalExpressionPlan lexp = lexpPlanList.get(i);

            LogicalExpression lex = (LogicalExpression) lexp.getSources().get(0);

            Iterator<Operator> opers = lexp.getOperators();

            // ProjExpr are initially attached to CubeOp. So re-attach it to

            // specified operator

            while (opers.hasNext()) {

                Operator oper = opers.next();

                try {

                    ((ProjectExpression) oper).setAttachedRelationalOp(lro);

                } catch (ClassCastException cce) {

                    throw new FrontendException("Column project expected.", cce);

                }

            }

            leList.add(lex);

        }

        return leList;

    }

    // This method connects the predecessors of cube operator with foreach

    // operator and disconnects the cube operator from its predecessors

    private void injectForeachOperator(SourceLocation loc, LOCube op, LOForEach foreach)

        throws FrontendException {

        // connect the foreach operator with predecessors of cube operator

        List<Operator> opers = op.getPlan().getPredecessors(op);

        for (Operator oper : opers) {

            OperatorPlan foreachPlan = foreach.getPlan();

            foreachPlan.connect(oper, (Operator) foreach);

        }

        // disconnect the cube operator from the plan

        opers = foreach.getPlan().getPredecessors(foreach);

        for (Operator lop : opers) {

            List<Operator> succs = lop.getPlan().getSuccessors(lop);

            for (Operator succ : succs) {

                if (succ instanceof LOCube) {

                    succ.getPlan().disconnect(lop, succ);

                    succ.getPlan().remove(succ);

                }

            }

        }

    }

    // This methods if the dimensions specified by the user has duplicates

    private void checkDuplicateProject(List<LogicalExpressionPlan> lExprPlan)

        throws FrontendException {

        for (int i = 0; i < lExprPlan.size(); i++) {

            for (int j = i + 1; j < lExprPlan.size(); j++) {

                LogicalExpression outer = (LogicalExpression) lExprPlan.get(i).getSources().get(0);

                LogicalExpression inner = (LogicalExpression) lExprPlan.get(j).getSources().get(0);

                String outColAlias = ((ProjectExpression) outer).getColAlias();

                String inColAlias = ((ProjectExpression) inner).getColAlias();

                if (outColAlias == null) {

                    outColAlias = outer.getFieldSchema().alias;

                }

                if (inColAlias == null) {

                    inColAlias = inner.getFieldSchema().alias;

                }

                if (outColAlias.equals(inColAlias) == true) {

                    lExprPlan.remove(j);

                    throw new FrontendException("Duplicate dimensions detected. Dimension name: "

                        + inColAlias);

                }

            }

        }

    }

    LOCogroup createGroupOp() {

        return new LOCogroup( plan );

    }

    String buildGroupOp(SourceLocation loc, LOCogroup op, String alias, List<String> inputAliases,

        MultiMap<Integer, LogicalExpressionPlan> expressionPlans, GROUPTYPE gt, List<Boolean> innerFlags,

        String partitioner) throws ParserValidationException {

        if( gt == GROUPTYPE.COLLECTED ) {

            if( inputAliases.size() > 1 ) {

                throw new ParserValidationException( intStream, loc,

                        "Collected group is only supported for single input" );

            }

            List<LogicalExpressionPlan> exprPlans = expressionPlans.get( 0 );

            for( LogicalExpressionPlan exprPlan : exprPlans ) {

                Iterator<Operator> it = exprPlan.getOperators();

                while( it.hasNext() ) {

                    if( !( it.next() instanceof ProjectExpression ) ) {

                        throw new ParserValidationException( intStream, loc,

                                "Collected group is only supported for columns or star projection" );

                    }

                }

            }

        }

        boolean[] flags = new boolean[innerFlags.size()];

        for( int i = 0; i < innerFlags.size(); i++ ) {

            flags[i] = innerFlags.get( i );

        }

        op.setExpressionPlans( expressionPlans );

        op.setGroupType( gt );

        op.setInnerFlags( flags );

        alias = buildOp( loc, op, alias, inputAliases, partitioner );

        try {

            (new ProjectStarExpander(op.getPlan())).visit(op);

            (new ProjStarInUdfExpander(op.getPlan())).visit(op);

            new SchemaResetter(op.getPlan(), true).visit(op);

        } catch (FrontendException e ) {

            throw new ParserValidationException( intStream, loc, e );

        }

        return alias;

    }

    String buildLoadOp(SourceLocation loc, String alias, String filename, FuncSpec funcSpec, LogicalSchema schema)

    throws ParserValidationException {

        String absolutePath;

        LoadFunc loFunc;

        try {

            // Load LoadFunc class from default properties if funcSpec is null. Fallback on PigStorage if LoadFunc is not specified in properties.

            funcSpec = funcSpec == null ? new FuncSpec(pigContext.getProperties().getProperty(PigConfiguration.PIG_DEFAULT_LOAD_FUNC, PigStorage.class.getName())) : funcSpec;

            loFunc = (LoadFunc)PigContext.instantiateFuncFromSpec(funcSpec);

            String fileNameKey = QueryParserUtils.constructFileNameSignature(filename, funcSpec) + "_" + (loadIndex++);

            absolutePath = fileNameMap.get(fileNameKey);

            if (absolutePath == null) {

                absolutePath = loFunc.relativeToAbsolutePath( filename, QueryParserUtils.getCurrentDir( pigContext ) );

                if (absolutePath!=null) {

                    QueryParserUtils.setHdfsServers( absolutePath, pigContext );

                }

                fileNameMap.put( fileNameKey, absolutePath );

            }

        } catch(Exception ex) {

            throw new ParserValidationException( intStream, loc, ex );

        }

        FileSpec loader = new FileSpec(absolutePath, funcSpec);

        LOLoad op = new LOLoad(

                loader,

                schema,

                plan,

                ConfigurationUtil.toConfiguration(pigContext.getProperties()),

                loFunc,

                alias + "_" + newOperatorKey());

        op.setTmpLoad(false);

        // Check if there's a store in the plan already that this load

        // depends on. If so, add it as an input alias

        List<String> inputAliases = new ArrayList<String>();

        // Get list of stores. The stores are not all sinks in the plan

        // if they've already got successors.

        Iterator<Operator> itr = plan.getOperators();

        List<LOStore> stores = new ArrayList<LOStore>();

        while (itr.hasNext()) {

            Operator lop = itr.next();

            if (lop instanceof LOStore) {

                stores.add((LOStore)lop);

            }

        }

        for (LOStore store : stores) {

            String ifile = op.getFileSpec().getFileName();

            String ofile = store.getFileSpec().getFileName();

            if (ofile.equals(ifile)) {

                inputAliases.add( store.getAlias() );

            }

        }

        return buildOp( loc, op, alias, inputAliases, null );

    }

    private String buildOp(SourceLocation loc, LogicalRelationalOperator op, String alias,

            String inputAlias, String partitioner) throws ParserValidationException {

        List<String> inputAliases = new ArrayList<String>();

        if( inputAlias != null )

            inputAliases.add( inputAlias );

        return buildOp( loc, op, alias, inputAliases, partitioner );

    }

    private void checkDuplicateAliases(List<String> inputAliases, SourceLocation loc,

            String opName) throws ParserValidationException {

        //Keep the count of the number of times the same Alias is used

        Map<Operator, Integer> inputAliasesMap = new HashMap<Operator, Integer>();

        for(String a : inputAliases) {

            Operator pred = operators.get( a );

            if (pred == null) {

                throw new ParserValidationException( intStream, loc, "Unrecognized alias " + a );

            }

            if (inputAliasesMap.containsKey(pred)) {

                throw new ParserValidationException( intStream, loc,

                        "Pig does not accept same alias as input for " + opName +

                        " operation : " + a );

            }

            else {

                inputAliasesMap.put(pred, 1);

            }

        }

    }

    private String buildOp(SourceLocation loc, LogicalRelationalOperator op, String alias,

            List<String> inputAliases, String partitioner) throws ParserValidationException {

        setAlias( op, alias );

        setPartitioner( op, partitioner );

        op.setLocation( loc );

        plan.add( op );

        for( String a : inputAliases ) {

            Operator pred = operators.get( a );

            if (pred==null) {

                throw new ParserValidationException( intStream, loc, "Unrecognized alias " + a );

            }

            plan.connect( pred, op );

        }

        operators.put( op.getAlias(), op );

        pigContext.setLastAlias(op.getAlias());

        return op.getAlias();

    }

    String buildStoreOp(SourceLocation loc, String alias, String inputAlias, String filename, FuncSpec funcSpec)

    throws ParserValidationException {

        try {

            // Load StoreFunc class from default properties if funcSpec is null. Fallback on PigStorage if StoreFunc is not specified in properties.

            funcSpec = funcSpec == null ? new FuncSpec(pigContext.getProperties().getProperty(

                    PigConfiguration.PIG_DEFAULT_STORE_FUNC, PigStorage.class.getName())) : funcSpec;

            StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec(funcSpec);

            String fileNameKey = inputAlias + "_" + (storeIndex++) ;

            String signature = inputAlias + "_" + newOperatorKey();

            stoFunc.setStoreFuncUDFContextSignature(signature);

            String absolutePath = fileNameMap.get(fileNameKey);

            if (absolutePath == null) {

                absolutePath = stoFunc.relToAbsPathForStoreLocation(

                        filename,

                        QueryParserUtils.getCurrentDir(pigContext));

                if (absolutePath!=null) {

                    QueryParserUtils.setHdfsServers(absolutePath, pigContext);

                }

                fileNameMap.put(fileNameKey, absolutePath);