Package org.apache.pig.newplan.logical.expression

Examples of org.apache.pig.newplan.logical.expression.LogicalExpressionPlan


     * @param partitionCols list of partition columns of the table which is
     * being loaded in the LOAD statement which is input to the filter
     */
    public FilterExtractor(LogicalExpressionPlan plan) {
        this.originalPlan = plan;
        this.filteredPlan = new LogicalExpressionPlan();
        this.pushdownExprPlan = new LogicalExpressionPlan();
    }
View Full Code Here


            if (schema != null) {
                ArrayList<LogicalFieldSchema> fields = (ArrayList<LogicalFieldSchema>) schema
                        .getFields();
                for (int i = 0; i < fields.size(); i++) {
                    LogicalExpressionPlan lEplan = new LogicalExpressionPlan();
                    new ProjectExpression(lEplan, i, fields.get(i).alias, null, gen);
                    allExprPlan.add(lEplan);
                }
            }
        }

        // iterate over all operations and generate corresponding UDFs
        for (int operIdx = 0; operIdx < operations.size(); operIdx++) {
            List<LogicalExpressionPlan> lexpPlanList = new ArrayList<LogicalExpressionPlan>();
            List<LogicalExpression> lexpList = new ArrayList<LogicalExpression>();

            lexpPlanList.addAll(expressionPlans.get(operIdx));

            // If duplicates exists in the dimension list then exception is
            // thrown
            checkDuplicateProject(lexpPlanList);

            // Construct ProjectExpression from the LogicalExpressionPlans
            lexpList = getProjectExpList(lexpPlanList, gen);

            for (int i = 0; i < lexpList.size(); i++) {
                // Retain the columns that needs to be pushed down.
                // Remove the dimension columns from the input column list
                // as it will be attached to CubeDimension UDF
                for (int j = 0; j < allExprPlan.size(); j++) {
                    LogicalExpression lexp = (LogicalExpression) allExprPlan.get(j).getSources()
                        .get(0);
                    String colAlias = ((ProjectExpression) lexpList.get(i)).getColAlias();
                    if (colAlias == null) {
                        colAlias = ((ProjectExpression) lexpList.get(i)).getFieldSchema().alias;
                    }

                    String projExpAlias = null;
                    try {
                        projExpAlias = ((ProjectExpression) lexp).getColAlias();
                    } catch (ClassCastException e) {
                        // if it is not projection then it should be
                        // UserFuncExpr.
                        // ignore and continue till next ProjExpr is encountered
                        continue;
                    }
                    if (colAlias.equals(projExpAlias) == true) {
                        allExprPlan.remove(j);
                    } else {
                        // if projected exp alias is a namespaced alias
                        if (projExpAlias.lastIndexOf(":") != -1) {
                            projExpAlias = projExpAlias.substring(
                                projExpAlias.lastIndexOf(":") + 1, projExpAlias.length());
                            if (colAlias.equals(projExpAlias) == true) {
                                allExprPlan.remove(j);
                            }
                        }
                    }
                }
            }

            // Create UDF with user specified dimensions
            LogicalExpressionPlan uexpPlan = new LogicalExpressionPlan();
            if (operations.get(operIdx).equals("CUBE")) {
                new UserFuncExpression(uexpPlan, new FuncSpec(CubeDimensions.class.getName()),
                        lexpList);
            } else {
                new UserFuncExpression(uexpPlan, new FuncSpec(RollupDimensions.class.getName()),
                        lexpList);
            }

            for (LogicalExpressionPlan lexp : lexpPlanList) {
                Iterator<Operator> it = lexp.getOperators();
                while (it.hasNext()) {
                    uexpPlan.add(it.next());
                }
            }
            // Add the UDF to logical expression plan that contains dependent
            // attributes (pushed down from input columns)
            allExprPlan.add(operIdx, uexpPlan);
        }

        // If the operator is a UserFuncExpression then set the flatten flags.
        List<Boolean> flattenFlags = new ArrayList<Boolean>();
        for (int idx = 0; idx < allExprPlan.size(); idx++) {
            List<Operator> opers = allExprPlan.get(idx).getSources();
            for (Operator oper : opers) {
                if (oper instanceof ProjectExpression) {
                    flattenFlags.add(false);
                } else if (oper instanceof UserFuncExpression) {
                    flattenFlags.add(true);
                }
            }
        }

        // Generate and Foreach operator creation
        String falias = null;
        try {
            buildGenerateOp(loc, (LOForEach) foreach, (LOGenerate) gen, allExprPlan,
                flattenFlags, getUserDefinedSchema(allExprPlan));
            falias = buildForeachOp(loc, (LOForEach) foreach, "cube", inputAlias, innerPlan);
        } catch (ParserValidationException pve) {
            throw new FrontendException(pve);
        }

        List<Boolean> innerFlags = new ArrayList<Boolean>();
        List<String> inpAliases = new ArrayList<String>();
        inpAliases.add(falias);
        innerFlags.add(false);

        // Get the output schema of foreach operator and reconstruct the
        // LogicalExpressionPlan for each dimensional attributes
        MultiMap<Integer, LogicalExpressionPlan> exprPlansCopy = new MultiMap<Integer, LogicalExpressionPlan>();

        for (LogicalExpressionPlan exp : expressionPlans.values()) {
            LogicalExpression lexp = (LogicalExpression) exp.getSources().get(0);
            LogicalExpressionPlan epGrp = new LogicalExpressionPlan();
            new ProjectExpression(epGrp, 0, lexp.getFieldSchema().alias, null, groupby);
            exprPlansCopy.put(0, epGrp);
        }

        // build group by operator
View Full Code Here

    private List<LogicalExpression> getProjectExpList(List<LogicalExpressionPlan> lexpPlanList,
        LogicalRelationalOperator lro) throws FrontendException {

        List<LogicalExpression> leList = new ArrayList<LogicalExpression>();
        for (int i = 0; i < lexpPlanList.size(); i++) {
            LogicalExpressionPlan lexp = lexpPlanList.get(i);
            LogicalExpression lex = (LogicalExpression) lexp.getSources().get(0);
            Iterator<Operator> opers = lexp.getOperators();

            // ProjExpr are initially attached to CubeOp. So re-attach it to
            // specified operator
            while (opers.hasNext()) {
                Operator oper = opers.next();
View Full Code Here

                ProjectExpression projExpr = (ProjectExpression)sink;
                String colAlias = projExpr.getColAlias();
                if( projExpr.isRangeProject()){

                    LOInnerLoad innerLoad = new LOInnerLoad( lp, foreach,
                            new ProjectExpression(projExpr, new LogicalExpressionPlan())
                    );
                    setupInnerLoadAndProj(innerLoad, projExpr, lp, inputs);
                } else if( colAlias != null ) {
                    // the project is using a column alias
                    Operator op = projExpr.getProjectedOperator();
View Full Code Here

            Map<String, Operator> operators, Map<String, LogicalExpressionPlan> exprPlans, String colAlias, int col)
    throws RecognitionException {
        ProjectExpression result = null;

        if( colAlias != null ) {
            LogicalExpressionPlan exprPlan = exprPlans.get( colAlias );
            if( exprPlan != null ) {
                LogicalExpressionPlan planCopy = null;
                try {
                    planCopy = exprPlan.deepCopy();
                    plan.merge( planCopy );
                } catch (FrontendException ex) {
                    throw new PlanGenerationFailureException( intStream, loc, ex );
                }
                // The projected alias is actually expression alias, so the projections in the represented
                // expression doesn't have any operator associated with it. We need to set it when we
                // substitute the expression alias with the its expression.
                if( op != null ) {
                    Iterator<Operator> it = plan.getOperators();
                    while( it.hasNext() ) {
                        Operator o = it.next();
                        if( o instanceof ProjectExpression ) {
                            ProjectExpression projExpr = (ProjectExpression)o;
                            projExpr.setAttachedRelationalOp( op );
                        }
                    }
                }
                LogicalExpression root = (LogicalExpression)planCopy.getSources().get( 0 );// get the root of the plan
                LogicalFieldSchema schema;
                try {
                    schema = root.getFieldSchema();
                    if (schema.alias == null) {
                        schema.alias = colAlias;
View Full Code Here

                        if (!(succ instanceof ConstantExpression)) {
                            return;
                        }
                    }
                    // All successors are constant, calculate the value
                    OperatorPlan expLogicalPlan = new LogicalExpressionPlan();
                    ((BaseOperatorPlan)currentWalker.getPlan()).moveTree(op, (BaseOperatorPlan)expLogicalPlan);
                    PhysicalPlan expPhysicalPlan = new PhysicalPlan();
                    Map<Operator, PhysicalOperator> logToPhyMap = new HashMap<Operator, PhysicalOperator>();
                    PlanWalker childWalker = new ReverseDependencyOrderWalkerWOSeenChk(expLogicalPlan);
View Full Code Here

    String buildSampleOp(SourceLocation loc, String alias, String inputAlias, double value,
            SourceLocation valLoc)
                    throws ParserValidationException {

        LogicalExpressionPlan filterPlan = new LogicalExpressionPlan();
        //  Generate a filter condition.
        LogicalExpression konst = new ConstantExpression( filterPlan, value);
        konst.setLocation( valLoc );
        UserFuncExpression udf = new UserFuncExpression( filterPlan, new FuncSpec( RANDOM.class.getName() ) );
        new LessThanExpression( filterPlan, udf, konst );
View Full Code Here

        return buildOp ( loc, op, alias, inputAlias, null );
    }

    String buildSplitOtherwiseOp(SourceLocation loc, LOSplitOutput op, String alias, String inputAlias)
            throws ParserValidationException, PlanGenerationFailureException {
        LogicalExpressionPlan splitPlan = new LogicalExpressionPlan();
        Operator losplit = lookupOperator(inputAlias);
        LogicalExpression currentExpr = null;
        for (Operator losplitoutput : plan.getSuccessors(losplit)) {
            // take all the LOSplitOutput and negate their filter plans
            LogicalExpressionPlan fragment = ((LOSplitOutput) losplitoutput)
                    .getFilterPlan();
            try {
                if (OptimizerUtils.planHasNonDeterministicUdf(fragment))
                    throw new ParserValidationException(
                            intStream, loc, new FrontendException(op,
                                    "Can not use Otherwise in Split with an expression containing a @Nondeterministic UDF", 1131));
            } catch (FrontendException e) {
                e.printStackTrace();
                throw new PlanGenerationFailureException(intStream, loc, e);
            }
            LogicalExpression root = null;
            try {
                // get the root expression of the filter plan in LOSplitOutput and copy it
                root = ((LogicalExpression) fragment.getSources().get(0))
                        .deepCopy(splitPlan);
            } catch (FrontendException e) {
                e.printStackTrace();
                throw new PlanGenerationFailureException(intStream, loc, e);
            }
View Full Code Here

                List<LogicalExpressionPlan> rankPlans = loRank.getRankColPlans();
                byte[] newTypes = new byte[rankPlans.size()];

                for(int i = 0; i < rankPlans.size(); i++) {
                    LogicalExpressionPlan loep = rankPlans.get(i);
                    Iterator<Operator> inpOpers = loep.getOperators();

                    while(inpOpers.hasNext()) {
                        Operator oper = inpOpers.next();
                        newTypes[i] = ((ProjectExpression) oper).getType();
                    }
View Full Code Here

            throw new AssertionError("Size of expPlans and ascorder should be same");
        }           
       
        for(int i=0; i < expPlans.size(); i++){
            //expand the plan
            LogicalExpressionPlan ithExpPlan = expPlans.get(i);
            List<LogicalExpressionPlan> expandedPlans = expandPlan(ithExpPlan, 0);
            newExpPlans.addAll(expandedPlans);

            //add corresponding isAsc flags
            Boolean isAsc = ascOrder.get(i);
View Full Code Here

TOP

Related Classes of org.apache.pig.newplan.logical.expression.LogicalExpressionPlan

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.