Package org.apache.pig.newplan

Examples of org.apache.pig.newplan.Operator


                PhysicalOperator op = logToPhyMap.get(l);
                if (l instanceof ProjectExpression ) {
                    int input = ((ProjectExpression)l).getInputNum();                   
                   
                    // for each sink projection, get its input logical plan and translate it
                    Operator pred = preds.get(input);
                    childWalker = new SubtreeDependencyOrderWalker(inner, pred);
                    pushWalker(childWalker);
                    childWalker.walk(this);
                    popWalker();
                   
View Full Code Here


        POLocalRearrange[] innerLRs = new POLocalRearrange[inputs.size()];
        int count = 0;
        List<PhysicalOperator> inpPOs = new ArrayList<PhysicalOperator>(inputs.size());
       
        for (int i=0;i<inputs.size();i++) {
            Operator op = inputs.get(i);
            PhysicalOperator physOp = logToPhyMap.get(op);
            inpPOs.add(physOp);
           
            List<LogicalExpressionPlan> plans = (List<LogicalExpressionPlan>)innerPlans.get(i);
            POLocalRearrange poInnerLR = new POLocalRearrange(new OperatorKey(DEFAULT_SCOPE, nodeGen.getNextNodeId(DEFAULT_SCOPE)));
View Full Code Here

        boolean[] innerFlags = loj.getInnerFlags();
        String alias = loj.getAlias();
        int parallel = loj.getRequestedParallelisam();
       
        for (int i=0;i<inputs.size();i++) {
            Operator op = inputs.get(i);
            PhysicalOperator physOp = logToPhyMap.get(op);
            inp.add(physOp);
            List<LogicalExpressionPlan> plans = (List<LogicalExpressionPlan>) loj.getJoinPlan(i);
           
            List<PhysicalPlan> exprPlans = translateExpressionPlans(loj, plans);

            ppLists.add(exprPlans);
            joinPlans.put(physOp, exprPlans);
           
            // Key could potentially be a tuple. So, we visit all exprPlans to get types of members of tuples.
            List<Byte> tupleKeyMemberTypes = new ArrayList<Byte>();
            for(PhysicalPlan exprPlan : exprPlans)
                tupleKeyMemberTypes.add(exprPlan.getLeaves().get(0).getResultType());
            keyTypes.add(tupleKeyMemberTypes);
        }

        if (loj.getJoinType() == LOJoin.JOINTYPE.SKEWED) {
            POSkewedJoin skj;
            try {
                skj = new POSkewedJoin(new OperatorKey(scope,nodeGen.getNextNodeId(scope)),
                                        parallel,inp, innerFlags);
                skj.setAlias(alias);
                skj.setJoinPlans(joinPlans);
            }
            catch (Exception e) {
                int errCode = 2015;
                String msg = "Skewed Join creation failed";
                throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
            }
            skj.setResultType(DataType.TUPLE);
           
            for (int i=0; i < inputs.size(); i++) {
                Operator op = inputs.get(i);
                if (!innerFlags[i]) {
                    try {
                        LogicalSchema s = ((LogicalRelationalOperator)op).getSchema();
                        // if the schema cannot be determined
                        if (s == null) {
View Full Code Here

        int count = 0;
        Byte type = null;
        List<Operator> inputs = relationalOp.getPlan().getPredecessors(relationalOp);
        for (int i=0;i<inputs.size();i++) {
            Operator op = inputs.get(i);
            List<LogicalExpressionPlan> plans = (List<LogicalExpressionPlan>)innerPlans.get(i);
            POLocalRearrange physOp = new POLocalRearrange(new OperatorKey(
                    DEFAULT_SCOPE, nodeGen.getNextNodeId(DEFAULT_SCOPE)), relationalOp.getRequestedParallelisam());
            physOp.setAlias(relationalOp.getAlias());
            List<PhysicalPlan> exprPlans = translateExpressionPlans(relationalOp, plans);
View Full Code Here

                // the parser would have marked the side
                // where we need to keep empty bags on
                // non matched as outer (innerFlags[i] would be
                // false)
                if(!(innerFlags[i])) {
                    Operator joinInput = inputs.get(i);
                    // for outer join add a bincond
                    // which will project nulls when bag is
                    // empty
                    updateWithEmptyBagCheck(fep1, joinInput);
                }
View Full Code Here

        PODistinct physOp = new PODistinct(new OperatorKey(scope,nodeGen.getNextNodeId(scope)), loDistinct.getRequestedParallelisam());
        physOp.setAlias(loDistinct.getAlias());
        currentPlan.add(physOp);
        physOp.setResultType(DataType.BAG);
        logToPhyMap.put(loDistinct, physOp);
        Operator op = loDistinct.getPlan().getPredecessors(loDistinct).get(0);

        PhysicalOperator from = logToPhyMap.get(op);
        try {
            currentPlan.connect(from, physOp);
        } catch (PlanException e) {
View Full Code Here

        physOp.setLimit(loLimit.getLimit());
        physOp.setAlias(loLimit.getAlias());
        currentPlan.add(physOp);
        physOp.setResultType(DataType.BAG);
        logToPhyMap.put(loLimit, physOp);
        Operator op = loLimit.getPlan().getPredecessors(loLimit).get(0);

        PhysicalOperator from = logToPhyMap.get(op);
        try {
            currentPlan.connect(from, physOp);
        } catch (PlanException e) {
View Full Code Here

        LogicalRelationalOperator referred = project.findReferent();
        List<Operator> srcs = referred.getPlan().getSources();
        List<LOInnerLoad> innerLoads = new ArrayList<LOInnerLoad>();
        for (Operator src:srcs) {
            if (src instanceof LOInnerLoad) {
                Operator succ = src;
                while (succ!=null) {
                    if (succ==referred)
                        innerLoads.add((LOInnerLoad)src);
                    if (referred.getPlan().getSuccessors(succ)==null)
                        break;
View Full Code Here

       
        // Loader does not support column pruning, insert foreach     
        if (columnPrune) {
            if (response==null || !response.getRequiredFieldResponse()) {
                LogicalPlan p = (LogicalPlan)load.getPlan();                       
                Operator next = p.getSuccessors(load).get(0);
                // if there is already a LOForEach after load, we don't need to
                // add another LOForEach
                if (next instanceof LOForEach) {
                    return;
                }
View Full Code Here

    @SuppressWarnings("unchecked")
    @Override
    public void visit(LOSplit split) throws FrontendException {
        List<Operator> branchOutputs = split.getPlan().getSuccessors(split);
        for (int i=0;i<branchOutputs.size();i++) {
            Operator branchOutput = branchOutputs.get(i);
            Set<Long> branchOutputUids = (Set<Long>)branchOutput.getAnnotation(ColumnPruneHelper.INPUTUIDS);
           
            if (branchOutputUids!=null) {
                Set<Integer> columnsToDrop = new HashSet<Integer>();
               
                for (int j=0;j<split.getSchema().size();j++) {
View Full Code Here

TOP

Related Classes of org.apache.pig.newplan.Operator

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.