Examples of POForEach


Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach

                feproj.setStar(true);
                feproj.setOverloaded(false);
                fep2.add(feproj);
                List<PhysicalPlan> fePlans = Arrays.asList(fep1, fep2);
               
                POForEach fe = new POForEach(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), cross.getRequestedParallelisam(), fePlans, flattenLst );
                fe.setAlias(cross.getAlias());
                currentPlan.add(fe);
                currentPlan.connect(logToPhyMap.get(op), fe);
               
                POLocalRearrange physOp = new POLocalRearrange(new OperatorKey(
                        scope, nodeGen.getNextNodeId(scope)), cross
                        .getRequestedParallelisam());
                physOp.setAlias(cross.getAlias());
                List<PhysicalPlan> lrPlans = new ArrayList<PhysicalPlan>();
                for(int i=0;i<inputs.size();i++){
                    PhysicalPlan lrp1 = new PhysicalPlan();
                    POProject lrproj1 = new POProject(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), cross.getRequestedParallelisam(), i);
                    lrproj1.setAlias(cross.getAlias());
                    lrproj1.setOverloaded(false);
                    lrproj1.setResultType(DataType.INTEGER);
                    lrp1.add(lrproj1);
                    lrPlans.add(lrp1);
                }
               
                physOp.setCross(true);
                physOp.setIndex(count++);
                physOp.setKeyType(DataType.TUPLE);
                physOp.setPlans(lrPlans);
                physOp.setResultType(DataType.TUPLE);
               
                currentPlan.add(physOp);
                currentPlan.connect(fe, physOp);
                currentPlan.connect(physOp, poGlobal);
            }
        } catch (PlanException e1) {
            int errCode = 2015;
            String msg = "Invalid physical operators in the physical plan" ;
            throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e1);
        } catch (ExecException e) {
            int errCode = 2058;
            String msg = "Unable to set index on newly create POLocalRearrange.";
            throw new VisitorException(msg, errCode, PigException.BUG, e);
        }
       
        poPackage.setKeyType(DataType.TUPLE);
        poPackage.setResultType(DataType.TUPLE);
        poPackage.setNumInps(count);
        boolean inner[] = new boolean[count];
        for (int i=0;i<count;i++) {
            inner[i] = true;
        }
        poPackage.setInner(inner);
       
        List<PhysicalPlan> fePlans = new ArrayList<PhysicalPlan>();
        List<Boolean> flattenLst = new ArrayList<Boolean>();
        for(int i=1;i<=count;i++){
            PhysicalPlan fep1 = new PhysicalPlan();
            POProject feproj1 = new POProject(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), cross.getRequestedParallelisam(), i);
            feproj1.setAlias(cross.getAlias());
            feproj1.setResultType(DataType.BAG);
            feproj1.setOverloaded(false);
            fep1.add(feproj1);
            fePlans.add(fep1);
            flattenLst.add(true);
        }
       
        POForEach fe = new POForEach(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), cross.getRequestedParallelisam(), fePlans, flattenLst );
        fe.setAlias(cross.getAlias());
        currentPlan.add(fe);
        try{
            currentPlan.connect(poPackage, fe);
        }catch (PlanException e1) {
            int errCode = 2015;
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach

        boolean[] flatten = gen.getFlattenFlags();
        List<Boolean> flattenList = new ArrayList<Boolean>();
        for(boolean fl: flatten) {
            flattenList.add(fl);
        }
        POForEach poFE = new POForEach(new OperatorKey(scope, nodeGen
                .getNextNodeId(scope)), foreach.getRequestedParallelisam(), innerPlans, flattenList);
        poFE.setAlias(foreach.getAlias());
        poFE.setResultType(DataType.BAG);
        logToPhyMap.put(foreach, poFE);
        currentPlan.add(poFE);

        // generate cannot have multiple inputs
        List<Operator> op = foreach.getPlan().getPredecessors(foreach);
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach

            }

            if(!usePOMergeJoin){
                // Now create and configure foreach which will flatten the output
                // of cogroup.
                POForEach fe = compileFE4Flattening(innerFlags,  scope, parallel, alias, inputs);
                currentPlan.add(fe);
                try {
                    currentPlan.connect(smj, fe);
                } catch (PlanException e) {
                    throw new LogicalToPhysicalTranslatorException(e.getMessage(),e.getErrorCode(),e.getErrorSource(),e);
                }
                logToPhyMap.put(loj, fe);               
            }
           
            return;
        }
        else if (loj.getJoinType() == LOJoin.JOINTYPE.HASH){
            POPackage poPackage = compileToLR_GR_PackTrio(loj, loj.getCustomPartitioner(), innerFlags, loj.getExpressionPlans());
            POForEach fe = compileFE4Flattening(innerFlags,  scope, parallel, alias, inputs);
            currentPlan.add(fe);
            try {
                currentPlan.connect(poPackage, fe);
            } catch (PlanException e) {
                throw new LogicalToPhysicalTranslatorException(e.getDetailedMessage(),
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach

            int parallel, String alias, List<Operator> inputs)
                throws FrontendException {
       
        List<PhysicalPlan> fePlans = new ArrayList<PhysicalPlan>();
        List<Boolean> flattenLst = new ArrayList<Boolean>();
        POForEach fe;
        try{
            for(int i=0;i< inputs.size();i++){
                PhysicalPlan fep1 = new PhysicalPlan();
                POProject feproj1 = new POProject(new OperatorKey(scope, nodeGen.getNextNodeId(scope)),
                        parallel, i+1); //i+1 since the first column is the "group" field
                feproj1.setAlias(alias);
                feproj1.setResultType(DataType.BAG);
                feproj1.setOverloaded(false);
                fep1.add(feproj1);
                fePlans.add(fep1);
                // the parser would have marked the side
                // where we need to keep empty bags on
                // non matched as outer (innerFlags[i] would be
                // false)
                if(!(innerFlags[i])) {
                    Operator joinInput = inputs.get(i);
                    // for outer join add a bincond
                    // which will project nulls when bag is
                    // empty
                    updateWithEmptyBagCheck(fep1, joinInput);
                }
                flattenLst.add(true);
            }
           
            fe = new POForEach(new OperatorKey(scope, nodeGen.getNextNodeId(scope)),
                    parallel, fePlans, flattenLst );
            fe.setAlias(alias);

        }catch (PlanException e1) {
            int errCode = 2015;
            String msg = "Invalid physical operators in the physical plan" ;
            throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e1);
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach

        sample.append(new String("S"));
        sample.append(new String("x"));
        sample.append(new String("S"));
        sample.append(new String("x"));

        POForEach fe1 = GenPhyOp.topForEachOPWithPlan(flds , sample);
       
        POForEach fe2 = GenPhyOp.topForEachOPWithPlan(flds , sample);
       
        sp = GenPhyOp.topUnionOp();
       
        PhysicalPlan plan = new PhysicalPlan();
       
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach

                }
            }
            flattened.add(true);
        }
       
        POForEach fe = new POForEach(new OperatorKey(scope,nodeGen.getNextNodeId(scope)),-1,eps,flattened);
       
        fe.setResultType(DataType.BAG);

        currentPlan.add(fe);
        logToPhyMap.put(join, fe);
        try {
            currentPlan.connect(poc, fe);
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach

            op = sucs.get(0);
            boolean lastInputFlattened = true;
            boolean allSimple = true;
            if (op instanceof POForEach)
            {
                POForEach forEach = (POForEach)op;
                List<PhysicalPlan> planList = forEach.getInputPlans();
                List<Boolean> flatten = forEach.getToBeFlattened();
                POProject projOfLastInput = null;
                int i = 0;
                // check all nested foreach plans
                // 1. If it is simple projection
                // 2. If last input is all flattened
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach

        }

        // visit the POForEach of the reduce plan. We can have Limit and Filter
        // in the middle
        PhysicalOperator currentNode = root;
        POForEach foreach = null;
        while (currentNode != null) {
            if (currentNode instanceof POPackage && !(currentNode instanceof POJoinPackage)
                    || currentNode instanceof POFilter
                    || currentNode instanceof POLimit) {
                List<PhysicalOperator> succs = mr.reducePlan
                        .getSuccessors(currentNode);
                if (succs == null) // We didn't find POForEach
                    return;
                if (succs.size() != 1) {
                    log.debug("See multiple output for " + currentNode
                            + " in reduce plan, skip secondary key optimizing");
                    return;
                }
                currentNode = succs.get(0);
            } else if (currentNode instanceof POForEach) {
                foreach = (POForEach) currentNode;
                break;
            } else { // Skip optimization
                return;
            }
        }
       
        // We do not find a foreach (we shall not come here, a trick to fool findbugs)
        if (foreach==null)
            return;

        sortsToRemove = new ArrayList<POToChange>();
        distinctsToChange = new ArrayList<POToChange>();

        for (PhysicalPlan innerPlan : foreach.getInputPlans()) {
            // visit inner plans to figure out the sort order for distinct /
            // sort
            SecondaryKeyDiscover innerPlanDiscover = new SecondaryKeyDiscover(
                    innerPlan, sortKeyInfos, secondarySortKeyInfo);
            try {
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach

        prj1.setColumn(1);
        prj1.setOverloaded(true);
        ep1.add(prj1);
        eps1.add(ep1);
        flat1.add(true);
        POForEach fe = new POForEach(new OperatorKey(scope, nig
                .getNextNodeId(scope)), -1, eps1, flat1);
        fe.setResultType(DataType.BAG);
        return fe;
    }
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach

            prj1.setColumn(0);
            prj1.setOverloaded(false);
            ep1.add(prj1);
            eps1.add(ep1);
            flat1.add(true);
            POForEach nfe1 = new POForEach(new OperatorKey(scope, nig
                    .getNextNodeId(scope)), op.getRequestedParallelism(), eps1,
                    flat1);
            nfe1.setResultType(DataType.BAG);
            curMROp.reducePlan.addAsLeaf(nfe1);
            curMROp.setNeedsDistinctCombiner(true);
            phyToMROpMap.put(op, curMROp);
        }catch(Exception e){
            int errCode = 2034;
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.