Package org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage


    }

    private PhysicalPlan createDemuxPlan(boolean sameKeyType, boolean isCombiner)
        throws VisitorException {
        PODemux demux = getDemux(isCombiner);
        POPackage pkg = getMultiQueryPackage(sameKeyType, isCombiner);

        PhysicalPlan pl = new PhysicalPlan();
        pl.add(pkg);
        try {
            pl.addAsLeaf(demux);
View Full Code Here


        return demux;
    }

    private POPackage getMultiQueryPackage(boolean sameMapKeyType,
            boolean inCombiner) {
        POPackage pkg = new POPackage(new OperatorKey(scope,
                nig.getNextNodeId(scope)));
        MultiQueryPackager pkgr = new MultiQueryPackager();
        pkgr.setInCombiner(inCombiner);
        pkgr.setSameMapKeyType(sameMapKeyType);
        pkg.setPkgr(pkgr);
        return pkg;
    }
View Full Code Here

                {
                    int errorCode = 2214;
                    throw new VisitorException("Cannot find POLocalRearrange to set secondary plan", errorCode);
                }
            }
            POPackage pack = (POPackage) root;
            pack.getPkgr().setUseSecondaryKey(true);
        }
        return secKeyOptimizerInfo;
    }
View Full Code Here

                MultiMap<Integer, LogicalExpressionPlan> expressionPlans = new MultiMap<Integer, LogicalExpressionPlan>();
                for(int i = 0 ; i < loRank.getRankColPlans().size() ; i++)
                    expressionPlans.put(i,loRank.getRankColPlans());

                POPackage poPackage = compileToLR_GR_PackTrio(loRank, null, flags, expressionPlans);
                poPackage.getPkgr().setPackageType(PackageType.GROUP);
                translateSoftLinks(loRank);

                List<Boolean> flattenLst = Arrays.asList(true, false);

                PhysicalPlan fep1 = new PhysicalPlan();
                POProject feproj1 = new POProject(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), -1);
                feproj1.addOriginalLocation(loRank.getAlias(), loRank.getLocation());
                feproj1.setColumn(0);
                feproj1.setResultType(poPackage.getPkgr().getKeyType());
                feproj1.setStar(false);
                feproj1.setOverloaded(false);
                fep1.add(feproj1);

View Full Code Here

        } else {
            POGlobalRearrange poGlobal = new POGlobalRearrange(new OperatorKey(
                    scope, nodeGen.getNextNodeId(scope)), cross
                    .getRequestedParallelism());
            poGlobal.addOriginalLocation(cross.getAlias(), cross.getLocation());
            POPackage poPackage = new POPackage(new OperatorKey(scope, nodeGen
                    .getNextNodeId(scope)), cross.getRequestedParallelism());
            poGlobal.addOriginalLocation(cross.getAlias(), cross.getLocation());
            poGlobal.setCross(true);
            currentPlan.add(poGlobal);
            currentPlan.add(poPackage);

            int count = 0;

            try {
                currentPlan.connect(poGlobal, poPackage);
                List<Boolean> flattenLst = Arrays.asList(true, true);

                for (Operator op : inputs) {
                    PhysicalPlan fep1 = new PhysicalPlan();
                    ConstantExpression ce1 = new ConstantExpression(new OperatorKey(scope, nodeGen.getNextNodeId(scope)),cross.getRequestedParallelism());
                    ce1.setValue(inputs.size());
                    ce1.setResultType(DataType.INTEGER);
                    fep1.add(ce1);

                    ConstantExpression ce2 = new ConstantExpression(new OperatorKey(scope, nodeGen.getNextNodeId(scope)),cross.getRequestedParallelism());
                    ce2.setValue(count);
                    ce2.setResultType(DataType.INTEGER);
                    fep1.add(ce2);
                    /*Tuple ce1val = TupleFactory.getInstance().newTuple(2);
                    ce1val.set(0,inputs.size());
                    ce1val.set(1,count);
                    ce1.setValue(ce1val);
                    ce1.setResultType(DataType.TUPLE);*/

                    POUserFunc gfc = new POUserFunc(new OperatorKey(scope, nodeGen.getNextNodeId(scope)),cross.getRequestedParallelism(),
                            Arrays.asList((PhysicalOperator)ce1,(PhysicalOperator)ce2), new FuncSpec(GFCross.class.getName()
                            + "('" + poGlobal.getOperatorKey().toString() + "')"));
                    gfc.addOriginalLocation(cross.getAlias(), cross.getLocation());
                    gfc.setResultType(DataType.BAG);
                    fep1.addAsLeaf(gfc);
                    gfc.setInputs(Arrays.asList((PhysicalOperator)ce1,(PhysicalOperator)ce2));
                    /*fep1.add(gfc);
                    fep1.connect(ce1, gfc);
                    fep1.connect(ce2, gfc);*/

                    PhysicalPlan fep2 = new PhysicalPlan();
                    POProject feproj = new POProject(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), cross.getRequestedParallelism());
                    feproj.addOriginalLocation(cross.getAlias(), cross.getLocation());
                    feproj.setResultType(DataType.TUPLE);
                    feproj.setStar(true);
                    feproj.setOverloaded(false);
                    fep2.add(feproj);
                    List<PhysicalPlan> fePlans = Arrays.asList(fep1, fep2);

                    POForEach fe = new POForEach(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), cross.getRequestedParallelism(), fePlans, flattenLst );
                    fe.setMapSideOnly(true);
                    fe.addOriginalLocation(cross.getAlias(), cross.getLocation());
                    currentPlan.add(fe);
                    currentPlan.connect(logToPhyMap.get(op), fe);

                    POLocalRearrange physOp = new POLocalRearrange(new OperatorKey(
                            scope, nodeGen.getNextNodeId(scope)), cross
                            .getRequestedParallelism());
                    physOp.addOriginalLocation(cross.getAlias(), cross.getLocation());
                    List<PhysicalPlan> lrPlans = new ArrayList<PhysicalPlan>();
                    for(int i=0;i<inputs.size();i++){
                        PhysicalPlan lrp1 = new PhysicalPlan();
                        POProject lrproj1 = new POProject(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), cross.getRequestedParallelism(), i);
                        lrproj1.addOriginalLocation(cross.getAlias(), cross.getLocation());
                        lrproj1.setOverloaded(false);
                        lrproj1.setResultType(DataType.INTEGER);
                        lrp1.add(lrproj1);
                        lrPlans.add(lrp1);
                    }

                    physOp.setCross(true);
                    physOp.setIndex(count++);
                    physOp.setKeyType(DataType.TUPLE);
                    physOp.setPlans(lrPlans);
                    physOp.setResultType(DataType.TUPLE);

                    currentPlan.add(physOp);
                    currentPlan.connect(fe, physOp);
                    currentPlan.connect(physOp, poGlobal);
                }
            } catch (PlanException e1) {
                int errCode = 2015;
                String msg = "Invalid physical operators in the physical plan" ;
                throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e1);
            } catch (ExecException e) {
                int errCode = 2058;
                String msg = "Unable to set index on newly create POLocalRearrange.";
                throw new VisitorException(msg, errCode, PigException.BUG, e);
            }

            poPackage.getPkgr().setKeyType(DataType.TUPLE);
            poPackage.setResultType(DataType.TUPLE);
            poPackage.setNumInps(count);
            boolean inner[] = new boolean[count];
            for (int i=0;i<count;i++) {
                inner[i] = true;
            }
            poPackage.getPkgr().setInner(inner);

            List<PhysicalPlan> fePlans = new ArrayList<PhysicalPlan>();
            List<Boolean> flattenLst = new ArrayList<Boolean>();
            for(int i=1;i<=count;i++){
                PhysicalPlan fep1 = new PhysicalPlan();
View Full Code Here

        switch (cg.getGroupType()) {
        case COLLECTED:
            translateCollectedCogroup(cg);
            break;
        case REGULAR:
            POPackage poPackage = compileToLR_GR_PackTrio(cg, cg.getCustomPartitioner(), cg.getInner(), cg.getExpressionPlans());
            poPackage.getPkgr().setPackageType(PackageType.GROUP);
            logToPhyMap.put(cg, poPackage);
            break;
        case MERGE:
            translateMergeCogroup(cg);
            break;
View Full Code Here

            }

            return;
        }
        else if (loj.getJoinType() == LOJoin.JOINTYPE.HASH){
            POPackage poPackage = compileToLR_GR_PackTrio(loj, loj.getCustomPartitioner(), innerFlags, loj.getExpressionPlans());
            POForEach fe = compileFE4Flattening(innerFlags,  scope, parallel, alias, location, inputs);
            currentPlan.add(fe);
            try {
                currentPlan.connect(poPackage, fe);
            } catch (PlanException e) {
                throw new LogicalToPhysicalTranslatorException(e.getDetailedMessage(),
                        e.getErrorCode(),e.getErrorSource(),e);
            }
            logToPhyMap.put(loj, fe);
            poPackage.getPkgr().setPackageType(PackageType.JOIN);
        }
        translateSoftLinks(loj);
    }
View Full Code Here

        POGlobalRearrange poGlobal = new POGlobalRearrange(new OperatorKey(
                DEFAULT_SCOPE, nodeGen.getNextNodeId(DEFAULT_SCOPE)), relationalOp.getRequestedParallelism());
        poGlobal.addOriginalLocation(relationalOp.getAlias(), relationalOp.getLocation());
        poGlobal.setCustomPartitioner(customPartitioner);
        POPackage poPackage = new POPackage(new OperatorKey(DEFAULT_SCOPE, nodeGen
                .getNextNodeId(DEFAULT_SCOPE)), relationalOp.getRequestedParallelism());
        poPackage.addOriginalLocation(relationalOp.getAlias(), relationalOp.getLocation());
        currentPlan.add(poGlobal);
        currentPlan.add(poPackage);

        try {
            currentPlan.connect(poGlobal, poPackage);
        } catch (PlanException e1) {
            int errCode = 2015;
            String msg = "Invalid physical operators in the physical plan" ;
            throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e1);
        }

        int count = 0;
        Byte type = null;
        List<Operator> inputs = relationalOp.getPlan().getPredecessors(relationalOp);
        for (int i=0;i<inputs.size();i++) {
            Operator op = inputs.get(i);
            List<LogicalExpressionPlan> plans = innerPlans.get(i);
            POLocalRearrange physOp = new POLocalRearrange(new OperatorKey(
                    DEFAULT_SCOPE, nodeGen.getNextNodeId(DEFAULT_SCOPE)), relationalOp.getRequestedParallelism());
            physOp.addOriginalLocation(relationalOp.getAlias(), relationalOp.getLocation());
            List<PhysicalPlan> exprPlans = translateExpressionPlans(relationalOp, plans);
            try {
                physOp.setPlans(exprPlans);
            } catch (PlanException pe) {
                int errCode = 2071;
                String msg = "Problem with setting up local rearrange's plans.";
                throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, pe);
            }
            try {
                physOp.setIndex(count++);
            } catch (ExecException e1) {
                int errCode = 2058;
                String msg = "Unable to set index on newly create POLocalRearrange.";
                throw new VisitorException(msg, errCode, PigException.BUG, e1);
            }
            if (plans.size() > 1) {
                type = DataType.TUPLE;
                physOp.setKeyType(type);
            } else {
                type = exprPlans.get(0).getLeaves().get(0).getResultType();
                physOp.setKeyType(type);
            }
            physOp.setResultType(DataType.TUPLE);

            currentPlan.add(physOp);

            try {
                currentPlan.connect(logToPhyMap.get(op), physOp);
                currentPlan.connect(physOp, poGlobal);
            } catch (PlanException e) {
                int errCode = 2015;
                String msg = "Invalid physical operators in the physical plan" ;
                throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
            }
        }

        poPackage.getPkgr().setKeyType(type);
        poPackage.setResultType(DataType.TUPLE);
        poPackage.setNumInps(count);
        poPackage.getPkgr().setInner(innerFlags);
        return poPackage;
    }
View Full Code Here

            op = mrRoots.get(0);
            if (!(op instanceof POPackage)) {
                return;
            }
            POPackage pack = (POPackage)op;

            List<PhysicalOperator> sucs = mr.reducePlan.getSuccessors(pack);
            if (sucs == null || sucs.size()!=1) {
                return;
            }

            op = sucs.get(0);
            boolean lastInputFlattened = true;
            boolean allSimple = true;
            if (op instanceof POForEach)
            {
                POForEach forEach = (POForEach)op;
                List<PhysicalPlan> planList = forEach.getInputPlans();
                List<Boolean> flatten = forEach.getToBeFlattened();
                POProject projOfLastInput = null;
                int i = 0;
                // check all nested foreach plans
                // 1. If it is simple projection
                // 2. If last input is all flattened
                for (PhysicalPlan p:planList)
                {
                    PhysicalOperator opProj = p.getRoots().get(0);
                    if (!(opProj instanceof POProject))
                    {
                        allSimple = false;
                        break;
                    }
                    POProject proj = (POProject)opProj;
                    // the project should just be for one column
                    // from the input
                    if(proj.isProjectToEnd() || proj.getColumns().size() != 1) {
                        allSimple = false;
                        break;
                    }

                    try {
                        // if input to project is the last input
                        if (proj.getColumn() == pack.getNumInps())
                        {
                            // if we had already seen another project
                            // which was also for the last input, then
                            // we might be trying to flatten twice on the
                            // last input in which case we can't optimize by
View Full Code Here

        // be present only in the reduce plan. Search in these two
        // plans accordingly
        if(!mr.combinePlan.isEmpty()) {
            PackageDiscoverer pkgDiscoverer = new PackageDiscoverer(mr.combinePlan);
            pkgDiscoverer.visit();
            POPackage pkg = pkgDiscoverer.getPkg();
            if(pkg != null) {
                handlePackage(mr, pkg);
            }
        }

        if(!mr.reducePlan.isEmpty()) {
            PackageDiscoverer pkgDiscoverer = new PackageDiscoverer(mr.reducePlan);
            pkgDiscoverer.visit();
            POPackage pkg = pkgDiscoverer.getPkg();
            if(pkg != null) {
                // if the POPackage is actually a POPostCombinerPackage, then we should
                // just look for the corresponding LocalRearrange(s) in the combine plan
                if (pkg.getPkgr() instanceof CombinerPackager) {
                    if(patchPackage(mr.combinePlan, pkg) != pkg.getNumInps()) {
                        int errCode = 2085;
                        String msg = "Unexpected problem during optimization." +
                        " Could not find LocalRearrange in combine plan.";
                        throw new OptimizerException(msg, errCode, PigException.BUG);
                    }
View Full Code Here

TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.