Package org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.Packager


    private void mergeOneReducePlanWithIndex(PhysicalPlan from,
            PhysicalPlan to, int initial, int current, byte mapKeyType) throws VisitorException {
        POPackage pk = (POPackage)from.getRoots().get(0);
        from.remove(pk);
        Packager fromPkgr = pk.getPkgr();

        if (!(fromPkgr instanceof MultiQueryPackager)) {
            // XXX the index of the original keyInfo map is always 0,
            // we need to shift the index so that the lookups works
            // with the new indexed key
            addShiftedKeyInfoIndex(initial, fromPkgr);
        }

        int total = current - initial;

        MultiQueryPackager toPkgr = (MultiQueryPackager) ((POPackage) to
                .getRoots().get(0)).getPkgr();
        int pkCount = 0;
        if (fromPkgr instanceof MultiQueryPackager) {
            List<Packager> pkgs = ((MultiQueryPackager) fromPkgr)
                    .getPackagers();
            for (Packager p : pkgs) {
                ((MultiQueryPackager) toPkgr).addPackager(p);
                pkCount++;
            }
            toPkgr.addIsKeyWrappedList(((MultiQueryPackager) fromPkgr)
                            .getIsKeyWrappedList());
            addShiftedKeyInfoIndex(initial, current,
                    (MultiQueryPackager) fromPkgr);
        } else {
            toPkgr.addPackager(fromPkgr, mapKeyType);
            pkCount = 1;
        }

        if (pkCount != total) {
            int errCode = 2146;
            String msg = "Internal Error. Inconsistency in key index found during optimization.";
            throw new OptimizerException(msg, errCode, PigException.BUG);
        }

        PODemux demux = (PODemux)to.getLeaves().get(0);
        int plCount = 0;
        PhysicalOperator root = from.getRoots().get(0);
        if (root instanceof PODemux) {
            // flattening the inner plans of the demux operator.
            // This is based on the fact that if a plan has a demux
            // operator, then it's the only operator in the plan.
            List<PhysicalPlan> pls = ((PODemux)root).getPlans();
            for (PhysicalPlan pl : pls) {
                demux.addPlan(pl);
                plCount++;
            }
        } else {
            demux.addPlan(from);
            plCount = 1;
        }

        if (plCount != total) {
            int errCode = 2146;
            String msg = "Internal Error. Inconsistency in key index found during optimization.";
            throw new OptimizerException(msg, errCode, PigException.BUG);
        }

        if (toPkgr.isSameMapKeyType()) {
            toPkgr.setKeyType(fromPkgr.getKeyType());
        } else {
            toPkgr.setKeyType(DataType.TUPLE);
        }
    }
View Full Code Here


            throw new OptimizerException(msg, errCode, PigException.BUG);
        }
        int i = 0;
        int curIndex = initialIndex;
        while (i < end) {
            Packager pkg = pkgs.get(i);
            addShiftedKeyInfoIndex(curIndex, pkg);
            curIndex++;
            i++;
        }
        return curIndex; // could be used in a caller who recursively called this function
View Full Code Here

    private void mergeOneCombinePlanWithIndex(PhysicalPlan from,
            PhysicalPlan to, int initial, int current, byte mapKeyType) throws VisitorException {
        POPackage cpk = (POPackage)from.getRoots().get(0);
        from.remove(cpk);
        Packager cpkgr = cpk.getPkgr();

        PODemux demux = (PODemux)to.getLeaves().get(0);

        MultiQueryPackager toPkgr = (MultiQueryPackager) ((POPackage) to
                .getRoots().get(0)).getPkgr();
View Full Code Here

        PhysicalOperator po_package = pos.get(0);
        if (!po_package.getClass().equals(POPackage.class)) {
            return;
        }

        Packager pkgr = ((POPackage) po_package).getPkgr();
        // Check that this is a standard package, not a subclass
        if (!pkgr.getClass().equals(Packager.class)) {
            return;
        }

        // if POPackage is for distinct, just return
        if (pkgr.isDistinct()) {
            return;
        }

        // if any input to POPackage is inner, just return
        boolean[] isInner = pkgr.getInner();
        for (boolean b: isInner) {
            if (b) {
                return;
            }
        }
View Full Code Here

            blocking(op);
            curMROp.customPartitioner = op.getCustomPartitioner();

            POPackage pkg = new POPackage(new OperatorKey(scope,nig.getNextNodeId(scope)));
            Packager pkgr = pkg.getPkgr();
            pkgr.setKeyType(DataType.TUPLE);
            pkgr.setDistinct(true);
            pkg.setNumInps(1);
            boolean[] inner = {false};
            pkgr.setInner(inner);
            curMROp.reducePlan.add(pkg);

            List<PhysicalPlan> eps1 = new ArrayList<PhysicalPlan>();
            List<Boolean> flat1 = new ArrayList<Boolean>();
            PhysicalPlan ep1 = new PhysicalPlan();
View Full Code Here

                curMROp.requestedParallelism = gr.getRequestedParallelism();
            compiledInputs = new MapReduceOper[] {curMROp};

            // create POPakcage
            POPackage pkg = new POPackage(new OperatorKey(scope,nig.getNextNodeId(scope)), rp);
            Packager pkgr = pkg.getPkgr();
            pkgr.setKeyType(type);
            pkg.setResultType(DataType.TUPLE);
            pkg.setNumInps(2);
            boolean [] inner = op.getInnerFlags();
            pkgr.setInner(inner);
            pkg.visit(this);
            compiledInputs = new MapReduceOper[] {curMROp};

            // create POForEach
            List<PhysicalPlan> eps = new ArrayList<PhysicalPlan>();
View Full Code Here

        mro.mapPlan.connect(nfe1, lr);

        mro.setMapDone(true);

        POPackage pkg = new POPackage(new OperatorKey(scope,nig.getNextNodeId(scope)));
        Packager pkgr = new Packager();
        pkg.setPkgr(pkgr);
        pkgr.setKeyType(DataType.CHARARRAY);
        pkg.setNumInps(1);
        boolean[] inner = {false};
        pkgr.setInner(inner);
        mro.reducePlan.add(pkg);

        // Lets start building the plan which will have the sort
        // for the foreach
        PhysicalPlan fe2Plan = new PhysicalPlan();
View Full Code Here

       
    private void mergeOneReducePlanWithIndex(PhysicalPlan from,
            PhysicalPlan to, int initial, int current, byte mapKeyType) throws VisitorException {                   
        POPackage pk = (POPackage)from.getRoots().get(0);
        from.remove(pk);
        Packager fromPkgr = pk.getPkgr();
        if (!(fromPkgr instanceof MultiQueryPackager)) {
            // XXX the index of the original keyInfo map is always 0,
            // we need to shift the index so that the lookups works
            // with the new indexed key
            addShiftedKeyInfoIndex(initial, fromPkgr);
        }
        
        int total = current - initial;
       
        MultiQueryPackager toPkgr = (MultiQueryPackager) ((POPackage) to
                .getRoots().get(0)).getPkgr();
        int pkCount = 0;
        if (fromPkgr instanceof MultiQueryPackager) {
            List<Packager> pkgs = ((MultiQueryPackager) fromPkgr)
                    .getPackagers();
            for (Packager p : pkgs) {
                ((MultiQueryPackager) toPkgr).addPackager(p);
                pkCount++;
            }
            toPkgr.addIsKeyWrappedList(((MultiQueryPackager) fromPkgr)
                            .getIsKeyWrappedList());
            addShiftedKeyInfoIndex(initial, current,
                    (MultiQueryPackager) fromPkgr);
        } else {
            toPkgr.addPackager(fromPkgr, mapKeyType);
            pkCount = 1;
        }
       
        if (pkCount != total) {
            int errCode = 2146;
            String msg = "Internal Error. Inconsistency in key index found during optimization.";
            throw new OptimizerException(msg, errCode, PigException.BUG);
        }
       
        PODemux demux = (PODemux)to.getLeaves().get(0);
        int plCount = 0;
        PhysicalOperator root = from.getRoots().get(0);
        if (root instanceof PODemux) {
            // flattening the inner plans of the demux operator.
            // This is based on the fact that if a plan has a demux
            // operator, then it's the only operator in the plan.
            List<PhysicalPlan> pls = ((PODemux)root).getPlans();
            for (PhysicalPlan pl : pls) {
                demux.addPlan(pl);
                plCount++;
            }
        } else {
            demux.addPlan(from);
            plCount = 1;
        }
       
        if (plCount != total) {
            int errCode = 2146;
            String msg = "Internal Error. Inconsistency in key index found during optimization.";
            throw new OptimizerException(msg, errCode, PigException.BUG);
        }

        if (toPkgr.isSameMapKeyType()) {
            toPkgr.setKeyType(fromPkgr.getKeyType());
        } else {
            toPkgr.setKeyType(DataType.TUPLE);
        }           
    }
View Full Code Here

            throw new OptimizerException(msg, errCode, PigException.BUG);
        }
        int i = 0;
        int curIndex = initialIndex;
        while (i < end) {
            Packager pkg = pkgs.get(i);
            addShiftedKeyInfoIndex(curIndex, pkg);
            curIndex++;
            i++;
        }
        return curIndex; // could be used in a caller who recursively called this function
View Full Code Here

    private void mergeOneCombinePlanWithIndex(PhysicalPlan from,
            PhysicalPlan to, int initial, int current, byte mapKeyType) throws VisitorException {
        POPackage cpk = (POPackage)from.getRoots().get(0);
        from.remove(cpk);
        Packager cpkgr = cpk.getPkgr();
       
        PODemux demux = (PODemux)to.getLeaves().get(0);
               
        MultiQueryPackager toPkgr = (MultiQueryPackager) ((POPackage) to
                .getRoots().get(0)).getPkgr();
View Full Code Here

TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.Packager

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.