Package org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.MultiQueryPackager


    private POPackage getMultiQueryPackage(boolean sameMapKeyType,
            boolean inCombiner) {
        POPackage pkg = new POPackage(new OperatorKey(scope,
                nig.getNextNodeId(scope)));
        MultiQueryPackager pkgr = new MultiQueryPackager();
        pkgr.setInCombiner(inCombiner);
        pkgr.setSameMapKeyType(sameMapKeyType);
        pkg.setPkgr(pkgr);
        return pkg;
    }
View Full Code Here


            addShiftedKeyInfoIndex(initial, fromPkgr);
        }

        int total = current - initial;

        MultiQueryPackager toPkgr = (MultiQueryPackager) ((POPackage) to
                .getRoots().get(0)).getPkgr();
        int pkCount = 0;
        if (fromPkgr instanceof MultiQueryPackager) {
            List<Packager> pkgs = ((MultiQueryPackager) fromPkgr)
                    .getPackagers();
            for (Packager p : pkgs) {
                ((MultiQueryPackager) toPkgr).addPackager(p);
                pkCount++;
            }
            toPkgr.addIsKeyWrappedList(((MultiQueryPackager) fromPkgr)
                            .getIsKeyWrappedList());
            addShiftedKeyInfoIndex(initial, current,
                    (MultiQueryPackager) fromPkgr);
        } else {
            toPkgr.addPackager(fromPkgr, mapKeyType);
            pkCount = 1;
        }

        if (pkCount != total) {
            int errCode = 2146;
            String msg = "Internal Error. Inconsistency in key index found during optimization.";
            throw new OptimizerException(msg, errCode, PigException.BUG);
        }

        PODemux demux = (PODemux)to.getLeaves().get(0);
        int plCount = 0;
        PhysicalOperator root = from.getRoots().get(0);
        if (root instanceof PODemux) {
            // flattening the inner plans of the demux operator.
            // This is based on the fact that if a plan has a demux
            // operator, then it's the only operator in the plan.
            List<PhysicalPlan> pls = ((PODemux)root).getPlans();
            for (PhysicalPlan pl : pls) {
                demux.addPlan(pl);
                plCount++;
            }
        } else {
            demux.addPlan(from);
            plCount = 1;
        }

        if (plCount != total) {
            int errCode = 2146;
            String msg = "Internal Error. Inconsistency in key index found during optimization.";
            throw new OptimizerException(msg, errCode, PigException.BUG);
        }

        if (toPkgr.isSameMapKeyType()) {
            toPkgr.setKeyType(fromPkgr.getKeyType());
        } else {
            toPkgr.setKeyType(DataType.TUPLE);
        }
    }
View Full Code Here

        from.remove(cpk);
        Packager cpkgr = cpk.getPkgr();

        PODemux demux = (PODemux)to.getLeaves().get(0);

        MultiQueryPackager toPkgr = (MultiQueryPackager) ((POPackage) to
                .getRoots().get(0)).getPkgr();

        boolean isSameKeyType = toPkgr.isSameMapKeyType();

        // if current > initial + 1, it means we had
        // a split in the map of the MROper we are trying to
        // merge. In that case we would have changed the indices
        // of the POLocalRearranges in the split to be in the
        // range initial to current. To handle key, value pairs
        // coming out of those POLocalRearranges, we add
        // the Packages in the 'from' POMultiQueryPackage (in this case,
        // it has to be a POMultiQueryPackage since we had
        // a POSplit in the map) to the 'to' POMultiQueryPackage.
        // These Packages would have correct positions in the package
        // list and would be able to handle the outputs from the different
        // POLocalRearranges.
        int total = current - initial;
        int pkCount = 0;
        if (cpkgr instanceof MultiQueryPackager) {
            List<Packager> pkgrs = ((MultiQueryPackager) cpkgr).getPackagers();
            for (Packager p : pkgrs) {
                toPkgr.addPackager(p);
                if (!isSameKeyType) {
                    p.setKeyType(DataType.TUPLE);
                }
                pkCount++;
            }
        } else {
            toPkgr.addPackager(cpkgr);
            pkCount = 1;
        }

        toPkgr.setSameMapKeyType(isSameKeyType);

        if (pkCount != total) {
            int errCode = 2146;
            String msg = "Internal Error. Inconsistency in key index found during optimization.";
            throw new OptimizerException(msg, errCode, PigException.BUG);
        }

        // all packages should have the same key type
        if (!isSameKeyType) {
            cpk.getPkgr().setKeyType(DataType.TUPLE);
        }

        toPkgr.setKeyType(cpk.getPkgr().getKeyType());

        // See comment above for why we flatten the Packages
        // in the from plan - for the same reason, we flatten
        // the inner plans of Demux operator now.
        int plCount = 0;
View Full Code Here

        from.remove(cpk);
        Packager cpkgr = cpk.getPkgr();
       
        PODemux demux = (PODemux)to.getLeaves().get(0);
               
        MultiQueryPackager toPkgr = (MultiQueryPackager) ((POPackage) to
                .getRoots().get(0)).getPkgr();
       
        boolean isSameKeyType = toPkgr.isSameMapKeyType();
       
        // if current > initial + 1, it means we had
        // a split in the map of the MROper we are trying to
        // merge. In that case we would have changed the indices
        // of the POLocalRearranges in the split to be in the
        // range initial to current. To handle key, value pairs
        // coming out of those POLocalRearranges, we add
        // the Packages in the 'from' POMultiQueryPackage (in this case,
        // it has to be a POMultiQueryPackage since we had
        // a POSplit in the map) to the 'to' POMultiQueryPackage.
        // These Packages would have correct positions in the package
        // list and would be able to handle the outputs from the different
        // POLocalRearranges.
        int total = current - initial;
        int pkCount = 0;
        if (cpkgr instanceof MultiQueryPackager) {
            List<Packager> pkgrs = ((MultiQueryPackager) cpkgr).getPackagers();
            for (Packager p : pkgrs) {
                toPkgr.addPackager(p);
                if (!isSameKeyType) {
                    p.setKeyType(DataType.TUPLE);
                }
                pkCount++;
            }
        } else {
            toPkgr.addPackager(cpkgr);
            pkCount = 1;
        }

        toPkgr.setSameMapKeyType(isSameKeyType);
       
        if (pkCount != total) {
            int errCode = 2146;
            String msg = "Internal Error. Inconsistency in key index found during optimization.";
            throw new OptimizerException(msg, errCode, PigException.BUG);
        }

        // all packages should have the same key type
        if (!isSameKeyType) {
            cpk.getPkgr().setKeyType(DataType.TUPLE);
        }
       
        toPkgr.setKeyType(cpk.getPkgr().getKeyType());
       
        // See comment above for why we flatten the Packages
        // in the from plan - for the same reason, we flatten
        // the inner plans of Demux operator now.
        int plCount = 0;
View Full Code Here

   
    private POPackage getMultiQueryPackage(boolean sameMapKeyType,
            boolean inCombiner) {
        POPackage pkg = new POPackage(new OperatorKey(scope,
                nig.getNextNodeId(scope)));
        MultiQueryPackager pkgr = new MultiQueryPackager();
        pkgr.setInCombiner(inCombiner);
        pkgr.setSameMapKeyType(sameMapKeyType);
        pkg.setPkgr(pkgr);
        return pkg;
    }  
View Full Code Here

            addShiftedKeyInfoIndex(initial, fromPkgr);
        }
        
        int total = current - initial;
       
        MultiQueryPackager toPkgr = (MultiQueryPackager) ((POPackage) to
                .getRoots().get(0)).getPkgr();
        int pkCount = 0;
        if (fromPkgr instanceof MultiQueryPackager) {
            List<Packager> pkgs = ((MultiQueryPackager) fromPkgr)
                    .getPackagers();
            for (Packager p : pkgs) {
                ((MultiQueryPackager) toPkgr).addPackager(p);
                pkCount++;
            }
            toPkgr.addIsKeyWrappedList(((MultiQueryPackager) fromPkgr)
                            .getIsKeyWrappedList());
            addShiftedKeyInfoIndex(initial, current,
                    (MultiQueryPackager) fromPkgr);
        } else {
            toPkgr.addPackager(fromPkgr, mapKeyType);
            pkCount = 1;
        }
       
        if (pkCount != total) {
            int errCode = 2146;
            String msg = "Internal Error. Inconsistency in key index found during optimization.";
            throw new OptimizerException(msg, errCode, PigException.BUG);
        }
       
        PODemux demux = (PODemux)to.getLeaves().get(0);
        int plCount = 0;
        PhysicalOperator root = from.getRoots().get(0);
        if (root instanceof PODemux) {
            // flattening the inner plans of the demux operator.
            // This is based on the fact that if a plan has a demux
            // operator, then it's the only operator in the plan.
            List<PhysicalPlan> pls = ((PODemux)root).getPlans();
            for (PhysicalPlan pl : pls) {
                demux.addPlan(pl);
                plCount++;
            }
        } else {
            demux.addPlan(from);
            plCount = 1;
        }
       
        if (plCount != total) {
            int errCode = 2146;
            String msg = "Internal Error. Inconsistency in key index found during optimization.";
            throw new OptimizerException(msg, errCode, PigException.BUG);
        }

        if (toPkgr.isSameMapKeyType()) {
            toPkgr.setKeyType(fromPkgr.getKeyType());
        } else {
            toPkgr.setKeyType(DataType.TUPLE);
        }           
    }
View Full Code Here

TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.MultiQueryPackager

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.