Package org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POMultiQueryPackage


        demux.setInCombiner(inCombiner);
        return demux;
    }
   
    private POMultiQueryPackage getMultiQueryPackage(boolean sameMapKeyType, boolean inCombiner){
        POMultiQueryPackage pkg = 
            new POMultiQueryPackage(new OperatorKey(scope, nig.getNextNodeId(scope)));
        pkg.setInCombiner(inCombiner);
        pkg.setSameMapKeyType(sameMapKeyType);
        return pkg;
    }  
View Full Code Here


        pkgOp.setBaseIndex(index++);
       
        List<POPackage> pkgs = pkgOp.getPackages();
        for (POPackage pkg : pkgs) {           
            if (pkg instanceof POMultiQueryPackage) {
                POMultiQueryPackage mpkg = (POMultiQueryPackage)pkg;
                index = setBaseIndexOnPackage(index, mpkg);
            } else {
                index++;
            }
        }
View Full Code Here

            // with the new indexed key
            addShiftedKeyInfoIndex(initial, pk);
        }
       
        if (pk instanceof POMultiQueryPackage) {
            POMultiQueryPackage mpkg = (POMultiQueryPackage)pk;
            setBaseIndexOnPackage(initial, mpkg);
            // we should update the keyinfo map of the
            // POPackage objects in the POMultiQueryPackage to
            // have the shifted index - The index now will be
            // starting from "initial" going up to "current"
            // ORed with the multi query bit mask
            int retIndex = addShiftedKeyInfoIndex(initial, current, mpkg);
            if(retIndex != current) {
                int errCode = 2146;
                String msg = "Internal Error. Inconsistency in key index found during optimization.";
                throw new OptimizerException(msg, errCode, PigException.BUG);
            }
        }
                               
        PhysicalOperator root = from.getRoots().get(0);
        if (root instanceof PODemux) {
            PODemux demux = (PODemux)root;
            setBaseIndexOnDemux(initial, demux);
        }
                   
        POMultiQueryPackage pkg = (POMultiQueryPackage)to.getRoots().get(0);       
        for (int i=initial; i<current; i++) {
            pkg.addPackage(pk);
        }
       
        PODemux demux = (PODemux)to.getLeaves().get(0);
        for (int i=initial; i<current; i++) {
            demux.addPlan(from, mapKeyType);
        }
              
        if (demux.isSameMapKeyType()) {
            pkg.setKeyType(pk.getKeyType());
        } else {
            pkg.setKeyType(DataType.TUPLE);
        }               
    }
View Full Code Here

            PhysicalPlan to, int initial, int current, byte mapKeyType) throws VisitorException {
        POPackage cpk = (POPackage)from.getRoots().get(0);
        from.remove(cpk);
      
        if (cpk instanceof POMultiQueryPackage) {
            POMultiQueryPackage mpkg = (POMultiQueryPackage)cpk;
            setBaseIndexOnPackage(initial, mpkg);
        }
       
        PODemux demux = (PODemux)to.getLeaves().get(0);
       
        boolean isSameKeyType = demux.isSameMapKeyType();
       
        PhysicalOperator leaf = from.getLeaves().get(0);
        if (leaf instanceof POLocalRearrange) {
            POLocalRearrange clr = (POLocalRearrange)leaf;
            try {
                clr.setMultiQueryIndex(initial);           
            } catch (ExecException e) {                                       
                int errCode = 2136;
                String msg = "Internal Error. Unable to set multi-query index for optimization.";
                throw new OptimizerException(msg, errCode, PigException.BUG, e);
            }
           
            // change the map key type to tuple when
            // multiple splittees have different map key types
            if (!isSameKeyType) {
                clr.setKeyType(DataType.TUPLE);
            }
        } else if (leaf instanceof PODemux) {
            PODemux locDemux = (PODemux)leaf;
            setBaseIndexOnDemux(initial, locDemux);
        }
      
        POMultiQueryPackage pkg = (POMultiQueryPackage)to.getRoots().get(0);
       
        // if current > initial + 1, it means we had
        // a split in the map of the MROper we are trying to
        // merge. In that case we would have changed the indices
        // of the POLocalRearranges in the split to be in the
        // range initial to current. To handle key, value pairs
        // coming out of those POLocalRearranges, we replicate
        // the Package as many times (in this case, the package
        // would have to be a POMultiQueryPackage since we had
        // a POSplit in the map). That Package would have a baseindex
        // correctly set (in the beginning of this method) and would
        // be able to handle the outputs from the different
        // POLocalRearranges.
        for (int i=initial; i<current; i++) {
            pkg.addPackage(cpk);
        }
       
        // all packages should have the same key type
        if (!isSameKeyType) {
            cpk.setKeyType(DataType.TUPLE);         
        }
       
        pkg.setKeyType(cpk.getKeyType());
       
        // See comment above for why we replicated the Package
        // in the from plan - for the same reason, we replicate
        // the Demux operators now.
        for (int i=initial; i<current; i++) {
View Full Code Here

    }
      
    private PhysicalPlan createDemuxPlan(boolean sameKeyType, boolean isCombiner)
        throws VisitorException {
        PODemux demux = getDemux(sameKeyType, isCombiner);
        POMultiQueryPackage pkg= getMultiQueryPackage();
       
        PhysicalPlan pl = new PhysicalPlan();
        pl.add(pkg);
        try {
            pl.addAsLeaf(demux);
View Full Code Here

        demux.setInCombiner(inCombiner);
        return demux;
    }
   
    private POMultiQueryPackage getMultiQueryPackage(){
        return new POMultiQueryPackage(new OperatorKey(scope, nig.getNextNodeId(scope)));
   
View Full Code Here

            addShiftedKeyInfoIndex(initial, pk);
        }
        
        int total = current - initial;
       
        POMultiQueryPackage pkg = (POMultiQueryPackage)to.getRoots().get(0);       
        int pkCount = 0;
        if (pk instanceof POMultiQueryPackage) {
            List<POPackage> pkgs = ((POMultiQueryPackage)pk).getPackages();
            for (POPackage p : pkgs) {
                pkg.addPackage(p);
                pkCount++;
            }
            pkg.addIsKeyWrappedList(((POMultiQueryPackage)pk).getIsKeyWrappedList());
            addShiftedKeyInfoIndex(initial, current, (POMultiQueryPackage)pk);
        } else {
            pkg.addPackage(pk, mapKeyType);
            pkCount = 1;
        }
       
        if (pkCount != total) {
            int errCode = 2146;
            String msg = "Internal Error. Inconsistency in key index found during optimization.";
            throw new OptimizerException(msg, errCode, PigException.BUG);
        }
       
        PODemux demux = (PODemux)to.getLeaves().get(0);
        int plCount = 0;
        PhysicalOperator root = from.getRoots().get(0);
        if (root instanceof PODemux) {
            // flattening the inner plans of the demux operator.
            // This is based on the fact that if a plan has a demux
            // operator, then it's the only operator in the plan.
            List<PhysicalPlan> pls = ((PODemux)root).getPlans();
            for (PhysicalPlan pl : pls) {
                demux.addPlan(pl);
                plCount++;
            }
        } else {
            demux.addPlan(from);
            plCount = 1;
        }
       
        if (plCount != total) {
            int errCode = 2146;
            String msg = "Internal Error. Inconsistency in key index found during optimization.";
            throw new OptimizerException(msg, errCode, PigException.BUG);
        }

        if (pkg.isSameMapKeyType()) {
            pkg.setKeyType(pk.getKeyType());
        } else {
            pkg.setKeyType(DataType.TUPLE);
        }           
    }
View Full Code Here

        POPackage cpk = (POPackage)from.getRoots().get(0);
        from.remove(cpk);
       
        PODemux demux = (PODemux)to.getLeaves().get(0);
               
        POMultiQueryPackage pkg = (POMultiQueryPackage)to.getRoots().get(0);
       
        boolean isSameKeyType = pkg.isSameMapKeyType();
       
        // if current > initial + 1, it means we had
        // a split in the map of the MROper we are trying to
        // merge. In that case we would have changed the indices
        // of the POLocalRearranges in the split to be in the
        // range initial to current. To handle key, value pairs
        // coming out of those POLocalRearranges, we add
        // the Packages in the 'from' POMultiQueryPackage (in this case,
        // it has to be a POMultiQueryPackage since we had
        // a POSplit in the map) to the 'to' POMultiQueryPackage.
        // These Packages would have correct positions in the package
        // list and would be able to handle the outputs from the different
        // POLocalRearranges.
        int total = current - initial;
        int pkCount = 0;
        if (cpk instanceof POMultiQueryPackage) {
            List<POPackage> pkgs = ((POMultiQueryPackage)cpk).getPackages();
            for (POPackage p : pkgs) {
                pkg.addPackage(p);
                if (!isSameKeyType) {
                    p.setKeyType(DataType.TUPLE);
                }
                pkCount++;
            }
        } else {
            pkg.addPackage(cpk);
            pkCount = 1;
        }

        pkg.setSameMapKeyType(isSameKeyType);
       
        if (pkCount != total) {
            int errCode = 2146;
            String msg = "Internal Error. Inconsistency in key index found during optimization.";
            throw new OptimizerException(msg, errCode, PigException.BUG);
        }

        // all packages should have the same key type
        if (!isSameKeyType) {
            cpk.setKeyType(DataType.TUPLE);         
        }
       
        pkg.setKeyType(cpk.getKeyType());
       
        // See comment above for why we flatten the Packages
        // in the from plan - for the same reason, we flatten
        // the inner plans of Demux operator now.
        int plCount = 0;
View Full Code Here

    }
      
    private PhysicalPlan createDemuxPlan(boolean sameKeyType, boolean isCombiner)
        throws VisitorException {
        PODemux demux = getDemux(isCombiner);
        POMultiQueryPackage pkg= getMultiQueryPackage(sameKeyType, isCombiner);
       
        PhysicalPlan pl = new PhysicalPlan();
        pl.add(pkg);
        try {
            pl.addAsLeaf(demux);
View Full Code Here

        demux.setInCombiner(inCombiner);
        return demux;
    }
   
    private POMultiQueryPackage getMultiQueryPackage(boolean sameMapKeyType, boolean inCombiner){
        POMultiQueryPackage pkg = 
            new POMultiQueryPackage(new OperatorKey(scope, nig.getNextNodeId(scope)));
        pkg.setInCombiner(inCombiner);
        pkg.setSameMapKeyType(sameMapKeyType);
        return pkg;
    }  
View Full Code Here

TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POMultiQueryPackage

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.