Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPartialAgg

Package org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPartialAgg

org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPartialAgg
Do partial aggregation in map plan. Inputs are buffered up in a hashmap until a threshold is reached; then the combiner functions are fed these buffered up inputs, and results stored in a secondary map. Once that map fills up or all input has been seen, results are piped out into the next operator (caller of getNext()).

                // No need to connect projections in cfe to cp, because
                // PigCombiner directly attaches output from package to
                // root of remaining plan.


                POLocalRearrange mlr = getNewRearrange(rearrange);
                POPartialAgg mapAgg = null;
                if (doMapAgg) {
                    mapAgg = createPartialAgg(cfe);
                }


                // A specialized local rearrange operator will replace

View Full Code Here

     * @return partial aggregate operator
     * @throws CloneNotSupportedException
     */
    private static POPartialAgg createPartialAgg(POForEach combineFE) throws CloneNotSupportedException {
        String scope = combineFE.getOperatorKey().scope;
        POPartialAgg poAgg = new POPartialAgg(new OperatorKey(scope,
                NodeIdGenerator.getGenerator().getNextNodeId(scope)));
        poAgg.addOriginalLocation(combineFE.getAlias(), combineFE.getOriginalLocations());
        poAgg.setResultType(combineFE.getResultType());


        // first plan in combine foreach is the group key
        poAgg.setKeyPlan(combineFE.getInputPlans().get(0).clone());


        List<PhysicalPlan> valuePlans = Lists.newArrayList();
        for (int i=1; i<combineFE.getInputPlans().size(); i++) {
            valuePlans.add(combineFE.getInputPlans().get(i).clone());
        }
        poAgg.setValuePlans(valuePlans);
        return poAgg;
    }

View Full Code Here

                // PigCombiner directly attaches output from package to
                // root of remaining plan.


                POLocalRearrange mlr = getNewRearrange(rearrange);


                POPartialAgg mapAgg = null;
                if(doMapAgg){
                    mapAgg = createPartialAgg(cfe);
                }


                // A specialized local rearrange operator will replace

View Full Code Here

     * @throws CloneNotSupportedException 
     */
    private POPartialAgg createPartialAgg(POForEach combineFE)
            throws CloneNotSupportedException {
        String scope = combineFE.getOperatorKey().scope;
        POPartialAgg poAgg = new POPartialAgg(new OperatorKey(scope, 
                NodeIdGenerator.getGenerator().getNextNodeId(scope)));
        poAgg.addOriginalLocation(combineFE.getAlias(), combineFE.getOriginalLocations());
        poAgg.setResultType(combineFE.getResultType());


        //first plan in combine foreach is the group key
        poAgg.setKeyPlan(combineFE.getInputPlans().get(0).clone());


        List<PhysicalPlan> valuePlans = new ArrayList<PhysicalPlan>();
        for(int i=1; i<combineFE.getInputPlans().size(); i++){
            valuePlans.add(combineFE.getInputPlans().get(i).clone());
        }
        poAgg.setValuePlans(valuePlans);
        return poAgg;
    }

View Full Code Here

                // PigCombiner directly attaches output from package to
                // root of remaining plan.


                POLocalRearrange mlr = getNewRearrange(rearrange);


                POPartialAgg mapAgg = null;
                if(doMapAgg){
                    mapAgg = createPartialAgg(cfe);
                }


                // A specialized local rearrange operator will replace

View Full Code Here

     * @throws CloneNotSupportedException 
     */
    private POPartialAgg createPartialAgg(POForEach combineFE)
            throws CloneNotSupportedException {
        String scope = combineFE.getOperatorKey().scope;
        POPartialAgg poAgg = new POPartialAgg(new OperatorKey(scope, 
                NodeIdGenerator.getGenerator().getNextNodeId(scope)));
        poAgg.setAlias(combineFE.getAlias());
        poAgg.setResultType(combineFE.getResultType());


        //first plan in combine foreach is the group key
        poAgg.setKeyPlan(combineFE.getInputPlans().get(0).clone());


        List<PhysicalPlan> valuePlans = new ArrayList<PhysicalPlan>();
        for(int i=1; i<combineFE.getInputPlans().size(); i++){
            valuePlans.add(combineFE.getInputPlans().get(i).clone());
        }
        poAgg.setValuePlans(valuePlans);
        return poAgg;
    }

View Full Code Here

                // PigCombiner directly attaches output from package to
                // root of remaining plan.


                POLocalRearrange mlr = getNewRearrange(rearrange);


                POPartialAgg mapAgg = null;
                if(doMapAgg){
                    mapAgg = createPartialAgg(cfe);
                }


                // A specialized local rearrange operator will replace

View Full Code Here

     * @throws CloneNotSupportedException 
     */
    private POPartialAgg createPartialAgg(POForEach combineFE)
            throws CloneNotSupportedException {
        String scope = combineFE.getOperatorKey().scope;
        POPartialAgg poAgg = new POPartialAgg(new OperatorKey(scope, 
                NodeIdGenerator.getGenerator().getNextNodeId(scope)));
        poAgg.addOriginalLocation(combineFE.getAlias(), combineFE.getOriginalLocations());
        poAgg.setResultType(combineFE.getResultType());


        //first plan in combine foreach is the group key
        poAgg.setKeyPlan(combineFE.getInputPlans().get(0).clone());


        List<PhysicalPlan> valuePlans = new ArrayList<PhysicalPlan>();
        for(int i=1; i<combineFE.getInputPlans().size(); i++){
            valuePlans.add(combineFE.getInputPlans().get(i).clone());
        }
        poAgg.setValuePlans(valuePlans);
        return poAgg;
    }

View Full Code Here

TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPartialAgg

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.CombinerOptimizer

org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result

org.apache.pig.backend.hadoop.executionengine.util.CombinerOptimizerUtil

org.apache.pig.data.DataBag

org.apache.pig.data.DefaultDataBag

org.apache.pig.data.SelfSpillBag.MemoryLimits

org.apache.pig.data.Tuple

org.apache.pig.backend.executionengine.ExecException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.