Package org.apache.pig.impl.plan.optimizer

Examples of org.apache.pig.impl.plan.optimizer.OptimizerException

958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
                    mergeOneCombinePlanWithIndex(
                            mrOp.combinePlan, comPl, index, incIndex, mrOp.mapKeyType);
                } else {        
                    int errCode = 2141;
                    String msg = "Internal Error. Cannot merge non-combiner with combiners for optimization.";
                    throw new OptimizerException(msg, errCode, PigException.BUG);
                }
            }
           
            // merge the reducer plan
            mergeOneReducePlanWithIndex(
                    mrOp.reducePlan, redPl, index, incIndex, mrOp.mapKeyType);
          
            index = incIndex;
           
            log.info("Merged MR job " + mrOp.getOperatorKey().getId()
                    + " into MR job " + splitter.getOperatorKey().getId());
        }

        PhysicalPlan splitterPl = splitter.mapPlan;
        PhysicalOperator leaf = splitterPl.getLeaves().get(0);
        PhysicalOperator storeOp = splitterPl.getLeaves().get(0);
        List<PhysicalOperator> storePreds = splitterPl.getPredecessors(storeOp); 
        // replace store operator in the splitter with split operator
        if (leaf instanceof POStore) {                           
            splitOp.setInputs(storePreds);
            try {
                splitterPl.replace(storeOp, splitOp);;
            } catch (PlanException e) {                  
                int errCode = 2132;
                String msg = "Internal Error. Unable to replace store with split operator for optimization.";
                throw new OptimizerException(msg, errCode, PigException.BUG, e);
            }
        }    
       
        splitter.setMapDone(true);
        splitter.reducePlan = redPl;
View Full Code Here
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
            try {
                splitterPl.replace(storeOp, splitOp);;
            } catch (PlanException e) {
                int errCode = 2132;
                String msg = "Internal Error. Unable to replace store with split operator for optimization.";
                throw new OptimizerException(msg, errCode, PigException.BUG, e);
            } 
        }
       
        removeAndReconnect(mapReduce, splitter);          
    }
View Full Code Here
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
                try {                  
                    getPlan().connect(newMR, succ);
                } catch (PlanException e) {
                    int errCode = 2133;
                    String msg = "Internal Error. Unable to connect map plan with successors for optimization.";
                    throw new OptimizerException(msg, errCode, PigException.BUG, e);
                }
            }
        }
       
        // reconnect the mapper's predecessors
        if (predsCopy != null) {
            for (MapReduceOper pred : predsCopy) {
                if (newMR.getOperatorKey().equals(pred.getOperatorKey())) {
                    continue;
                }
                try {                   
                    getPlan().connect(pred, newMR);
                } catch (PlanException e) {
                    int errCode = 2134;
                    String msg = "Internal Error. Unable to connect map plan with predecessors for optimization.";
                    throw new OptimizerException(msg, errCode, PigException.BUG, e);
                }
            }
        }    
       
        mergeMROperProperties(mr, newMR);
View Full Code Here
297
298
299
300
301
302
303
304
305
306
                    // there should be only one successor to package
                    sucs.get(0).setInputs(packList);
                } catch (Exception e) {
                    int errCode = 2018;
                    String msg = "Internal error. Unable to introduce the combiner for optimization.";
                    throw new OptimizerException(msg, errCode, PigException.BUG, e);
                }
            }
        }
    }
View Full Code Here
205
206
207
208
209
210
211
212
213
214
215
                storePlan.addAsLeaf(storeOp);
                splitOp.addPlan(storePlan);
            } catch (PlanException e) {
                int errCode = 2129;
                String msg = "Internal Error. Unable to add store to the split plan for optimization.";
                throw new OptimizerException(msg, errCode, PigException.BUG, e);
            }   
        }

        // case 6: special diamond case with trivial MR operator at the head
        if (numMerges == 0 && isDiamondMROper(mr)) {
View Full Code Here
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
                try {
                    clone = pl.clone();
                } catch (CloneNotSupportedException e) {
                    int errCode = 2127;
                    String msg = "Internal Error: Cloning of plan failed for optimization.";
                    throw new OptimizerException(msg, errCode, PigException.BUG, e);
                }
                succ.mapPlan.remove(op);
                while (!clone.isEmpty()) {
                    PhysicalOperator oper = clone.getLeaves().get(0);
                    clone.remove(oper);
                    succ.mapPlan.add(oper);
                    try {
                        succ.mapPlan.connect(oper, opSucc);
                        opSucc = oper;
                    } catch (PlanException e) {
                        int errCode = 2131;
                        String msg = "Internal Error. Unable to connect split plan for optimization.";
                        throw new OptimizerException(msg, errCode, PigException.BUG, e);
                    }               
                }
            }
        }
       
        // finally, remove the splitter from the MR plan
        List<MapReduceOper> mrPreds = getPlan().getPredecessors(mr);
        if (mrPreds != null) {
            for (MapReduceOper pred : mrPreds) {
                for (MapReduceOper succ : succs) {
                    try {
                        getPlan().connect(pred, succ);
                    } catch (PlanException e) {
                        int errCode = 2131;
                        String msg = "Internal Error. Unable to connect split plan for optimization.";
                        throw new OptimizerException(msg, errCode, PigException.BUG, e);
                    }
                }
            }
        }
       
View Full Code Here
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
        try {
            splitterPl.merge(pl);
        } catch (PlanException e) {
            int errCode = 2130;
            String msg = "Internal Error. Unable to merge split plans for optimization.";
            throw new OptimizerException(msg, errCode, PigException.BUG, e);
        }               
       
        // connect two plans  
        List<PhysicalOperator> roots = pl.getRoots();
        for (PhysicalOperator pred : predsCopy) {
            for (PhysicalOperator root : roots) {
                try {
                    splitterPl.connect(pred, root);
                } catch (PlanException e) {
                    int errCode = 2131;
                    String msg = "Internal Error. Unable to connect split plan for optimization.";
                    throw new OptimizerException(msg, errCode, PigException.BUG, e);

                }
            }
        }
    }
View Full Code Here
412
413
414
415
416
417
418
419
420
421
422
        try {
            splitterPl.replace(storeOp, splitOp);;
        } catch (PlanException e) {
            int errCode = 2132;
            String msg = "Internal Error. Unable to replace store with split operator for optimization.";
            throw new OptimizerException(msg, errCode, PigException.BUG, e);
        }   
       
        // remove all the map-only splittees from the MROperPlan
        for (MapReduceOper mapper : mappers) {
            removeAndReconnect(mapper, splitter);                 
View Full Code Here
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
        try {
            mrOper.mapPlan.addAsLeaf(getStore());
        } catch (PlanException e) {                  
            int errCode = 2137;
            String msg = "Internal Error. Unable to add store to the plan as leaf for optimization.";
            throw new OptimizerException(msg, errCode, PigException.BUG, e);
        }

        // connect the new MR operator to the splitter
        try {
            getPlan().add(mrOper);
            getPlan().connect(splitter, mrOper);
        } catch (PlanException e) {
            int errCode = 2133;
            String msg = "Internal Error. Unable to connect splitter with successors for optimization.";
            throw new OptimizerException(msg, errCode, PigException.BUG, e);
        }

        // merger the splittees into the new MR operator
        mergeAllMapReduceSplittees(mergeList, mrOper, getSplit());
       
View Full Code Here
574
575
576
577
578
579
580
581
582
583
584
                try {
                    lr.setMultiQueryIndex(index++);
                } catch (ExecException e) {                   
                    int errCode = 2136;
                    String msg = "Internal Error. Unable to set multi-query index for optimization.";
                    throw new OptimizerException(msg, errCode, PigException.BUG, e);                  
                }
               
                // change the map key type to tuple when
                // multiple splittees have different map key types
                if (!sameKeyType) {
View Full Code Here
TOP

Related Classes of org.apache.pig.impl.plan.optimizer.OptimizerException

  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.CombinerOptimizer
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.CombinerOptimizer$DistinctPatcher
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MultiQueryOptimizer
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.POPackageAnnotator
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.POPackageAnnotator$LoRearrangeDiscoverer
  • org.apache.pig.backend.hadoop.executionengine.tez.plan.TezPOPackageAnnotator
  • org.apache.pig.backend.hadoop.executionengine.tez.plan.TezPOPackageAnnotator$LoRearrangeDiscoverer
  • org.apache.pig.backend.hadoop.executionengine.util.CombinerOptimizerUtil
  • org.apache.pig.backend.hadoop.executionengine.util.CombinerOptimizerUtil$DistinctPatcher
  • org.apache.pig.impl.logicalLayer.ColumnPruner
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.