Package org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserFunc

488
489
490
491
492
493
494
495
496
497
498
499
500
                       
                    }
                    // Also set the Distinct's function to type Initial in map
                    // to type Intermediate in combine plan and to type Final in
                    // the reduce
                    POUserFunc distinctFunc = (POUserFunc)getDistinctUserFunc(plans[j], leaf);
                    try {
                        distinctFunc.setAlgebraicFunction(funcTypes[j]);
                    } catch (ExecException e) {
                        int errCode = 2074;
                        String msg = "Could not configure distinct's algebraic functions in map reduce plan.";
                        throw new PlanException(msg, errCode, PigException.BUG, e);
                    }
View Full Code Here
656
657
658
659
660
661
662
663
664
665
666
667
668
        if (!(leaf instanceof POUserFunc)) {
            int errCode = 2020;
            String msg = "Expected to find plan with UDF leaf. Found " + leaf.getClass().getSimpleName();
            throw new PlanException(msg, errCode, PigException.BUG);
        }
        POUserFunc func = (POUserFunc)leaf;
        try {
            func.setAlgebraicFunction(type);
        } catch (ExecException e) {
            int errCode = 2075;
            String msg = "Could not set algebraic function type.";
            throw new PlanException(msg, errCode, PigException.BUG, e);
        }
View Full Code Here
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
                ce1.setValue(ce1val);
                ce1.setResultType(DataType.TUPLE);*/
               
               

                POUserFunc gfc = new POUserFunc(new OperatorKey(scope, nodeGen.getNextNodeId(scope)),cross.getRequestedParallelisam(), Arrays.asList((PhysicalOperator)ce1,(PhysicalOperator)ce2), new FuncSpec(GFCross.class.getName()));
                gfc.setAlias(cross.getAlias());
                gfc.setResultType(DataType.BAG);
                fep1.addAsLeaf(gfc);
                gfc.setInputs(Arrays.asList((PhysicalOperator)ce1,(PhysicalOperator)ce2));
                /*fep1.add(gfc);
                fep1.connect(ce1, gfc);
                fep1.connect(ce2, gfc);*/
               
                PhysicalPlan fep2 = new PhysicalPlan();
View Full Code Here
485
486
487
488
489
490
491
492
493
494
495
    @Override
    public void visit( UserFuncExpression op ) throws FrontendException {      
        Object f = PigContext.instantiateFuncFromSpec(op.getFuncSpec());
        PhysicalOperator p;
        if (f instanceof EvalFunc) {
            p = new POUserFunc(new OperatorKey(DEFAULT_SCOPE, nodeGen
                    .getNextNodeId(DEFAULT_SCOPE)), -1,
                    null, op.getFuncSpec(), (EvalFunc) f);
        } else {
            p = new POUserComparisonFunc(new OperatorKey(DEFAULT_SCOPE, nodeGen
                    .getNextNodeId(DEFAULT_SCOPE)), -1,
View Full Code Here
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
       
        List<PhysicalOperator> ufInps = new ArrayList<PhysicalOperator>();
        ufInps.add(prjStar);
       
      PhysicalPlan ep = new PhysicalPlan();
      POUserFunc uf = new POUserFunc(new OperatorKey(scope,nig.getNextNodeId(scope)), -1, ufInps,
                  new FuncSpec(GetMemNumRows.class.getName(), (String[])null));
      uf.setResultType(DataType.TUPLE);
      ep.add(uf);    
      ep.add(prjStar);
      ep.connect(prjStar, uf);

        transformPlans.add(ep);     
View Full Code Here
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
        ep4.add(prjStar4);
       
        List<PhysicalOperator> ufInps = new ArrayList<PhysicalOperator>();
        ufInps.add(prjStar4);
     
        POUserFunc uf = new POUserFunc(new OperatorKey(scope,nig.getNextNodeId(scope)), -1, ufInps,
            new FuncSpec(udfClassName, udfArgs));
        ep4.add(uf);
        ep4.connect(prjStar4, uf);
       
        List<PhysicalPlan> ep4s = new ArrayList<PhysicalPlan>();
View Full Code Here
917
918
919
920
921
922
923
924
925
926
927
928
929
                    // explicitly set distinctPredecessor's result type to
                    // be tuple - this is relevant when distinctPredecessor is
                    // originally a POForeach with return type BAG - we need to
                    // set it to tuple so we get a stream of tuples.
                    distinctPredecessor.setResultType(DataType.TUPLE);
                    POUserFunc func = new POUserFunc(new OperatorKey(scope,
                            NodeIdGenerator.getGenerator().getNextNodeId(scope)),-1, funcInput, fSpec);
                    func.setResultType(DataType.BAG);
                    mPlan.replace(proj, func);
                    mPlan.remove(pred);
                    // connect the the newly added "func" to
                    // the predecessor to the earlier PODistinct
                    mPlan.connect(distinctPredecessor, func);
View Full Code Here
426
427
428
429
430
431
432
433
434
435
436
437
                if (toBeFlattened) return ExprType.NOT_ALGEBRAIC;
            }
            return ExprType.SIMPLE_PROJECT;
        } else if (leaf instanceof POUserFunc) {
           
            POUserFunc userFunc = (POUserFunc)leaf;
            if(!userFunc.combinable() ){
                return ExprType.NOT_ALGEBRAIC;
            }
            // The leaf userFunc may be combinable, but there might be other
            // algebraic userFuncs in the predecessors, if there are
            // we choose not to fire combiner.
View Full Code Here
526
527
528
529
530
531
532
533
534
535
536
537
538
                       
                    }
                    // Also set the Distinct's function to type Initial in map
                    // to type Intermediate in combine plan and to type Final in
                    // the reduce
                    POUserFunc distinctFunc = (POUserFunc)getDistinctUserFunc(plans[j], leaf);
                    try {
                        distinctFunc.setAlgebraicFunction(funcTypes[j]);
                    } catch (ExecException e) {
                        int errCode = 2074;
                        String msg = "Could not configure distinct's algebraic functions in map reduce plan.";
                        throw new PlanException(msg, errCode, PigException.BUG, e);
                    }
View Full Code Here
694
695
696
697
698
699
700
701
702
703
704
705
706
        if (!(leaf instanceof POUserFunc)) {
            int errCode = 2020;
            String msg = "Expected to find plan with UDF leaf. Found " + leaf.getClass().getSimpleName();
            throw new PlanException(msg, errCode, PigException.BUG);
        }
        POUserFunc func = (POUserFunc)leaf;
        try {
            func.setAlgebraicFunction(type);
        } catch (ExecException e) {
            int errCode = 2075;
            String msg = "Could not set algebraic function type.";
            throw new PlanException(msg, errCode, PigException.BUG, e);
        }
View Full Code Here
TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserFunc

  • org.apache.hadoop.conf.Configuration
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.CombinerOptimizer
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.CombinerOptimizer$DistinctPatcher
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.util.MonitoredUDFExecutor
  • org.apache.pig.backend.hadoop.executionengine.tez.plan.TezCompiler
  • org.apache.pig.backend.hadoop.executionengine.util.CombinerOptimizerUtil
  • org.apache.pig.backend.hadoop.executionengine.util.CombinerOptimizerUtil$DistinctPatcher
  • org.apache.pig.data.DataByteArray
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.