Package org.apache.pig.newplan.logical.relational

Examples of org.apache.pig.newplan.logical.relational.LOGenerate

2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
            load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1)));;
            LogicalPlan innerRelPlan = new LogicalPlan();
            LOForEach foreach1 = new LOForEach(plan);
            foreach1.setInnerPlan(innerRelPlan);

            LOGenerate loGen = new LOGenerate(innerRelPlan);
            innerRelPlan.add(loGen);

            LOInnerLoad innerLoad1 = new LOInnerLoad(innerRelPlan, foreach1, 2);
            innerRelPlan.add(innerLoad1);
            innerRelPlan.connect(innerLoad1, loGen);

            // Create expression inner plan #1 of input #1
            LogicalExpressionPlan innerPlan1 = new LogicalExpressionPlan();
            ProjectExpression project11 = new ProjectExpression(innerPlan1, 0, 2, loGen);

            List<Integer> projections1 = new ArrayList<Integer>();
            projections1.add(1);
            projections1.add(2);
            //ProjectExpression project12 = new ProjectExpression(innerPlan1, project11, projections1);
            //project12.setSentinel(false);
            DereferenceExpression project12 = new DereferenceExpression(innerPlan1, projections1);
           // innerPlan1.connect(project11, project12);
            innerPlan1.connect(project12, project11);

            // Create expression inner plan #1 of input #2
            LogicalExpressionPlan innerPlan2 = new LogicalExpressionPlan();
            innerLoad1 = new LOInnerLoad(innerRelPlan, foreach1, 0);
            innerRelPlan.add(innerLoad1);
            innerRelPlan.connect(innerLoad1, loGen);
            ProjectExpression project21 = new ProjectExpression(innerPlan2, 1, 0, loGen);

            innerLoad1 = new LOInnerLoad(innerRelPlan, foreach1, 1);
            innerRelPlan.add(innerLoad1);
            innerRelPlan.connect(innerLoad1, loGen);
            //ProjectExpression project22 = new ProjectExpression(innerPlan2, 2, 1, loGen);
            ProjectExpression project22 = new ProjectExpression(innerPlan2, 2, 0, loGen);

            AddExpression add21 = new AddExpression(innerPlan2, project21, project22);

            // List of plans
            ArrayList<LogicalExpressionPlan> generatePlans = new ArrayList<LogicalExpressionPlan>();
            generatePlans.add(innerPlan1);
            generatePlans.add(innerPlan2);

            // List of flatten flags
            boolean [] flattens = {true, false};

            loGen.setFlattenFlags(flattens);
            loGen.setOutputPlans(generatePlans);

            // construct the main plan
            plan.add(load1);
            plan.add(foreach1);
View Full Code Here
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
             // load as map and use as map
             query = "a= load '" + inputFileName + "' as (m:[]);"
             + "b = foreach a generate m#'k1';";

             LOForEach foreach = getForeachFromPlan(query);
             LOGenerate loGen = (LOGenerate) foreach.getInnerPlan().getSinks().get(0);
             Operator outExp = loGen.getOutputPlans().get(0).getSources().get(0);
             assertFalse("outExp is not cast", outExp instanceof CastExpression);

             // load as bytearray and use as map
             ps.registerQuery("a = load '" + inputFileName + "' as (m);");
             ps.registerQuery("b = foreach a generate m#'k1';");
View Full Code Here
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
        }

        ////////////////////////// Helper //////////////////////////////////
        private void checkForEachCasting(LOForEach foreach, int idx, boolean isCast, byte toType)
        throws FrontendException {
            LOGenerate gen = (LOGenerate) foreach.getInnerPlan().getSinks().get(0);
            LogicalExpressionPlan plan = gen.getOutputPlans().get(idx);

            if (isCast) {
                List<Operator> leaveList = plan.getSources();
                assertEquals(1, leaveList.size());
                assertTrue(leaveList.get(0) instanceof CastExpression);
View Full Code Here
316
317
318
319
320
321
322
323
324
325
326
        // have Project. Fields that need casting will have Project + Cast
        ArrayList<LogicalExpressionPlan> generatePlans = new ArrayList<LogicalExpressionPlan>() ;
        LogicalPlan innerPlan = new LogicalPlan();

        // create LOGenerate for foreach
        LOGenerate loGen = new LOGenerate(innerPlan, generatePlans,
                new boolean[toSchema.size()]);
        innerPlan.add(loGen);

        // Create ForEach to be inserted
        LOForEach foreach = new LOForEach(plan);
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
98
99
100
    }

    // moved from newplan/logical/rules/DuplicateForEachColumnRewrite.check()
    private List<LogicalExpressionPlan> findAllExpPlansToInsertIdentity(LOForEach foreach) throws FrontendException {

        LOGenerate gen = (LOGenerate)foreach.getInnerPlan().getSinks().get(0);

        List<LogicalExpressionPlan> expPlans = gen.getOutputPlans();
        boolean[] flattens = gen.getFlattenFlags();

        List<LogicalExpressionPlan> expPlansToInsertIdentity = new ArrayList<LogicalExpressionPlan>();
        List<Long> uidSeen = new ArrayList<Long>();

        for (int i=0;i<expPlans.size();i++) {
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
        currentWalker.walk(this);
        popWalker();

        //get the LOGenerate
        List<Operator> feOutputs = innerPlan.getSinks();
        LOGenerate gen = null;
        for( Operator op  : feOutputs){
            if(op instanceof LOGenerate){
                if(gen != null){
                    String msg = "Expected single LOGenerate output in innerplan of foreach";
                    throw new VisitorException(foreach,
                            msg,
                            2266,
                            PigException.BUG
                    );
                }
                gen = (LOGenerate) op;
            }
        }

        List<Operator> loGenPreds = innerPlan.getPredecessors(gen);

        if(loGenPreds == null){
            // there are no LOInnerLoads , must be working on just constants
            // no project-star expansion to be done
            return;
        }

        //get mapping of LOGenerate predecessor current position to object
        Map<Integer, LogicalRelationalOperator> oldPos2Rel =
            new HashMap<Integer, LogicalRelationalOperator>();
       
        for(int i=0; i<loGenPreds.size(); i++){
            oldPos2Rel.put(i, (LogicalRelationalOperator) loGenPreds.get(i));
        }

        //store mapping between the projection in inner plans of
        // of LOGenerate to the input relation object
        Map<ProjectExpression, LogicalRelationalOperator> proj2InpRel =
            new HashMap<ProjectExpression, LogicalRelationalOperator>();
       
        List<LOInnerLoad> expandedInLoads = new ArrayList<LOInnerLoad>();
       
        //visit each expression plan, and expand the projects in the udf
        for( OperatorPlan plan : gen.getOutputPlans()){
            ProjExpanderForForeach projExpander = new ProjExpanderForForeach(
                    plan,
                    gen,
                    oldPos2Rel,
                    proj2InpRel,
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
                continue;
           
            LOForEach foreach = new LOForEach( plan );
            LogicalPlan innerPlan = new LogicalPlan();

            LOGenerate gen = new LOGenerate( innerPlan );
            boolean[] flattenFlags = new boolean[fieldCount];
            List<LogicalExpressionPlan> exprPlans = new ArrayList<LogicalExpressionPlan>( fieldCount );
            List<Operator> genInputs = new ArrayList<Operator>();
           
            // Get exprPlans, and genInputs
            for( LogicalFieldSchema fs : outputSchema.getFields() ) {
                LogicalExpressionPlan exprPlan = new LogicalExpressionPlan();
                exprPlans.add( exprPlan );
                int pos = -1;
                //do a match with subname also
                LogicalFieldSchema matchFS = opSchema.getFieldSubNameMatch(fs.alias);
                if(matchFS != null){
                    pos = opSchema.getFieldPosition(matchFS.alias);
                }
                if( pos == -1 ) {
                    ConstantExpression constExp = new ConstantExpression( exprPlan, null);
                    if(fs.type != DataType.BYTEARRAY){
                        LogicalSchema.LogicalFieldSchema constFs = fs.deepCopy();
                        constFs.resetUid();
                        new CastExpression(exprPlan, constExp, constFs);
                    }
                } else {
                    ProjectExpression projExpr =
                        new ProjectExpression( exprPlan, genInputs.size(), 0, gen );
                    if( fs.type != DataType.BYTEARRAY
                        && opSchema.getField( pos ).type != fs.type ) {
                        new CastExpression( exprPlan, projExpr, fs );
                    }
                    genInputs.add( new LOInnerLoad( innerPlan, foreach, pos ) );
                }
            }
           
            gen.setFlattenFlags( flattenFlags );
            gen.setOutputPlans( exprPlans );
            gen.setUserDefinedSchema( fieldSchemas );
            innerPlan.add( gen );
            for( Operator input : genInputs ) {
                innerPlan.add(input);
                innerPlan.connect( input, gen );
            }
View Full Code Here
598
599
600
601
602
603
604
605
606
607
608
609
610
        LogicalPlan innerPlan =
            forEach.getInnerPlan();
       
        assertEquals( 1, innerPlan.getSinks().size() );       
        assertEquals( LOGenerate.class, innerPlan.getSinks().get(0).getClass() );
        LOGenerate gen = (LOGenerate)innerPlan.getSinks().get(0);
        assertEquals( 1, gen.getOutputPlans().size() );
        LogicalExpressionPlan genExp = gen.getOutputPlans().get(0);
       
        assertEquals( 1, genExp.getSources().size() );
       
       
        // Main Tests start here
View Full Code Here
653
654
655
656
657
658
659
660
661
662
663
664
665
        LogicalPlan innerPlan =
            forEach.getInnerPlan();
       
        assertEquals( 1, innerPlan.getSinks().size() );       
        assertEquals( LOGenerate.class, innerPlan.getSinks().get(0).getClass() );
        LOGenerate gen = (LOGenerate)innerPlan.getSinks().get(0);
        assertEquals( 1, gen.getOutputPlans().size() );
        LogicalExpressionPlan genExp = gen.getOutputPlans().get(0);
       
        assertEquals( 1, genExp.getSources().size() );
       
        // Main Tests start here
        assertEquals( SubtractExpression.class, genExp.getSources().get(0).getClass() );
View Full Code Here
707
708
709
710
711
712
713
714
715
716
717
718
719
        LogicalPlan innerPlan =
            forEach.getInnerPlan();
       
        assertEquals( 1, innerPlan.getSinks().size() );       
        assertEquals( LOGenerate.class, innerPlan.getSinks().get(0).getClass() );
        LOGenerate gen = (LOGenerate)innerPlan.getSinks().get(0);
        assertEquals( 1, gen.getOutputPlans().size() );
        LogicalExpressionPlan genExp = gen.getOutputPlans().get(0);
       
        assertEquals( 1, genExp.getSources().size() );
       
        // Main Tests start here
        assertEquals( MultiplyExpression.class, genExp.getSources().get(0).getClass() );
View Full Code Here
TOP

Related Classes of org.apache.pig.newplan.logical.relational.LOGenerate

  • org.apache.pig.newplan.logical.expression.LogicalExpression
  • org.apache.pig.newplan.logical.ForeachInnerPlanVisitor
  • org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema
  • org.apache.pig.newplan.logical.rules.ColumnPruneHelper$ColumnDependencyVisitor
  • org.apache.pig.newplan.logical.rules.ColumnPruneVisitor
  • org.apache.pig.newplan.logical.rules.DuplicateForEachColumnRewrite$DuplicateForEachColumnRewriteTransformer
  • org.apache.pig.newplan.logical.rules.LimitOptimizer$OptimizeLimitTransformer
  • org.apache.pig.newplan.logical.rules.MergeForEach$MergeForEachTransformer
  • org.apache.pig.newplan.logical.rules.OptimizerUtils
  • org.apache.pig.newplan.logical.rules.PushDownForEachFlatten$PushDownForEachFlattenTransformer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.