Package org.apache.pig.newplan.logical.relational

Examples of org.apache.pig.newplan.logical.relational.LOGenerate

104
105
106
107
108
109
110
111
112
113
114
                return;
            Operator next = currentPlan.getSuccessors(op).get(0);
            currentPlan.insertBetween(op, foreach, next);
           
            List<LogicalExpressionPlan> exps = new ArrayList<LogicalExpressionPlan>();
            LOGenerate gen = new LOGenerate(innerPlan, exps, new boolean[s.size()]);
            innerPlan.add(gen);
           
            // if we are inserting casts in a load and if the loader
            // implements determineSchema(), insert casts only where necessary
            // Note that in this case, the data coming out of the loader is not
View Full Code Here
516
517
518
519
520
521
522
523
524
525
526
      throws FrontendException {

  LOForEach foreach = new LOForEach(plan);
  LOCogroup groupby = new LOCogroup(plan);
  LogicalPlan innerPlan = new LogicalPlan();
  LogicalRelationalOperator gen = new LOGenerate(innerPlan);

  injectForeachOperator(loc, op, foreach);

  // Get all column attributes from the input relation.
  // Create ProjectExpression for all columns. Based on the
View Full Code Here
924
925
926
927
928
929
930
        expandAndResetVisitor(loc, op);
        return alias;
    }

    LOGenerate createGenerateOp(LogicalPlan plan) {
        return new LOGenerate( plan );
    }
View Full Code Here
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
        LogicalPlan lp = new LogicalPlan(); // f's inner plan
        LOForEach f = new LOForEach( innerPlan );
        f.setInnerPlan( lp );
        f.setLocation( loc );
        LOGenerate gen = new LOGenerate( lp );
        boolean[] flatten = new boolean[exprPlans.size()];

        List<Operator> innerLoads = new ArrayList<Operator>( exprPlans.size() );
        for( LogicalExpressionPlan plan : exprPlans ) {
            ProjectExpression pe = (ProjectExpression)plan.getSinks().get( 0 );
            String al = pe.getColAlias();
            LOInnerLoad iload = ( al == null ) ?
                    new LOInnerLoad( lp, f, pe.getColNum() ) : createInnerLoad(loc, lp, f, al );
            iload.setLocation( pe.getLocation() );
            pe.setColNum( -1 );
            pe.setInputNum( innerLoads.size() );
            pe.setAttachedRelationalOp( gen );
            innerLoads.add( iload );
        }

        gen.setOutputPlans( exprPlans );
        gen.setFlattenFlags( flatten );
        lp.add( gen );

        for( Operator il : innerLoads ) {
            lp.add( il );
            lp.connect( il, gen );
View Full Code Here
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
        @SuppressWarnings("unchecked")
        @Override
        public void visit(LOForEach foreach) throws FrontendException {
            Set<Long> output = setOutputUids(foreach);

            LOGenerate gen = OptimizerUtils.findGenerate(foreach);
            gen.annotate(OUTPUTUIDS, output);

            visit(gen);

            Set<Long> input = (Set<Long>)gen.getAnnotation(INPUTUIDS);

            // Make sure at least one column will retain
            if (input.isEmpty()) {
                LogicalRelationalOperator pred = (LogicalRelationalOperator)plan.getPredecessors(foreach).get(0);
                if (pred.getSchema()!=null)
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
            // flatten(distinct({(1), (1)})) is (1), (1)
           
            // in both cases correctness is not affected
           
            LOForEach foreach = (LOForEach)matched.getSources().get(0);
            LOGenerate gen = OptimizerUtils.findGenerate( foreach );
           
            if( !OptimizerUtils.hasFlatten( gen ) )
                return false;
           
            // If a foreach contains a nondeterministic udf, we shouldn't push it down.
            for (LogicalExpressionPlan p : gen.getOutputPlans()) {
                if (OptimizerUtils.planHasNonDeterministicUdf(p))
                    return false;
            }
           
            List<Operator> succs = currentPlan.getSuccessors( foreach );
            if( succs == null || succs.size() != 1 )
                return false;
           
            List<Long> uids = getNonFlattenFieldUids( gen );

            Operator succ = succs.get( 0  );
            if( !( succ instanceof LOSort || succ instanceof LOJoin || succ instanceof LOCross ) )
                return false;
           
            if( succ instanceof LOSort ) {
                // Check if the expressions for the foreach generate are purely projection including flatten fields.
                List<LogicalExpressionPlan> exprs = gen.getOutputPlans();
                for( LogicalExpressionPlan expr : exprs ) {
                    if( !isPureProjection( expr ) )
                        return false;
                }
View Full Code Here
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
                Map<Integer, LogicalSchema> cachedUserDefinedSchema = new HashMap<Integer, LogicalSchema>();
                boolean[] flags = null;
                int fieldCount = 0;
                for( Operator op : preds ) {
                    if( op == foreach ) {
                        LOGenerate gen = OptimizerUtils.findGenerate( foreach );
                        flags = gen.getFlattenFlags();
                        for( int i = 0; i < flags.length; i++ ) {
                            if( flags[i] ) {
                                fieldsToBeFlattaned.add(fieldCount);
                                if (gen.getUserDefinedSchema()!=null && gen.getUserDefinedSchema().get(i)!=null) {
                                    cachedUserDefinedSchema.put(fieldCount, gen.getUserDefinedSchema().get(i));
                                    gen.getUserDefinedSchema().set(i, null);
                                }
                                fieldCount++;
                            } else {
                                fieldCount++;
                            }
                        }
                    } else {
                        fieldCount += ( (LogicalRelationalOperator)op ).getSchema().size();
                    }
                }
               
               
                boolean[] flattenFlags = new boolean[fieldCount];
                List<LogicalSchema> mUserDefinedSchema = null;
                if (cachedUserDefinedSchema!=null) {
                    mUserDefinedSchema = new ArrayList<LogicalSchema>();
                    for (int i=0;i<fieldCount;i++)
                        mUserDefinedSchema.add(null);
                }
                for( Integer i : fieldsToBeFlattaned ) {
                    flattenFlags[i] = true;
                    if (cachedUserDefinedSchema.containsKey(i)) {
                        mUserDefinedSchema.set(i, cachedUserDefinedSchema.get(i));
                    }
                }
               
                // Now create a new foreach after cross/join and insert it into the plan.
                LOForEach newForeach = new LOForEach( currentPlan );
                LogicalPlan innerPlan = new LogicalPlan();
                List<LogicalExpressionPlan> exprs = new ArrayList<LogicalExpressionPlan>( fieldCount );
                LOGenerate gen = new LOGenerate( innerPlan, exprs, flattenFlags );
                if (mUserDefinedSchema!=null)
                    gen.setUserDefinedSchema(mUserDefinedSchema);
                innerPlan.add( gen );
                newForeach.setInnerPlan( innerPlan );
                for( int i = 0; i < fieldCount; i++ ) {
                    LogicalExpressionPlan expr = new LogicalExpressionPlan();
                    expr.add( new ProjectExpression( expr, i, -1, gen ) );
View Full Code Here
106
107
108
109
110
111
112
113
114
115
116
                return;
            Operator next = currentPlan.getSuccessors(op).get(0);
            currentPlan.insertBetween(op, foreach, next);
           
            List<LogicalExpressionPlan> exps = new ArrayList<LogicalExpressionPlan>();
            LOGenerate gen = new LOGenerate(innerPlan, exps, new boolean[s.size()]);
            innerPlan.add(gen);
           
            // if we are inserting casts in a load and if the loader
            // implements determineSchema(), insert casts only where necessary
            // Note that in this case, the data coming out of the loader is not
View Full Code Here
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
           
            LogicalPlan innerRelPlan = new LogicalPlan();
            LOForEach foreach1 = new LOForEach(plan);
            foreach1.setInnerPlan(innerRelPlan);
           
            LOGenerate loGen = new LOGenerate(innerRelPlan);
            innerRelPlan.add(loGen);
           
            LOInnerLoad innerLoad1 = new LOInnerLoad(innerRelPlan, foreach1, 0);
            innerRelPlan.add(innerLoad1);
            innerRelPlan.connect(innerLoad1, loGen);

            // Create expression inner plan #1
            LogicalExpressionPlan innerPlan1 = new LogicalExpressionPlan() ;
            ProjectExpression project11 = new ProjectExpression(innerPlan1, 0, 0, loGen) ;
           
           
            ConstantExpression const11 = new ConstantExpression(innerPlan1, 26F) ;

            SubtractExpression subtract11 = new SubtractExpression(innerPlan1, project11, const11) ;
           
            // Create expression inner plan #2
            LogicalExpressionPlan innerPlan2 = new LogicalExpressionPlan() ;
            LOInnerLoad innerLoad2 = new LOInnerLoad(innerRelPlan, foreach1, 0);
            innerRelPlan.add(innerLoad2);
            innerRelPlan.connect(innerLoad2, loGen);
            ProjectExpression project21 = new ProjectExpression(innerPlan2, 1, 0, loGen) ;
           
            LOInnerLoad innerLoad3 = new LOInnerLoad(innerRelPlan, foreach1, 1);
            innerRelPlan.add(innerLoad3);
            innerRelPlan.connect(innerLoad3, loGen);

            ProjectExpression project22 = new ProjectExpression(innerPlan2, 2, 0, loGen) ;
           
            AddExpression add21 = new AddExpression(innerPlan2, project21, project22 ) ;
   
            // List of plans
            ArrayList<LogicalExpressionPlan> generatePlans = new ArrayList<LogicalExpressionPlan>() ;
            generatePlans.add(innerPlan1);
            generatePlans.add(innerPlan2);
   
            // List of flatten flags
            boolean [] flattens = {true, false};

   
            loGen.setFlattenFlags(flattens);
            loGen.setOutputPlans(generatePlans);
           
   
            // construct the main plan
            plan.add(load1) ;
            plan.add(foreach1) ;
View Full Code Here
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
   
            LogicalPlan innerRelPlan = new LogicalPlan();
            LOForEach foreach1 = new LOForEach(plan);
            foreach1.setInnerPlan(innerRelPlan);
           
            LOGenerate loGen = new LOGenerate(innerRelPlan);
            innerRelPlan.add(loGen);
           
            LOInnerLoad innerLoad1 = new LOInnerLoad(innerRelPlan, foreach1, 0);
            innerRelPlan.add(innerLoad1);
            innerRelPlan.connect(innerLoad1, loGen);

           
            // Create expression inner plan #1
            LogicalExpressionPlan innerPlan1 = new LogicalExpressionPlan() ;
            ProjectExpression project11 = new ProjectExpression(innerPlan1, 0, 0, loGen) ;
           
            ConstantExpression const11 = new ConstantExpression(innerPlan1, "26F") ;
            SubtractExpression subtract11 = new SubtractExpression(innerPlan1, project11, const11) ;

            // Create expression inner plan #2
            LogicalExpressionPlan innerPlan2 = new LogicalExpressionPlan() ;
            innerLoad1 = new LOInnerLoad(innerRelPlan, foreach1, 0);
            innerRelPlan.add(innerLoad1);
            innerRelPlan.connect(innerLoad1, loGen);
            ProjectExpression project21 = new ProjectExpression(innerPlan2, 1, 0, loGen) ;
           
           
            innerLoad1 = new LOInnerLoad(innerRelPlan, foreach1, 0);
            innerRelPlan.add(innerLoad1);
            innerRelPlan.connect(innerLoad1, loGen);
            ProjectExpression project22 = new ProjectExpression(innerPlan2, 2, 1, loGen) ;
           
            AddExpression add21 = new AddExpression(innerPlan2, project21, project22) ;
   
            // List of plans
            ArrayList<LogicalExpressionPlan> generatePlans = new ArrayList<LogicalExpressionPlan>() ;
            generatePlans.add(innerPlan1);
            generatePlans.add(innerPlan2);
   
            // List of flatten flags
            boolean [] flattens = {true, false};
   
            loGen.setFlattenFlags(flattens);
            loGen.setOutputPlans(generatePlans);
           
            // construct the main plan
            plan.add(load1) ;
            plan.add(foreach1) ;
   
View Full Code Here
TOP

Related Classes of org.apache.pig.newplan.logical.relational.LOGenerate

  • org.apache.pig.newplan.logical.expression.LogicalExpression
  • org.apache.pig.newplan.logical.ForeachInnerPlanVisitor
  • org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema
  • org.apache.pig.newplan.logical.rules.ColumnPruneHelper$ColumnDependencyVisitor
  • org.apache.pig.newplan.logical.rules.ColumnPruneVisitor
  • org.apache.pig.newplan.logical.rules.DuplicateForEachColumnRewrite$DuplicateForEachColumnRewriteTransformer
  • org.apache.pig.newplan.logical.rules.LimitOptimizer$OptimizeLimitTransformer
  • org.apache.pig.newplan.logical.rules.MergeForEach$MergeForEachTransformer
  • org.apache.pig.newplan.logical.rules.OptimizerUtils
  • org.apache.pig.newplan.logical.rules.PushDownForEachFlatten$PushDownForEachFlattenTransformer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.