Package org.apache.pig.newplan.logical.relational

Examples of org.apache.pig.newplan.logical.relational.LOGenerate

88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
            // flatten(distinct({(1), (1)})) is (1), (1)
           
            // in both cases correctness is not affected
           
            LOForEach foreach = (LOForEach)matched.getSources().get(0);
            LOGenerate gen = OptimizerUtils.findGenerate( foreach );
            if( !OptimizerUtils.hasFlatten( gen ) )
                return false;
           
            List<Operator> succs = currentPlan.getSuccessors( foreach );
            if( succs == null || succs.size() != 1 )
                return false;
           
            List<Long> uids = getNonFlattenFieldUids( gen );

            Operator succ = succs.get( 0  );
            if( !( succ instanceof LOSort || succ instanceof LOJoin || succ instanceof LOCross ) )
                return false;
           
            if( succ instanceof LOSort ) {
                // Check if the expressions for the foreach generate are purely projection including flatten fields.
                List<LogicalExpressionPlan> exprs = gen.getOutputPlans();
                for( LogicalExpressionPlan expr : exprs ) {
                    if( !isPureProjection( expr ) )
                        return false;
                }
View Full Code Here
126
127
128
129
130
131
132
133
134
135
136
       
        LogicalSchema schema = op.getSchema();
       
        // build foreach inner plan
        List<LogicalExpressionPlan> exps = new ArrayList<LogicalExpressionPlan>();
        LOGenerate gen = new LOGenerate(innerPlan, exps, new boolean[schema.size()-columnsToDrop.size()]);
        innerPlan.add(gen);
       
        for (int i=0, j=0; i<schema.size(); i++) {
            if (columnsToDrop.contains(i)) {
                continue;
View Full Code Here
252
253
254
255
256
257
258
259
260
261
262
        org.apache.pig.newplan.logical.relational.LogicalPlan newForEachInnerPlan
            = new org.apache.pig.newplan.logical.relational.LogicalPlan();       
        newForEach.setInnerPlan(newForEachInnerPlan);
        List<LogicalExpressionPlan> expPlans = new ArrayList<LogicalExpressionPlan>();
        boolean[] flattens = new boolean[foreach.getForEachPlans().size()];
        LOGenerate generate = new LOGenerate(newForEachInnerPlan, expPlans, flattens);
        newForEachInnerPlan.add(generate);
       
        for (int i=0;i<foreach.getForEachPlans().size();i++) {
            LogicalPlan innerPlan = foreach.getForEachPlans().get(i);
            // Assume only one project is allowed in this level of foreach
View Full Code Here
222
223
224
225
226
227
228
229
230
231
232
        innerPlan.add(l1);
        LOInnerLoad l2 = new LOInnerLoad(innerPlan, foreach, 1);
        innerPlan.add(l2);
       
        List<LogicalExpressionPlan> eps = new ArrayList<LogicalExpressionPlan>();
        LOGenerate gen = new LOGenerate(innerPlan, eps, new boolean[] {false, true});
        LogicalExpressionPlan p1 = new LogicalExpressionPlan();
        p1.add(new ProjectExpression(p1, 0, -1, gen));
        LogicalExpressionPlan p2 = new LogicalExpressionPlan();
        p2.add(new ProjectExpression(p2, 1, -1, gen));
        eps.add(p1);
View Full Code Here
356
357
358
359
360
361
362
363
364
365
366
        innerPlan.add(l1);
        LOInnerLoad l2 = new LOInnerLoad(innerPlan, foreach2, 1);
        innerPlan.add(l2);
       
        List<LogicalExpressionPlan>  eps = new ArrayList<LogicalExpressionPlan>();
        LOGenerate gen = new LOGenerate(innerPlan, eps, new boolean[] {false, true});
        LogicalExpressionPlan p1 = new LogicalExpressionPlan();
        new ProjectExpression(p1, 0, -1, gen);
        LogicalExpressionPlan p2 = new LogicalExpressionPlan();       
        new ProjectExpression(p2, 1, -1, gen);
        eps.add(p1);
View Full Code Here
135
136
137
138
139
140
141
142
143
144
145
146
        LogicalPlan inner1 = foreach.getInnerPlan();
        Iterator<Operator> it = inner1.getOperators();
        while( it.hasNext() ) {
            Operator op = it.next();
            if( op instanceof LOGenerate ) {
                LOGenerate gen = (LOGenerate)op;
                boolean[] flattenFlags = gen.getFlattenFlags();
                if( flattenFlags != null ) {
                    for( boolean flatten : flattenFlags ) {
                        if( flatten ) {
                            hasFlatten = true;
                            break;
View Full Code Here
275
276
277
278
279
280
281
282
      
    private int getOutputExprCount(LogicalPlan plan) throws IOException {
        LOForEach foreach = getForEachOperator( plan );
        LogicalPlan inner = foreach.getInnerPlan();
        List<Operator> ops = inner.getSinks();
        LOGenerate gen = (LOGenerate)ops.get( 0 );
        return gen.getOutputPlans().size();
    }
View Full Code Here
584
585
586
587
588
589
590
591
592
593
594
595
596
        org.apache.pig.newplan.logical.relational.LogicalPlan innerPlan =
            forEach.getInnerPlan();
       
        assertEquals( 1, innerPlan.getSinks().size() );       
        assertEquals( LOGenerate.class, innerPlan.getSinks().get(0).getClass() );
        LOGenerate gen = (LOGenerate)innerPlan.getSinks().get(0);
        assertEquals( 1, gen.getOutputPlans().size() );
        LogicalExpressionPlan genExp = gen.getOutputPlans().get(0);
       
        assertEquals( 1, genExp.getSources().size() );
       
       
        // Main Tests start here
View Full Code Here
639
640
641
642
643
644
645
646
647
648
649
650
651
        org.apache.pig.newplan.logical.relational.LogicalPlan innerPlan =
            forEach.getInnerPlan();
       
        assertEquals( 1, innerPlan.getSinks().size() );       
        assertEquals( LOGenerate.class, innerPlan.getSinks().get(0).getClass() );
        LOGenerate gen = (LOGenerate)innerPlan.getSinks().get(0);
        assertEquals( 1, gen.getOutputPlans().size() );
        LogicalExpressionPlan genExp = gen.getOutputPlans().get(0);
       
        assertEquals( 1, genExp.getSources().size() );
       
        // Main Tests start here
        assertEquals( SubtractExpression.class, genExp.getSources().get(0).getClass() );
View Full Code Here
693
694
695
696
697
698
699
700
701
702
703
704
705
        org.apache.pig.newplan.logical.relational.LogicalPlan innerPlan =
            forEach.getInnerPlan();
       
        assertEquals( 1, innerPlan.getSinks().size() );       
        assertEquals( LOGenerate.class, innerPlan.getSinks().get(0).getClass() );
        LOGenerate gen = (LOGenerate)innerPlan.getSinks().get(0);
        assertEquals( 1, gen.getOutputPlans().size() );
        LogicalExpressionPlan genExp = gen.getOutputPlans().get(0);
       
        assertEquals( 1, genExp.getSources().size() );
       
        // Main Tests start here
        assertEquals( MultiplyExpression.class, genExp.getSources().get(0).getClass() );
View Full Code Here
TOP

Related Classes of org.apache.pig.newplan.logical.relational.LOGenerate

  • org.apache.pig.newplan.logical.expression.LogicalExpression
  • org.apache.pig.newplan.logical.ForeachInnerPlanVisitor
  • org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema
  • org.apache.pig.newplan.logical.rules.ColumnPruneHelper$ColumnDependencyVisitor
  • org.apache.pig.newplan.logical.rules.ColumnPruneVisitor
  • org.apache.pig.newplan.logical.rules.DuplicateForEachColumnRewrite$DuplicateForEachColumnRewriteTransformer
  • org.apache.pig.newplan.logical.rules.LimitOptimizer$OptimizeLimitTransformer
  • org.apache.pig.newplan.logical.rules.MergeForEach$MergeForEachTransformer
  • org.apache.pig.newplan.logical.rules.OptimizerUtils
  • org.apache.pig.newplan.logical.rules.PushDownForEachFlatten$PushDownForEachFlattenTransformer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.