Package org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach

737
738
739
740
741
742
743
744
745
746
747
748
       
        PhysicalPlan phyPlan = translatePlan(newLogicalPlan);
       
        PhysicalOperator pFE = phyPlan.getSuccessors( phyPlan.getRoots().get(0) ).get(0);
        assertEquals( POForEach.class, pFE.getClass() );
        POForEach pForEach = (POForEach)pFE;
        PhysicalPlan inputPln = pForEach.getInputPlans().get(0);
       
        assertEquals(1, ls.getField(0).uid);
        assertEquals(2, ls.getField(1).uid);
       
        LogicalRelationalOperator fe =
View Full Code Here
791
792
793
794
795
796
797
798
799
800
801
802
       
        PhysicalPlan phyPlan = translatePlan(newLogicalPlan);
       
        PhysicalOperator pFE = phyPlan.getSuccessors( phyPlan.getRoots().get(0) ).get(0);
        assertEquals( POForEach.class, pFE.getClass() );
        POForEach pForEach = (POForEach)pFE;
        PhysicalPlan inputPln = pForEach.getInputPlans().get(0);
       
        assertEquals(1, ls.getField(0).uid);
        assertEquals(2, ls.getField(1).uid);
       
        LogicalRelationalOperator fe =
View Full Code Here
845
846
847
848
849
850
851
852
853
854
855
856
       
        PhysicalPlan phyPlan = translatePlan(newLogicalPlan);
       
        PhysicalOperator pFE = phyPlan.getSuccessors( phyPlan.getRoots().get(0) ).get(0);
        assertEquals( POForEach.class, pFE.getClass() );
        POForEach pForEach = (POForEach)pFE;
        PhysicalPlan inputPln = pForEach.getInputPlans().get(0);
       
        assertEquals(1, ls.getField(0).uid);
        assertEquals(2, ls.getField(1).uid);
       
        LogicalRelationalOperator fe =
View Full Code Here
977
978
979
980
981
982
983
984
985
986
987
988
       
        PhysicalPlan phyPlan = translatePlan(newLogicalPlan);
       
        PhysicalOperator pFE = phyPlan.getSuccessors( phyPlan.getRoots().get(0) ).get(0);
        assertEquals( POForEach.class, pFE.getClass() );
        POForEach pForEach = (POForEach)pFE;
        PhysicalPlan inputPln = pForEach.getInputPlans().get(0);
       
        assertEquals(1, ls.getField(0).uid);
        assertEquals(2, ls.getField(1).uid);
       
        LogicalRelationalOperator fe =
View Full Code Here
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
       
        PhysicalPlan phyPlan = translatePlan(newLogicalPlan);
       
        PhysicalOperator pFE = phyPlan.getSuccessors( phyPlan.getRoots().get(0) ).get(0);
        assertEquals( POForEach.class, pFE.getClass() );
        POForEach pForEach = (POForEach)pFE;
        PhysicalPlan inputPln1 = pForEach.getInputPlans().get(0);
       
        assertEquals(1, ls.getField(0).uid);
        assertEquals(2, ls.getField(1).uid);
       
        LogicalRelationalOperator fe =
            (LogicalRelationalOperator) newLogicalPlan.getSuccessors(load).get(0);
        assertEquals( LOForEach.class, fe.getClass() );
        LOForEach forEach = (LOForEach)fe;
       
        org.apache.pig.experimental.logical.relational.LogicalPlan innerPlan =
            forEach.getInnerPlan();
       
        assertEquals( 1, innerPlan.getSinks().size() );       
        assertEquals( LOGenerate.class, innerPlan.getSinks().get(0).getClass() );
        LOGenerate gen = (LOGenerate)innerPlan.getSinks().get(0);
        assertEquals( 2, gen.getOutputPlans().size() );
       
        LogicalExpressionPlan genExp1 = gen.getOutputPlans().get(0);
       
        assertEquals( 1, genExp1.getSources().size() );
        assertEquals( ProjectExpression.class, genExp1.getSources().get(0).getClass() );
        ProjectExpression prj1  = (ProjectExpression) genExp1.getSources().get(0);
        assertEquals( ls.getField(0).uid, prj1.getUid() );
       
        LogicalExpressionPlan genExp2 = gen.getOutputPlans().get(1);
        assertEquals( UserFuncExpression.class, genExp2.getSources().get(0).getClass() );
        assertEquals( ProjectExpression.class, genExp2.getSinks().get(0).getClass() );
        ProjectExpression prj2 = (ProjectExpression)genExp2.getSinks().get(0);
        assertEquals( ls.getField(1).uid, prj2.getUid() );
       
        assertEquals( 1, inputPln1.getLeaves().size() );
        assertEquals( 1, inputPln1.getRoots().size() );
        assertEquals( POProject.class, inputPln1.getLeaves().get(0).getClass() );
        assertEquals( 0, (( POProject) inputPln1.getLeaves().get(0)).getColumn() );
        PhysicalPlan inputPln2 = pForEach.getInputPlans().get(1);
        assertEquals( POUserFunc.class, inputPln2.getLeaves().get(0).getClass() );
        assertEquals( "org.apache.pig.builtin.COUNT",
                ((POUserFunc) inputPln2.getLeaves().get(0)).getFuncSpec().getClassName() );
        assertEquals( POProject.class, inputPln2.getRoots().get(0).getClass() );
        assertEquals( 1, ((POProject)inputPln2.getRoots().get(0)).getColumn() );
View Full Code Here
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
       
        PhysicalPlan phyPlan = translatePlan(newLogicalPlan);
       
        PhysicalOperator pFE = phyPlan.getSuccessors( phyPlan.getRoots().get(0) ).get(0);
        assertEquals( POForEach.class, pFE.getClass() );
        POForEach pForEach = (POForEach)pFE;
        PhysicalPlan inputPln = pForEach.getInputPlans().get(0);
       
        assertEquals(1, ls.getField(0).uid);
        assertEquals(2, ls.getField(1).uid);
       
        LogicalRelationalOperator fe =
            (LogicalRelationalOperator) newLogicalPlan.getSuccessors(load).get(0);
        assertEquals( LOForEach.class, fe.getClass() );
        LOForEach forEach = (LOForEach)fe;
       
        org.apache.pig.experimental.logical.relational.LogicalPlan innerPlan =
            forEach.getInnerPlan();
       
        assertEquals( 1, innerPlan.getSinks().size() );       
        assertEquals( LOGenerate.class, innerPlan.getSinks().get(0).getClass() );
        LOGenerate gen = (LOGenerate)innerPlan.getSinks().get(0);
        assertEquals( 2, gen.getOutputPlans().size() );
       
        LogicalExpressionPlan genExp1 = gen.getOutputPlans().get(0);
       
        assertEquals( 1, genExp1.getSources().size() );
        assertEquals( ProjectExpression.class, genExp1.getSources().get(0).getClass() );
        ProjectExpression prj1  = (ProjectExpression) genExp1.getSources().get(0);
        assertEquals( ls.getField(0).uid, prj1.getUid() );
       
        LogicalExpressionPlan genExp2 = gen.getOutputPlans().get(1);
        assertEquals( UserFuncExpression.class, genExp2.getSources().get(0).getClass() );
        assertEquals( ProjectExpression.class, genExp2.getSinks().get(0).getClass() );
        ProjectExpression prj2 = (ProjectExpression)genExp2.getSinks().get(0);
        assertEquals( ls.getField(1).uid, prj2.getUid() );
        assertEquals( BagDereferenceExpression.class, genExp2.getPredecessors(prj2).get(0).getClass() );
        assertEquals( 0, ((BagDereferenceExpression)genExp2.getPredecessors(prj2).get(0)).getBagColNum() );
       
        assertEquals( 1, inputPln.getRoots().size() );
        assertEquals( POProject.class, inputPln.getRoots().get(0).getClass() );
        assertEquals( 0, ((POProject)inputPln.getRoots().get(0)).getColumn() );
       
        PhysicalPlan inputPln2 = pForEach.getInputPlans().get(1);
        assertEquals( 1, inputPln2.getRoots().size() );
        assertEquals( POProject.class, inputPln2.getRoots().get(0).getClass() );
        assertEquals(1, ((POProject)inputPln2.getRoots().get(0)).getColumn() );
        assertEquals( POUserFunc.class, inputPln2.getLeaves().get(0).getClass() );
        assertEquals( "org.apache.pig.builtin.COUNT",
View Full Code Here
841
842
843
844
845
846
847
848
849
850
        prj1.setColumn(1);
        prj1.setOverloaded(true);
        ep1.add(prj1);
        eps1.add(ep1);
        flat1.add(true);
        POForEach fe = new POForEach(new OperatorKey(scope, nig
                .getNextNodeId(scope)), -1, eps1, flat1);
        fe.setResultType(DataType.BAG);
        return fe;
    }
View Full Code Here
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
            prj1.setColumn(0);
            prj1.setOverloaded(false);
            ep1.add(prj1);
            eps1.add(ep1);
            flat1.add(true);
            POForEach nfe1 = new POForEach(new OperatorKey(scope, nig
                    .getNextNodeId(scope)), op.getRequestedParallelism(), eps1,
                    flat1);
            nfe1.setResultType(DataType.BAG);
            curMROp.reducePlan.addAsLeaf(nfe1);
            curMROp.setNeedsDistinctCombiner(true);
            phyToMROpMap.put(op, curMROp);
        }catch(Exception e){
            int errCode = 2034;
View Full Code Here
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
              CompilerUtils.addEmptyBagOuterJoin(ep, op.getSchema(i));
          }
          flat.add(true);
      }

      POForEach fe = new POForEach(new OperatorKey(scope,nig.getNextNodeId(scope)), -1, eps, flat);
      fe.setResultType(DataType.TUPLE);
     
      fe.visit(this);
     
      curMROp.setSkewedJoinPartitionFile(partitionFile.getFileName());
      phyToMROpMap.put(op, curMROp);
        }catch(PlanException e) {
            int errCode = 2034;
View Full Code Here
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
            prj_c1.setOverloaded(false);
            prj_c1.setResultType(DataType.BAG);
            ep_c1.add(prj_c1);
            eps_c1.add(ep_c1);
            flat_c1.add(true);
            POForEach fe_c1 = new POForEach(new OperatorKey(scope,nig.getNextNodeId(scope)),
                -1, eps_c1, flat_c1);
            fe_c1.setResultType(DataType.TUPLE);
            mro.combinePlan.addAsLeaf(fe_c1);
           
            POLimit pLimit = new POLimit(new OperatorKey(scope,nig.getNextNodeId(scope)));
          pLimit.setLimit(limit);
          mro.combinePlan.addAsLeaf(pLimit);
           
            List<PhysicalPlan> eps_c2 = new ArrayList<PhysicalPlan>();
            eps_c2.addAll(sort.getSortPlans());
       
          POLocalRearrange lr_c2 = new POLocalRearrange(new OperatorKey(scope,nig.getNextNodeId(scope)));
          try {
                lr_c2.setIndex(0);
            } catch (ExecException e) {
              int errCode = 2058;
              String msg = "Unable to set index on newly created POLocalRearrange.";             
                throw new PlanException(msg, errCode, PigException.BUG, e);
            }
          lr_c2.setKeyType((fields.length>1) ? DataType.TUPLE : keyType);
          lr_c2.setPlans(eps_c2);
          lr_c2.setResultType(DataType.TUPLE);
          mro.combinePlan.addAsLeaf(lr_c2);
        }
       
        POPackageLite pkg = new POPackageLite(new OperatorKey(scope,nig.getNextNodeId(scope)));
        pkg.setKeyType((fields == null || fields.length>1) ? DataType.TUPLE :
            keyType);
        pkg.setNumInps(1);      
        mro.reducePlan.add(pkg);
       
        PhysicalPlan ep = new PhysicalPlan();
        POProject prj = new POProject(new OperatorKey(scope,nig.getNextNodeId(scope)));
        prj.setColumn(1);
        prj.setOverloaded(false);
        prj.setResultType(DataType.BAG);
        ep.add(prj);
        List<PhysicalPlan> eps2 = new ArrayList<PhysicalPlan>();
        eps2.add(ep);
        List<Boolean> flattened = new ArrayList<Boolean>();
        flattened.add(true);
        POForEach nfe1 = new POForEach(new OperatorKey(scope,nig.getNextNodeId(scope)),-1,eps2,flattened);
        mro.reducePlan.add(nfe1);
        mro.reducePlan.connect(pkg, nfe1);
       
        if (limit!=-1)
        {
View Full Code Here
TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach

  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.CombinerOptimizer
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler$LastInputStreamingOptimizer
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRUtil
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SecondaryKeyOptimizer
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result
  • org.apache.pig.backend.hadoop.executionengine.tez.plan.TezCompiler
  • org.apache.pig.backend.hadoop.executionengine.tez.util.TezCompilerUtil
  • org.apache.pig.backend.hadoop.executionengine.util.CombinerOptimizerUtil
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.