Package org.apache.pig.backend.hadoop.executionengine.mapReduceLayer

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper

102
103
104
105
106
107
108
109
110
111
112
113
        if(lrFound != pkg.getNumInps()) {
            // we did not find the LocalRearrange(s) in the map plan
            // let's look in the predecessors
            List<MapReduceOper> preds = this.mPlan.getPredecessors(mr);
            for (Iterator<MapReduceOper> it = preds.iterator(); it.hasNext();) {
                MapReduceOper mrOper = it.next();
                if (mrOper.isLimitOnly())
                    mrOper = this.mPlan.getPredecessors(mrOper).get(0);
                lrFound += patchPackage(mrOper.reducePlan, pkg);
                if(lrFound == pkg.getNumInps()) {
                    break;
                }    
View Full Code Here
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
            LogicalPlan lp = checkLogicalPlan(1, 2, 14);
            PhysicalPlan pp = checkPhysicalPlan(lp, 1, 2, 20);
            MROperPlan mp = checkMRPlan(pp, 1, 2, 3);

            MapReduceOper mo1 = mp.getRoots().get(0);
            MapReduceOper mo2 = mp.getLeaves().get(0);
            MapReduceOper mo3 = mp.getLeaves().get(1);

            checkPhysicalPlan(mo1.mapPlan, 1, 1, 3);
            checkPhysicalPlan(mo1.reducePlan, 1, 1, 2);
            PhysicalOperator leaf = mo1.reducePlan.getLeaves().get(0);
           
View Full Code Here
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
            LogicalPlan lp = checkLogicalPlan(1, 1, 7);
            PhysicalPlan pp = checkPhysicalPlan(lp, 1, 1, 11);
            MROperPlan mp = checkMRPlan(pp, 1, 1, 2);

            MapReduceOper mo1 = mp.getRoots().get(0);
            MapReduceOper mo2 = mp.getLeaves().get(0);

            checkPhysicalPlan(mo1.mapPlan, 1, 1, 3);
            checkPhysicalPlan(mo1.reducePlan, 1, 1, 2);
            PhysicalOperator leaf = mo1.reducePlan.getLeaves().get(0);
           
View Full Code Here
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
            LogicalPlan lp = checkLogicalPlan(1, 3, 10);
            PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 10);
            MROperPlan mp = checkMRPlan(pp, 1, 1, 1);

            MapReduceOper mo = mp.getRoots().get(0);

            checkPhysicalPlan(mo.mapPlan, 1, 1, 4);
            PhysicalOperator leaf = mo.mapPlan.getLeaves().get(0);
           
            Assert.assertTrue(leaf instanceof POSplit);
View Full Code Here
110
111
112
113
114
115
116
117
118
119
120
            lp = lpt.buildPlan("store C into 'out';");
            MROperPlan mrPlan = Util.buildMRPlan(Util.buildPhysicalPlan(lp, pc),pc);           
            assertEquals(2,mrPlan.size());

            Iterator<MapReduceOper> itr = mrPlan.iterator();
            MapReduceOper oper = itr.next();
            assertTrue(oper.reducePlan.isEmpty());
            assertFalse(oper.mapPlan.isEmpty());

            oper = itr.next();
            assertFalse(oper.reducePlan.isEmpty());
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
                    null);
            MROperPlan mp = getMRPlan(pp, pig.getPigContext());
           
            assertEquals(3, mp.getKeys().size());
           
            MapReduceOper mro = mp.getRoots().get(0);
            assertEquals("A,B,C", getAlias(mro));
           
            mro = mp.getSuccessors(mro).get(0);
            assertEquals("D", getAlias(mro));
            
View Full Code Here
169
170
171
172
173
174
175
176
177
        PhysicalPlan pp = Util.buildPhysicalPlanFromNewLP(lp, pc);
        POStore poStore = (POStore)pp.getLeaves().get(0);
        assert(poStore.getAlias().equals("B"));
       
        MROperPlan mrp = Util.buildMRPlanWithOptimizer(pp, pc);
        MapReduceOper mrOper = mrp.getLeaves().get(0);
        poStore = (POStore)mrOper.mapPlan.getLeaves().get(0);
        assert(poStore.getAlias().equals("B"));
    }
View Full Code Here
768
769
770
771
772
773
774
775
    public static void setR(Random r) {
        GenPhyOp.r = r;
    }
   
    public static MapReduceOper MROp(){
        MapReduceOper ret = new MapReduceOper(new OperatorKey("",r.nextLong()));
        return ret;
    }
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
        LogicalPlan lp = planTester.buildPlan("store B into '/tmp';");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);

        int count = 1;
        MapReduceOper mrOper = mrPlan.getRoots().get(0);
        while(mrPlan.getSuccessors(mrOper) != null) {
            mrOper = mrPlan.getSuccessors(mrOper).get(0);
            ++count;
        }
       
View Full Code Here
110
111
112
113
114
115
116
117
118
119
120
        LogicalPlan lp = planTester.buildPlan("store C into 'output';");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);

        int count = 1;
        MapReduceOper mrOper = mrPlan.getRoots().get(0);
        while(mrPlan.getSuccessors(mrOper) != null) {
            mrOper = mrPlan.getSuccessors(mrOper).get(0);
            ++count;
        }       
        // Before optimizer visits, number of MR jobs = 3.
View Full Code Here
TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper

  • com.twitter.ambrose.pig.AmbrosePigProgressNotificationListener
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.POPackageAnnotator
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POCounter
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POUnion
  • org.apache.pig.impl.plan.OperatorKey
  • org.apache.pig.pen.LocalMapReduceSimulator
  • org.apache.pig.test.TestFRJoin2
  • org.apache.pig.test.TestJobControlCompiler
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.