Package org.apache.pig.backend.hadoop.executionengine.mapReduceLayer

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper

1202
1203
1204
1205
1206
1207
1208
1209
1210
            "c = join a by $0, b by $0 using 'merge';" +
            "store c into 'output';";

        PhysicalPlan pp = Util.buildPp(pigServer, query);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
        MapReduceOper mrOper = mrPlan.getRoots().get(0);

        assertTrue(mrOper.UDFs.contains(TestIndexableLoadFunc.class.getName()));
    }
View Full Code Here
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
            "c = limit b 10;" +
            "store c into 'output';";

        PhysicalPlan pp = Util.buildPp(pigServer, query);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
        MapReduceOper secondMrOper = mrPlan.getLeaves().get(0);
        POStore store = (POStore)secondMrOper.reducePlan.getLeaves().get(0);
        assertEquals(
                "compare load and store schema",
                store.getSchema(),
                Utils.getSchemaFromString("a : int,b :float ,c : int")
View Full Code Here
1244
1245
1246
1247
1248
1249
1250
1251
1252
       
        LimitAdjuster la = new LimitAdjuster(mrPlan, pc);
        la.visit();
        la.adjust();
       
        MapReduceOper firstMrOper = mrPlan.getRoots().get(0);
        POStore store = (POStore)firstMrOper.reducePlan.getLeaves().get(0);
        assertEquals(store.getStoreFunc().getClass().getName(), "org.apache.pig.impl.io.InterStorage");
    }
View Full Code Here
113
114
115
116
117
118
119
120
121
122
123
    PigContext pigContext = pigServer.getPigContext();
    pigContext.connect();
    pigContext.addJar(tmpFile.getAbsolutePath());
    JobControlCompiler jobControlCompiler = new JobControlCompiler(pigContext, CONF);
    MROperPlan plan = new MROperPlan();
    MapReduceOper mro = new MapReduceOper(new OperatorKey());
    mro.UDFs = new HashSet<String>();
    mro.UDFs.add(className+"()");
    plan.add(mro);

    // compiling the job
View Full Code Here
178
179
180
181
182
183
184
185
186
187
188
                StringUtils.join(tarArchives, ","));
        final JobControlCompiler jobControlCompiler = new JobControlCompiler(
                pigContext, CONF);

        final MROperPlan plan = new MROperPlan();
        plan.add(new MapReduceOper(new OperatorKey()));

        final JobControl jobControl = jobControlCompiler.compile(plan, "test");
        final JobConf jobConf = jobControl.getWaitingJobs().get(0).getJobConf();
       
        assertFilesInDistributedCache(DistributedCache.getCacheFiles(jobConf),
View Full Code Here
203
204
205
206
207
208
209
210
211
        Assert.assertEquals(3, JobControlCompiler.estimateNumberOfReducers(
            new Job(CONF), createMockPOLoadMapReduceOper(2L * 1000 * 1000 * 1001)));
    }

    private static MapReduceOper createMockPOLoadMapReduceOper(long size) throws Exception {
        MapReduceOper mro = new MapReduceOper(new OperatorKey());
        mro.mapPlan.add(createPOLoadWithSize(size, new PigStorage()));
        return mro;
    }
View Full Code Here
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
            LogicalPlan lp = checkLogicalPlan(1, 2, 13);
            PhysicalPlan pp = checkPhysicalPlan(lp, 1, 2, 20);
            MROperPlan mp = checkMRPlan(pp, 1, 2, 3);

            MapReduceOper mo1 = mp.getRoots().get(0);
            MapReduceOper mo2 = mp.getLeaves().get(0);
            MapReduceOper mo3 = mp.getLeaves().get(1);

            checkPhysicalPlan(mo1.mapPlan, 1, 1, 3);
            checkPhysicalPlan(mo1.reducePlan, 1, 1, 2);
            PhysicalOperator leaf = mo1.reducePlan.getLeaves().get(0);
           
View Full Code Here
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
            LogicalPlan lp = checkLogicalPlan(1, 1, 6);
            PhysicalPlan pp = checkPhysicalPlan(lp, 1, 1, 11);
            MROperPlan mp = checkMRPlan(pp, 1, 1, 2);

            MapReduceOper mo1 = mp.getRoots().get(0);
            MapReduceOper mo2 = mp.getLeaves().get(0);

            checkPhysicalPlan(mo1.mapPlan, 1, 1, 3);
            checkPhysicalPlan(mo1.reducePlan, 1, 1, 2);
            PhysicalOperator leaf = mo1.reducePlan.getLeaves().get(0);
           
View Full Code Here
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
            LogicalPlan lp = checkLogicalPlan(1, 3, 9);
            PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 10);
            MROperPlan mp = checkMRPlan(pp, 1, 1, 1);

            MapReduceOper mo = mp.getRoots().get(0);

            checkPhysicalPlan(mo.mapPlan, 1, 1, 4);
            PhysicalOperator leaf = mo.mapPlan.getLeaves().get(0);
           
            Assert.assertTrue(leaf instanceof POSplit);
View Full Code Here
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
            //MROperPlan mrp = checkMRPlan(pp, 1, 1, 1);
            //MapReduceOper mrop = mrp.getRoots().get(0);

            //Instead of 1 merged mapreduce job, there will be two.
            MROperPlan mrp = checkMRPlan(pp, 1, 1, 2);
            MapReduceOper mrop = mrp.getLeaves().get(0);
            Assert.assertTrue(mrop.getCustomPartitioner().equals(SimpleCustomPartitioner.class.getName()));

        } catch (Exception e) {
            e.printStackTrace();
            Assert.fail();
        }
View Full Code Here
TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper

  • com.twitter.ambrose.pig.AmbrosePigProgressNotificationListener
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.POPackageAnnotator
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POCounter
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POUnion
  • org.apache.pig.impl.plan.OperatorKey
  • org.apache.pig.pen.LocalMapReduceSimulator
  • org.apache.pig.test.TestFRJoin2
  • org.apache.pig.test.TestJobControlCompiler
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.