Package org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan

297
298
299
300
301
302
303
304
305
306
307
        planTester.buildPlan("B = group A by $0;");
        planTester.buildPlan("C = foreach B { D = limit A 10; E = order D by $0, $1, $2; generate group, E;};");
       
        LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
       
        SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
        so.visit();
       
        assertTrue(so.getNumMRUseSecondaryKey()==1);
View Full Code Here
316
317
318
319
320
321
322
323
324
325
326
        planTester.buildPlan("B = group A by $0;");
        planTester.buildPlan("C = foreach B { D = limit A 10; E = order D by $1; F = order E by $2; generate group, F;};");
       
        LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
       
        SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
        so.visit();
       
        assertTrue(so.getNumMRUseSecondaryKey()==1);
View Full Code Here
335
336
337
338
339
340
341
342
343
344
345
        planTester.buildPlan("B = group A by $0;");
        planTester.buildPlan("C = foreach B { D = order A by $0 desc; generate group, D;};");
       
        LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
       
        SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
        so.visit();
       
        assertTrue(so.getNumMRUseSecondaryKey()==1);
View Full Code Here
354
355
356
357
358
359
360
361
362
363
364
        planTester.buildPlan("B = group A by ($0, $1);");
        planTester.buildPlan("C = foreach B { D = order A by $0, $1 desc; generate group, D;};");
       
        LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
       
        SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
        so.visit();
       
        assertTrue(so.getNumMRUseSecondaryKey()==1);
View Full Code Here
374
375
376
377
378
379
380
381
382
383
384
        planTester.buildPlan("B = group A all;");
        planTester.buildPlan("C = foreach B { D = order A by $0 desc; generate DIFF(D, D);};");
       
        LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
       
        SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
        so.visit();
       
        assertTrue(so.getNumMRUseSecondaryKey()==1);
View Full Code Here
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
            pig.registerQuery("store E into 'alias_output';");
           
            LogicalPlan lp = getLogicalPlan(pig);
            PhysicalPlan pp = pig.getPigContext().getExecutionEngine().compile(lp,
                    null);
            MROperPlan mp = getMRPlan(pp, pig.getPigContext());
           
            assertEquals(3, mp.getKeys().size());
           
            MapReduceOper mro = mp.getRoots().get(0);
            assertEquals("A,B,C", getAlias(mro));
           
            mro = mp.getSuccessors(mro).get(0);
            assertEquals("D", getAlias(mro));
            
            mro = mp.getSuccessors(mro).get(0);
            assertEquals("D", getAlias(mro));
        } finally {
            File outputfile = new File("alias_output");
            if (outputfile.exists()) {
                // Hadoop Local mode creates a directory
View Full Code Here
200
201
202
203
204
205
206
207
208
209
210
            PigContext pigContext) throws MRCompilerException {
        super(plan, new DepthFirstWalker<PhysicalOperator, PhysicalPlan>(plan));
        this.plan = plan;
        this.pigContext = pigContext;
        splitsSeen = new HashMap<OperatorKey, MapReduceOper>();
        MRPlan = new MROperPlan();
        nig = NodeIdGenerator.getGenerator();
        r = new Random(1331);
        FileLocalizer.setR(r);
        udfFinder = new UDFFinder();
        List<PhysicalOperator> roots = plan.getRoots();
View Full Code Here
450
451
452
453
454
455
456
457
458
459
460
461
462
      planTester.buildPlan("a = load 'input';");
      LogicalPlan lp = planTester.buildPlan("b = order a by $0;");
      PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
      POStore store = GenPhyOp.dummyPigStorageOp();
      pp.addAsLeaf(store);
      MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
     
      for(MapReduceOper mro: mrPlan.getLeaves()) {
        if(mro.reducePlan != null) {
          PhysicalOperator po = mro.reducePlan.getRoots().get(0);
          if(po instanceof POPackage) {
            ((POPackage)po).setKeyType(DataType.BAG);
            mro.setGlobalSort(true);
View Full Code Here
483
484
485
486
487
488
489
490
491
492
493
        planTester.buildPlan("a = load 'input';");
        LogicalPlan lp = planTester.buildPlan("b = group a by $0;");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        POStore store = GenPhyOp.dummyPigStorageOp();
        pp.addAsLeaf(store);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);

        HExecutionEngine exe = pc.getExecutionEngine();
        ConfigurationValidator.validatePigProperties(exe.getConfiguration());
        Configuration conf = ConfigurationUtil.toConfiguration(exe.getConfiguration());
        JobControlCompiler jcc = new JobControlCompiler(pc, conf);
View Full Code Here
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
        planTester.buildPlan("a = load 'input';");
        LogicalPlan lp = planTester.buildPlan("b = order a by $0;");
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        POStore store = GenPhyOp.dummyPigStorageOp();
        pp.addAsLeaf(store);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);

        HExecutionEngine exe = pc.getExecutionEngine();
        ConfigurationValidator.validatePigProperties(exe.getConfiguration());
        Configuration conf = ConfigurationUtil.toConfiguration(exe.getConfiguration());
        JobControlCompiler jcc = new JobControlCompiler(pc, conf);
       
        // Get the sort job
        Iterator<MapReduceOper> iter = mrPlan.getKeys().values().iterator();
        int counter = 0;
        while (iter.hasNext()) {
            MapReduceOper op = iter.next();
            counter++;
            if (op.isGlobalSort()) {
View Full Code Here
TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan

  • com.netflix.lipstick.MRPlanCalculatorTest
  • com.netflix.lipstick.pigtolipstick.BasicP2LClientTest
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.LocalLauncher
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler
  • org.apache.pig.pen.LocalMapReduceSimulator
  • org.apache.pig.test.TestFRJoin2
  • org.apache.pig.test.TestGroupConstParallel
  • org.apache.pig.test.TestGroupConstParallelMR
  • org.apache.pig.test.TestJobControlCompiler
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.