Package org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan

194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
        POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
        prj.setResultType(DataType.DOUBLE);
        PONegative pn = new PONegative(new OperatorKey("", r.nextLong()), -1, prj);
        pn.setResultType(DataType.DOUBLE);
       
        PhysicalPlan plan = new PhysicalPlan();
        plan.add(prj); plan.add(pn);
        plan.connect(prj, pn);
       
        for(Iterator<Tuple> it = bag.iterator(); it.hasNext(); ) {
            Tuple t = it.next();
            plan.attachInput(t);
      Double expected = -(Double)t.get(0);
      double output = (Double) pn.getNext(expected).result;
      assertEquals(expected.doubleValue(), output);
        }
    }
View Full Code Here
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
        POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
        prj.setResultType(DataType.DOUBLE);
        PONegative pn = new PONegative(new OperatorKey("", r.nextLong()), -1, prj);
        pn.setResultType(DataType.DOUBLE);
       
        PhysicalPlan plan = new PhysicalPlan();
        plan.add(prj); plan.add(pn);
        plan.connect(prj, pn);
       
        for(Iterator<Tuple> it = bag.iterator(); it.hasNext(); ) {
            Tuple t = it.next();
            plan.attachInput(t);
           
            if(t.get(0) == null) {
              Double output = (Double )pn.getNext((Double )null).result;
                assertEquals(null, output);
            } else  {
View Full Code Here
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
        POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
        prj.setResultType(DataType.FLOAT);
        PONegative pn = new PONegative(new OperatorKey("", r.nextLong()), -1, prj);
        pn.setResultType(DataType.FLOAT);
       
        PhysicalPlan plan = new PhysicalPlan();
        plan.add(prj); plan.add(pn);
        plan.connect(prj, pn);
       
        for(Iterator<Tuple> it = bag.iterator(); it.hasNext(); ) {
            Tuple t = it.next();
            plan.attachInput(t);
            Float expected = -(Float)t.get(0);
            float output = (Float) pn.getNext(expected).result;
            assertEquals(expected.floatValue(), output);
        }
    }
View Full Code Here
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
        POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
        prj.setResultType(DataType.FLOAT);
        PONegative pn = new PONegative(new OperatorKey("", r.nextLong()), -1, prj);
        pn.setResultType(DataType.FLOAT);
       
        PhysicalPlan plan = new PhysicalPlan();
        plan.add(prj); plan.add(pn);
        plan.connect(prj, pn);
       
        for(Iterator<Tuple> it = bag.iterator(); it.hasNext(); ) {
            Tuple t = it.next();
            plan.attachInput(t);
           
            if(t.get(0) == null) {
              Float output = (Float)pn.getNext((Float)null).result;
                assertEquals(null, output);
            } else  {
View Full Code Here
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
                lps.println("Logical plan is empty.");
                pps.println("Physical plan is empty.");
                eps.println("Execution plan is empty.");
                return;
            }
            PhysicalPlan pp = compilePp(lp);
            lp.explain(lps, format, verbose);
            if( pigContext.getProperties().getProperty("pig.usenewlogicalplan", "true").equals("true") ) {
                LogicalPlanMigrationVistor migrator = new LogicalPlanMigrationVistor(lp);
                migrator.visit();
                org.apache.pig.newplan.logical.relational.LogicalPlan newPlan = migrator.getNewLogicalPlan();
               
                HashSet<String> optimizerRules = null;
                try {
                    optimizerRules = (HashSet<String>) ObjectSerializer
                            .deserialize(pigContext.getProperties().getProperty(
                                    "pig.optimizer.rules"));
                } catch (IOException ioe) {
                    int errCode = 2110;
                    String msg = "Unable to deserialize optimizer rules.";
                    throw new FrontendException(msg, errCode, PigException.BUG, ioe);
                }
               
                LogicalPlanOptimizer optimizer = new LogicalPlanOptimizer(newPlan, 3, optimizerRules);
                optimizer.optimize();               
               
                newPlan.explain(lps, format, verbose);
            }
            pp.explain(pps, format, verbose);
            pigContext.getExecutionEngine().explain(pp, eps, format, verbose);
            if (markAsExecute) {
                currDAG.markAsExecuted();
            }
        } catch (Exception e) {
View Full Code Here
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
    }
   
    private PigStats executeCompiledLogicalPlan(LogicalPlan compiledLp) throws ExecException, FrontendException {
        // discover pig features used in this script
        ScriptState.get().setScriptFeatures(compiledLp);
        PhysicalPlan pp = compilePp(compiledLp);
        // execute using appropriate engine
        List<ExecJob> jobs = pigContext.getExecutionEngine().execute(pp, "job_pigexec_");
        PigStats stats = null;
        if (jobs.size() > 0) {
            stats = jobs.get(0).getStatistics();
View Full Code Here
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
        return lp;
    }

    private PhysicalPlan compilePp(LogicalPlan lp) throws FrontendException {
        // translate lp to physical plan
        PhysicalPlan pp = pigContext.getExecutionEngine().compile(lp, null);

        // TODO optimize

        return pp;
    }
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
        POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
        prj.setResultType(DataType.INTEGER);
        PONegative pn = new PONegative(new OperatorKey("", r.nextLong()), -1, prj);
        pn.setResultType(DataType.INTEGER);
       
        PhysicalPlan plan = new PhysicalPlan();
        plan.add(prj); plan.add(pn);
        plan.connect(prj, pn);
       
        for(Iterator<Tuple> it = bag.iterator(); it.hasNext(); ) {
            Tuple t = it.next();
            plan.attachInput(t);
            Integer expected = -(Integer)t.get(0);
            int output = (Integer) pn.getNext(expected).result;
            assertEquals(expected.intValue(), output);
        }
    }
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
        POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
        prj.setResultType(DataType.INTEGER);
        PONegative pn = new PONegative(new OperatorKey("", r.nextLong()), -1, prj);
        pn.setResultType(DataType.INTEGER);
       
        PhysicalPlan plan = new PhysicalPlan();
        plan.add(prj); plan.add(pn);
        plan.connect(prj, pn);
       
        for(Iterator<Tuple> it = bag.iterator(); it.hasNext(); ) {
            Tuple t = it.next();
            plan.attachInput(t);
           
            if(t.get(0) == null) {
                Integer output = (Integer)pn.getNext((Integer)null).result;
                assertEquals(null, output);
View Full Code Here
426
427
428
429
430
431
432
433
434
435
436
437
438
    @Test
    public void testJobControlCompilerErr() throws Exception {
      LogicalPlanTester planTester = new LogicalPlanTester() ;
      planTester.buildPlan("a = load 'input';");
      LogicalPlan lp = planTester.buildPlan("b = order a by $0;");
      PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
      POStore store = GenPhyOp.dummyPigStorageOp();
      pp.addAsLeaf(store);
      MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
     
      for(MapReduceOper mro: mrPlan.getLeaves()) {
        if(mro.reducePlan != null) {
          PhysicalOperator po = mro.reducePlan.getRoots().get(0);
View Full Code Here
TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan

  • com.netflix.lipstick.MRPlanCalculatorTest
  • org.apache.pig.backend.hadoop.executionengine.HExecutionEngine
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.CombinerOptimizer
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRUtil
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MultiQueryOptimizer
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.NoopFilterRemover$PhysicalRemover
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.NoopStoreRemover
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.NoopStoreRemover$PhysicalRemover
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.