Package org.apache.pig.backend.hadoop.executionengine.mapReduceLayer

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher

255
256
257
258
259
260
261
262
263
264
265
266
    public ExecJob execute(PhysicalPlan plan,
                           String jobName) throws ExecException {
        try {
            FileSpec spec = ExecTools.checkLeafIsStore(plan, pigContext);

            MapReduceLauncher launcher = new MapReduceLauncher();
            boolean success = launcher.launchPig(plan, jobName, pigContext);
            if(success)
                return new HJob(ExecJob.JOB_STATUS.COMPLETED, pigContext, spec);
            else
                return new HJob(ExecJob.JOB_STATUS.FAILED, pigContext, null);
View Full Code Here
288
289
290
291
292
293
294
295
296
297
298
            printer.visit();
            stream.println();

            ExecTools.checkLeafIsStore(plan, pigContext);

            MapReduceLauncher launcher = new MapReduceLauncher();
            launcher.explain(plan, pigContext, stream);
        } catch (Exception ve) {
            throw new RuntimeException(ve);
        }
    }
View Full Code Here
637
638
639
640
641
642
643
644
645
646
647
648
649
650
        return pp;
    }

    private boolean executePlan(PhysicalPlan pp) throws IOException {
        boolean failed = true;
        MapReduceLauncher launcher = new MapReduceLauncher();
        PigStats stats = null;
        try {
            stats = launcher.launchPig(pp, "execute", myPig.getPigContext());
        } catch (Exception e) {
            e.printStackTrace(System.out);
            throw new IOException(e);
        }
        Iterator<JobStats> iter = stats.getJobGraph().iterator();
View Full Code Here
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
        System.out.println("===== check map-reduce plan =====");

        MapRedUtil.checkLeafIsStore(pp, myPig.getPigContext());
       
        MapReduceLauncher launcher = new MapReduceLauncher();

        MROperPlan mrp = null;

        try {
            java.lang.reflect.Method compile = launcher.getClass()
                    .getDeclaredMethod("compile",
                            new Class[] { PhysicalPlan.class, PigContext.class });

            compile.setAccessible(true);
View Full Code Here
852
853
854
855
856
857
858
859
860
861
862
863
864
    }

    public static MROperPlan buildMRPlanWithOptimizer(PhysicalPlan pp, PigContext pc) throws Exception {
        MapRedUtil.checkLeafIsStore(pp, pc);

        MapReduceLauncher launcher = new MapReduceLauncher();

        java.lang.reflect.Method compile = launcher.getClass()
                .getDeclaredMethod("compile",
                        new Class[] { PhysicalPlan.class, PigContext.class });

        compile.setAccessible(true);
View Full Code Here
232
233
234
235
236
237
238
239
240
241
242
243
        buildLp.setAccessible(true);
        return (LogicalPlan ) buildLp.invoke( pig );
    }
   
    public static MROperPlan getMRPlan(PhysicalPlan pp, PigContext ctx) throws Exception {
        MapReduceLauncher launcher = new MapReduceLauncher();
        java.lang.reflect.Method compile = launcher.getClass()
                .getDeclaredMethod("compile",
                        new Class[] { PhysicalPlan.class, PigContext.class });
        compile.setAccessible(true);
        return (MROperPlan) compile.invoke(launcher, new Object[] { pp, ctx });
    }
View Full Code Here
476
477
478
479
480
481
482
483
        }
    }
   
    private void submit() throws Exception{
        assertEquals(true, FileLocalizer.fileExists(hadoopLdFile, pc));
        MapReduceLauncher mrl = new MapReduceLauncher();
        mrl.launchPig(php, grpName, pc); 
    }
View Full Code Here
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
            currDAG.lp.explain(lps, format, verbose);

            pp.explain(pps, format, verbose);
           
            MapRedUtil.checkLeafIsStore(pp, pigContext);
            MapReduceLauncher launcher = new MapReduceLauncher();
            launcher.explain(pp, pigContext, eps, format, verbose);

            if (markAsExecute) {
                currDAG.markAsExecuted();
            }
        } catch (Exception e) {
View Full Code Here
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
     * @return The PigStats object
     * @throws ExecException
     * @throws FrontendException
     */
    protected PigStats launchPlan(PhysicalPlan pp, String jobName) throws ExecException, FrontendException {
        MapReduceLauncher launcher = new MapReduceLauncher();
        PigStats stats = null;
        try {
            stats = launcher.launchPig(pp, jobName, pigContext);
        } catch (Exception e) {
            // There are a lot of exceptions thrown by the launcher.  If this
            // is an ExecException, just let it through.  Else wrap it.
            if (e instanceof ExecException){
                throw (ExecException)e;
            } else if (e instanceof FrontendException) {
                throw (FrontendException)e;
            } else {
                int errCode = 2043;
                String msg = "Unexpected error during execution.";
                throw new ExecException(msg, errCode, PigException.BUG, e);
            }
        } finally {
            launcher.reset();
        }
       
        for (OutputStats output : stats.getOutputStats()) {
            if (!output.isSuccessful()) {
                POStore store = output.getPOStore();
View Full Code Here
985
986
987
988
989
990
991
992
993
994
995
996
            currDAG.lp.explain(lps, format, verbose);

            pp.explain(pps, format, verbose);

            MapRedUtil.checkLeafIsStore(pp, pigContext);
            MapReduceLauncher launcher = new MapReduceLauncher();
            launcher.explain(pp, pigContext, eps, format, verbose);

            if (markAsExecute) {
                currDAG.markAsExecuted();
            }
        } catch (Exception e) {
View Full Code Here
TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher

  • org.apache.hadoop.conf.Configuration
  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.mapred.Counters
  • org.apache.hadoop.mapred.JobClient
  • org.apache.hadoop.mapred.JobConf
  • org.apache.hadoop.mapred.jobcontrol.Job
  • org.apache.hadoop.mapred.jobcontrol.JobControl
  • org.apache.hadoop.mapred.JobID
  • org.apache.hadoop.mapred.RunningJob
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.