Package org.apache.pig.backend.hadoop.executionengine.mapReduceLayer

Examples of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher


        }
    }

    public List<ExecJob> execute(PhysicalPlan plan,
                                 String jobName) throws ExecException {
        MapReduceLauncher launcher = new MapReduceLauncher();
        List<ExecJob> jobs = new ArrayList<ExecJob>();

        try {
            PigStats stats = launcher.launchPig(plan, jobName, pigContext);

            for (FileSpec spec: launcher.getSucceededFiles()) {
                jobs.add(new HJob(ExecJob.JOB_STATUS.COMPLETED, pigContext, spec, stats));
            }

            for (FileSpec spec: launcher.getFailedFiles()) {
                HJob j = new HJob(ExecJob.JOB_STATUS.FAILED, pigContext, spec, stats);
                j.setException(launcher.getError(spec));
                jobs.add(j);
            }

            return jobs;
        } catch (Exception e) {
            // There are a lot of exceptions thrown by the launcher.  If this
            // is an ExecException, just let it through.  Else wrap it.
            if (e instanceof ExecException) throw (ExecException)e;
            else {
                int errCode = 2043;
                String msg = "Unexpected error during execution.";
                throw new ExecException(msg, errCode, PigException.BUG, e);
            }
        } finally {
            launcher.reset();
        }

    }
View Full Code Here


    public void explain(PhysicalPlan plan, PrintStream stream, String format, boolean verbose) {
        try {
            ExecTools.checkLeafIsStore(plan, pigContext);

            MapReduceLauncher launcher = new MapReduceLauncher();
            launcher.explain(plan, pigContext, stream, format, verbose);

        } catch (Exception ve) {
            throw new RuntimeException(ve);
        }
    }
View Full Code Here

        }
    }

    public List<ExecJob> execute(PhysicalPlan plan,
                                 String jobName) throws ExecException, FrontendException {
        MapReduceLauncher launcher = new MapReduceLauncher();
        List<ExecJob> jobs = new ArrayList<ExecJob>();

        Map<String, PhysicalOperator> leafMap = new HashMap<String, PhysicalOperator>();
        for (PhysicalOperator physOp : plan.getLeaves()) {
            log.info(physOp);
            if (physOp instanceof POStore) {
                FileSpec spec = ((POStore) physOp).getSFile();
                if (spec != null)
                    leafMap.put(spec.toString(), physOp);
            }
        }
        try {
            PigStats stats = launcher.launchPig(plan, jobName, pigContext);

            for (OutputStats output : stats.getOutputStats()) {
                POStore store = output.getPOStore();              
                String alias = store.getAlias();
                if (output.isSuccessful()) {
                    jobs.add(new HJob(ExecJob.JOB_STATUS.COMPLETED, pigContext, store, alias, stats));
                } else {
                    HJob j = new HJob(ExecJob.JOB_STATUS.FAILED, pigContext, store, alias, stats)
                    j.setException(launcher.getError(store.getSFile()));
                    jobs.add(j);
                }
            }

            return jobs;
        } catch (Exception e) {
            // There are a lot of exceptions thrown by the launcher.  If this
            // is an ExecException, just let it through.  Else wrap it.
            if (e instanceof ExecException){
              throw (ExecException)e;
            } else if (e instanceof FrontendException) {
              throw (FrontendException)e;
            } else {
                int errCode = 2043;
                String msg = "Unexpected error during execution.";
                throw new ExecException(msg, errCode, PigException.BUG, e);
            }
        } finally {
            launcher.reset();
        }

    }
View Full Code Here

    public void explain(PhysicalPlan plan, PrintStream stream, String format, boolean verbose) {
        try {
            MapRedUtil.checkLeafIsStore(plan, pigContext);

            MapReduceLauncher launcher = new MapReduceLauncher();
            launcher.explain(plan, pigContext, stream, format, verbose);

        } catch (Exception ve) {
            throw new RuntimeException(ve);
        }
    }
View Full Code Here

        assertEquals(reducer, sort.getRequestedParallelism());
    }
   
    private void submit() throws Exception{
        assertEquals(true, FileLocalizer.fileExists(hadoopLdFile, pc));
        MapReduceLauncher mrl = new MapReduceLauncher();
        mrl.launchPig(php, grpName, pc)
    }
View Full Code Here

    }
   
    public static MROperPlan buildMRPlanWithOptimizer(PhysicalPlan pp, PigContext pc) throws Exception {
        MapRedUtil.checkLeafIsStore(pp, pc);
       
        MapReduceLauncher launcher = new MapReduceLauncher();

        java.lang.reflect.Method compile = launcher.getClass()
                .getDeclaredMethod("compile",
                        new Class[] { PhysicalPlan.class, PigContext.class });

        compile.setAccessible(true);
View Full Code Here

        System.out.println("===== check map-reduce plan =====");

        MapRedUtil.checkLeafIsStore(pp, myPig.getPigContext());
       
        MapReduceLauncher launcher = new MapReduceLauncher();

        MROperPlan mrp = null;

        try {
            java.lang.reflect.Method compile = launcher.getClass()
                    .getDeclaredMethod("compile",
                            new Class[] { PhysicalPlan.class, PigContext.class });

            compile.setAccessible(true);
View Full Code Here

        pp.add(pcount);
        //pp.connect(proj, st);
        pp.connect(proj, pcount);
        pp.connect(pcount, st);
        pc.setExecType(ExecType.LOCAL);
        return new MapReduceLauncher().launchPig(pp, "TestStore", pc);
    }
View Full Code Here

        compileLp.setAccessible(true);
        return (LogicalPlan) compileLp.invoke(pig, new Object[] { null });
    }
   
    public static MROperPlan getMRPlan(PhysicalPlan pp, PigContext ctx) throws Exception {
        MapReduceLauncher launcher = new MapReduceLauncher();
        java.lang.reflect.Method compile = launcher.getClass()
                .getDeclaredMethod("compile",
                        new Class[] { PhysicalPlan.class, PigContext.class });
        compile.setAccessible(true);
        return (MROperPlan) compile.invoke(launcher, new Object[] { pp, ctx });
    }
View Full Code Here

    }
   
    public static MROperPlan buildMRPlanWithOptimizer(PhysicalPlan pp, PigContext pc) throws Exception {
        MapRedUtil.checkLeafIsStore(pp, pc);
       
        MapReduceLauncher launcher = new MapReduceLauncher();

        java.lang.reflect.Method compile = launcher.getClass()
                .getDeclaredMethod("compile",
                        new Class[] { PhysicalPlan.class, PigContext.class });

        compile.setAccessible(true);
View Full Code Here

TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.