Package org.apache.pig.backend.hadoop.executionengine

Examples of org.apache.pig.backend.hadoop.executionengine.HExecutionEngine

95
96
97
98
99
100
101
102
103
104
105
106
        return p2lMap;
    }

    private MROperPlan getMROperPlan(LipstickPigServer lps) throws Exception {
        HExecutionEngine he = new HExecutionEngine(lps.getPigContext());
        PhysicalPlan pp = he.compile(getLogicalPlan(lps), null);

        MRCompiler mrc = new MRCompiler(pp, lps.getPigContext());
        mrc.compile();
        return mrc.getMRPlan();
    }
View Full Code Here
233
234
235
236
237
238
239
240
241
242
243
        switch (execType) {
            case LOCAL:
            case MAPREDUCE:
            {
                executionEngine = new HExecutionEngine (this);

                executionEngine.init();

                dfs = executionEngine.getDataStorage();
View Full Code Here
224
225
226
227
228
229
230
231
        // front end and back end and needs to be changed.
        // Right now I am not clear on how the Job Id comes from to tell
        // the back end to kill a given job (mJobClient is used only in
        // processKill)
        //
        HExecutionEngine execEngine = mPigServer.getPigContext().getExecutionEngine();
        mJobConf = execEngine.getJobConf();
    }
View Full Code Here
173
174
175
176
177
178
179
180
181
182
183
            }
            break;

            case MAPREDUCE:
            {
                executionEngine = new HExecutionEngine (this);

                executionEngine.init();
               
                dfs = executionEngine.getDataStorage();
               
View Full Code Here
150
151
152
153
154
155
156
157
158
159
     * @throws ExecException if running the job fails.
     */
    public List<ExecJob> runPlan(LogicalPlan newPlan,
                                 String jobName) throws FrontendException, ExecException {

        HExecutionEngine engine = new HExecutionEngine(pigContext);
        PhysicalPlan pp = engine.compile(newPlan, null);
        PigStats stats = launchPlan(pp, jobName);
        return getJobs(stats);
    }
View Full Code Here
294
295
296
297
298
299
300
301
302
303
304
        switch (execType) {
            case LOCAL:
            case MAPREDUCE:
            {
                executionEngine = new HExecutionEngine (this);

                executionEngine.init();

                dfs = executionEngine.getDataStorage();
View Full Code Here
236
237
238
239
240
241
242
243
        // front end and back end and needs to be changed.
        // Right now I am not clear on how the Job Id comes from to tell
        // the back end to kill a given job (mJobClient is used only in
        // processKill)
        //
        HExecutionEngine execEngine = mPigServer.getPigContext().getExecutionEngine();
        mJobConf = execEngine.getJobConf();
    }
View Full Code Here
153
154
155
156
157
158
159
160
161
162
163
164
        MROperPlan mrp = compile(php, pc);

        ConfigurationValidator.validatePigProperties(pc.getProperties());
        Configuration conf = ConfigurationUtil.toConfiguration(pc.getProperties());

        HExecutionEngine exe = pc.getExecutionEngine();
        JobClient jobClient = new JobClient(exe.getJobConf());

        JobControlCompiler jcc = new JobControlCompiler(pc, conf);

        ScriptState.get().addWorkflowAdjacenciesToConf(mrp, conf);
View Full Code Here
150
151
152
153
154
155
156
157
158
159
160
161
        MROperPlan mrp = compile(php, pc);
               
        ConfigurationValidator.validatePigProperties(pc.getProperties());
        Configuration conf = ConfigurationUtil.toConfiguration(pc.getProperties());
       
        HExecutionEngine exe = pc.getExecutionEngine();
        JobClient jobClient = new JobClient(exe.getJobConf());
       
        JobControlCompiler jcc = new JobControlCompiler(pc, conf);
       
        // start collecting statistics
        PigStatsUtil.startCollection(pc, jobClient, jcc, mrp);
View Full Code Here
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
        PhysicalPlan rpep = new PhysicalPlan();
        ConstantExpression rpce = new ConstantExpression(new OperatorKey(scope,nig.getNextNodeId(scope)));
        rpce.setRequestedParallelism(rp);
        int val = rp;
        if(val<=0){
            HExecutionEngine eng = pigContext.getExecutionEngine();
            if(pigContext.getExecType() != ExecType.LOCAL){
                try {
                    if(val<=0)
                        val = pigContext.defaultParallel;
                    if (val<=0)
                        val = eng.getJobConf().getNumReduceTasks();
                    if (val<=0)
                        val = 1;
                } catch (Exception e) {
                    int errCode = 6015;
                    String msg = "Problem getting the default number of reduces from the Job Client.";
View Full Code Here
TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.HExecutionEngine

  • com.netflix.lipstick.MRPlanCalculatorTest
  • com.netflix.lipstick.pigtolipstick.BasicP2LClientTest
  • org.apache.hadoop.conf.Configuration
  • org.apache.hadoop.hdfs.DistributedFileSystem
  • org.apache.hadoop.mapred.JobClient
  • org.apache.hadoop.mapred.JobConf
  • org.apache.pig.backend.executionengine.ExecLogicalPlan
  • org.apache.pig.backend.hadoop.datastorage.HDataStorage
  • org.apache.pig.backend.hadoop.executionengine.fetch.FetchLauncher
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.