Package org.apache.pig.backend.executionengine

Examples of org.apache.pig.backend.executionengine.ExecutionEngine

140
141
142
143
144
145
146
147
148
149
150
        // front end and back end and needs to be changed.
        // Right now I am not clear on how the Job Id comes from to tell
        // the back end to kill a given job (mJobClient is used only in
        // processKill)
        //
        ExecutionEngine execEngine = mPigServer.getPigContext().getExecutionEngine();
        if (execEngine instanceof HExecutionEngine) {
            mJobClient = ((HExecutionEngine)execEngine).getJobClient();
        }
        else {
            mJobClient = null;
View Full Code Here
463
464
465
466
467
468
469
470
471
472
473
474
475
            mro.setGlobalSort(true);
          }
        }
      }
     
        ExecutionEngine exe = pc.getExecutionEngine();
        ConfigurationValidator.validatePigProperties(exe.getConfiguration());
        Configuration conf = ConfigurationUtil.toConfiguration(exe.getConfiguration());
        JobControlCompiler jcc = new JobControlCompiler();
        try {
          jcc.compile(mrPlan, "Test", conf, pc);
        } catch (JobCreationException jce) {
          assertTrue(jce.getErrorCode() == 1068);
View Full Code Here
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
        PhysicalPlan rpep = new PhysicalPlan();
        ConstantExpression rpce = new ConstantExpression(new OperatorKey(scope,nig.getNextNodeId(scope)));
        rpce.setRequestedParallelism(rp);
        int val = rp;
        if(val<=0){
            ExecutionEngine eng = pigContext.getExecutionEngine();
            if(eng instanceof HExecutionEngine){
                try {
                    val = Math.round(0.9f * ((HExecutionEngine)eng).getJobClient().getDefaultReduces());
                    if(val<=0)
                        val = 1;
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
88
89
                                                   Exception {
        long sleepTime = 5000;
        aggregateWarning = "true".equalsIgnoreCase(pc.getProperties().getProperty("aggregate.warning"));
        MROperPlan mrp = compile(php, pc);
       
        ExecutionEngine exe = pc.getExecutionEngine();
        ConfigurationValidator.validatePigProperties(exe.getConfiguration());
        Configuration conf = ConfigurationUtil.toConfiguration(exe.getConfiguration());
        JobClient jobClient = ((HExecutionEngine)exe).getJobClient();

        JobControlCompiler jcc = new JobControlCompiler();
       
        JobControl jc = jcc.compile(mrp, grpName, conf, pc);
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
69
                                  IOException, ExecException,
                                  JobCreationException, Exception {
        long sleepTime = 500;
        MROperPlan mrp = compile(php, pc);
       
        ExecutionEngine exe = pc.getExecutionEngine();
        Properties validatedProperties = ConfigurationValidator.getValidatedProperties(exe.getConfiguration());
        Configuration conf = ConfigurationUtil.toConfiguration(validatedProperties);
        conf.set("mapred.job.tracker", "local");
        JobClient jobClient = new JobClient(new JobConf(conf));

        JobControlCompiler jcc = new JobControlCompiler();
View Full Code Here
194
195
196
197
198
199
200
201
202
203
204
        // front end and back end and needs to be changed.
        // Right now I am not clear on how the Job Id comes from to tell
        // the back end to kill a given job (mJobClient is used only in
        // processKill)
        //
        ExecutionEngine execEngine = mPigServer.getPigContext().getExecutionEngine();
        if (execEngine instanceof HExecutionEngine) {
            mJobClient = ((HExecutionEngine)execEngine).getJobClient();
        }
        else {
            mJobClient = null;
View Full Code Here
463
464
465
466
467
468
469
470
471
472
473
474
475
            mro.setGlobalSort(true);
          }
        }
      }
     
        ExecutionEngine exe = pc.getExecutionEngine();
        ConfigurationValidator.validatePigProperties(exe.getConfiguration());
        Configuration conf = ConfigurationUtil.toConfiguration(exe.getConfiguration());
        JobControlCompiler jcc = new JobControlCompiler(pc, conf);
        try {
          jcc.compile(mrPlan, "Test");
        } catch (JobCreationException jce) {
            assertTrue(jce.getErrorCode() == 1068);
View Full Code Here
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
        PhysicalPlan rpep = new PhysicalPlan();
        ConstantExpression rpce = new ConstantExpression(new OperatorKey(scope,nig.getNextNodeId(scope)));
        rpce.setRequestedParallelism(rp);
        int val = rp;
        if(val<=0){
            ExecutionEngine eng = pigContext.getExecutionEngine();
            if(eng instanceof HExecutionEngine){
                try {
                    val = Math.round(0.9f * ((HExecutionEngine)eng).getJobClient().getDefaultReduces());
                    if(val<=0)
                        val = 1;
View Full Code Here
102
103
104
105
106
107
108
109
110
111
112
113
114
        PigStats stats = new PigStats();
        stats.setMROperatorPlan(mrp);
        stats.setExecType(pc.getExecType());
        stats.setPhysicalPlan(php);
       
        ExecutionEngine exe = pc.getExecutionEngine();
        ConfigurationValidator.validatePigProperties(exe.getConfiguration());
        Configuration conf = ConfigurationUtil.toConfiguration(exe.getConfiguration());
        JobClient jobClient = ((HExecutionEngine)exe).getJobClient();

        JobControlCompiler jcc = new JobControlCompiler(pc, conf);
       
        List<Job> failedJobs = new LinkedList<Job>();
View Full Code Here
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
        PhysicalPlan rpep = new PhysicalPlan();
        ConstantExpression rpce = new ConstantExpression(new OperatorKey(scope,nig.getNextNodeId(scope)));
        rpce.setRequestedParallelism(rp);
        int val = rp;
        if(val<=0){
            ExecutionEngine eng = pigContext.getExecutionEngine();
            if(eng instanceof HExecutionEngine){
                try {
                    val = ((JobConf)((HExecutionEngine)eng).getJobClient().getConf()).getNumReduceTasks();
                    if(val<=0)
                        val = 1;
View Full Code Here
TOP

Related Classes of org.apache.pig.backend.executionengine.ExecutionEngine

  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.LocalLauncher
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler
  • org.apache.pig.test.TestJobSubmission
  • org.apache.pig.tools.grunt.GruntParser
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.