JobControlCompiler jcc = new JobControlCompiler(pc, conf);
List<Job> failedJobs = new LinkedList<Job>();
List<Job> succJobs = new LinkedList<Job>();
JobControl jc;
int totalMRJobs = mrp.size();
int numMRJobsCompl = 0;
int numMRJobsCurrent = 0;
double lastProg = -1;
//create the exception handler for the job control thread
//and register the handler with the job control thread
JobControlThreadExceptionHandler jctExceptionHandler = new JobControlThreadExceptionHandler();
while((jc = jcc.compile(mrp, grpName)) != null) {
numMRJobsCurrent = jc.getWaitingJobs().size();
Thread jcThread = new Thread(jc);
jcThread.setUncaughtExceptionHandler(jctExceptionHandler);
jcThread.start();
while(!jc.allFinished()){
try {
Thread.sleep(sleepTime);
} catch (InterruptedException e) {}
double prog = (numMRJobsCompl+calculateProgress(jc, jobClient))/totalMRJobs;
if(prog>=(lastProg+0.01)){
int perCom = (int)(prog * 100);
if(perCom!=100)
log.info( perCom + "% complete");
}
lastProg = prog;
}
//check for the jobControlException first
//if the job controller fails before launching the jobs then there are
//no jobs to check for failure
if(jobControlException != null) {
if(jobControlException instanceof PigException) {
throw jobControlException;
} else {
int errCode = 2117;
String msg = "Unexpected error when launching map reduce job.";
throw new ExecException(msg, errCode, PigException.BUG, jobControlException);
}
}
numMRJobsCompl += numMRJobsCurrent;
failedJobs.addAll(jc.getFailedJobs());
if (!failedJobs.isEmpty()
&& "true".equalsIgnoreCase(
pc.getProperties().getProperty("stop.on.failure","false"))) {
int errCode = 6017;
StringBuilder msg = new StringBuilder("Execution failed, while processing ");
for (Job j: failedJobs) {
List<POStore> sts = jcc.getStores(j);
for (POStore st: sts) {
msg.append(st.getSFile().getFileName());
msg.append(", ");
}
}
throw new ExecException(msg.substring(0,msg.length()-2),
errCode, PigException.REMOTE_ENVIRONMENT);
}
List<Job> jobs = jc.getSuccessfulJobs();
jcc.moveResults(jobs);
succJobs.addAll(jobs);
stats.setJobClient(jobClient);
stats.setJobControl(jc);
stats.accumulateStats();
jc.stop();
}
log.info( "100% complete");
boolean failed = false;