ApplicationId appId = resMgrDelegate.getApplicationId();
FileSystem fs = FileSystem.get(conf);
// Loads the job.xml written by the user.
JobConf jobConf = new JobConf(new TezConfiguration(conf));
// Extract individual raw MR configs.
Configuration[] stageConfs = MultiStageMRConfToTezTranslator
.getStageConfs(jobConf);
// Transform all confs to use Tez keys
MultiStageMRConfToTezTranslator.translateVertexConfToTez(stageConfs[0],
null);
for (int i = 1; i < stageConfs.length; i++) {
MultiStageMRConfToTezTranslator.translateVertexConfToTez(stageConfs[i],
stageConfs[i - 1]);
}
// create inputs to tezClient.submit()
// FIXME set up job resources
Map<String, LocalResource> jobLocalResources =
createJobLocalResources(stageConfs[0], jobSubmitDir);
// FIXME createDAG should take the tezConf as a parameter, instead of using
// MR keys.
DAG dag = createDAG(fs, jobId, stageConfs, jobSubmitDir, ts,
jobLocalResources);
List<String> vargs = new LinkedList<String>();
// admin command opts and user command opts
String mrAppMasterAdminOptions = conf.get(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS,
MRJobConfig.DEFAULT_MR_AM_ADMIN_COMMAND_OPTS);
warnForJavaLibPath(mrAppMasterAdminOptions, "app master",
MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS, MRJobConfig.MR_AM_ADMIN_USER_ENV);
vargs.add(mrAppMasterAdminOptions);
// Add AM user command opts
String mrAppMasterUserOptions = conf.get(MRJobConfig.MR_AM_COMMAND_OPTS,
MRJobConfig.DEFAULT_MR_AM_COMMAND_OPTS);
warnForJavaLibPath(mrAppMasterUserOptions, "app master",
MRJobConfig.MR_AM_COMMAND_OPTS, MRJobConfig.MR_AM_ENV);
vargs.add(mrAppMasterUserOptions);
StringBuilder javaOpts = new StringBuilder();
for (String varg : vargs) {
javaOpts.append(varg).append(" ");
}
// Setup the CLASSPATH in environment
// i.e. add { Hadoop jars, job jar, CWD } to classpath.
Map<String, String> environment = new HashMap<String, String>();
// Setup the environment variables for AM
MRHelpers.updateEnvironmentForMRAM(conf, environment);
TezConfiguration dagAMConf = getDAGAMConfFromMRConf();
dagAMConf.set(TezConfiguration.TEZ_AM_JAVA_OPTS, javaOpts.toString());
// Submit to ResourceManager
try {
dagAMConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
jobSubmitDir);
// Set Tez parameters based on MR parameters.
String queueName = jobConf.get(JobContext.QUEUE_NAME,
YarnConfiguration.DEFAULT_QUEUE_NAME);
dagAMConf.set(TezConfiguration.TEZ_QUEUE_NAME, queueName);
int amMemMB = jobConf.getInt(MRJobConfig.MR_AM_VMEM_MB, MRJobConfig.DEFAULT_MR_AM_VMEM_MB);
int amCores = jobConf.getInt(MRJobConfig.MR_AM_CPU_VCORES, MRJobConfig.DEFAULT_MR_AM_CPU_VCORES);
dagAMConf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, amMemMB);
dagAMConf.setInt(TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES, amCores);
dagAMConf.setInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS,
jobConf.getInt(MRJobConfig.MR_AM_MAX_ATTEMPTS, MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS));
AMConfiguration amConfig = new AMConfiguration(
environment,
jobLocalResources, dagAMConf, ts);