System.out.println("\tJobManager memory = "+jmMemory);
System.out.println("\tTaskManager memory = "+tmMemory);
System.out.println("\tTaskManager cores = "+tmCores);
// Create application via yarnClient
YarnClientApplication app = yarnClient.createApplication();
GetNewApplicationResponse appResponse = app.getNewApplicationResponse();
Resource maxRes = appResponse.getMaximumResourceCapability();
if(tmMemory > maxRes.getMemory() || tmCores > maxRes.getVirtualCores()) {
LOG.error("The cluster does not have the requested resources for the TaskManagers available!\n"
+ "Maximum Memory: "+maxRes.getMemory() +", Maximum Cores: "+tmCores);
yarnClient.stop();
System.exit(1);
}
if(jmMemory > maxRes.getMemory() ) {
LOG.error("The cluster does not have the requested resources for the JobManager available!\n"
+ "Maximum Memory: "+maxRes.getMemory());
yarnClient.stop();
System.exit(1);
}
int totalMemoryRequired = jmMemory + tmMemory * taskManagerCount;
ClusterResourceDescription freeClusterMem = getCurrentFreeClusterResources(yarnClient);
if(freeClusterMem.totalFreeMemory < totalMemoryRequired) {
LOG.error("This YARN session requires "+totalMemoryRequired+"MB of memory in the cluster. "
+ "There are currently only "+freeClusterMem.totalFreeMemory+"MB available.");
yarnClient.stop();
System.exit(1);
}
if( tmMemory > freeClusterMem.containerLimit) {
LOG.error("The requested amount of memory for the TaskManagers ("+tmMemory+"MB) is more than "
+ "the largest possible YARN container: "+freeClusterMem.containerLimit);
yarnClient.stop();
System.exit(1);
}
if( jmMemory > freeClusterMem.containerLimit) {
LOG.error("The requested amount of memory for the JobManager ("+jmMemory+"MB) is more than "
+ "the largest possible YARN container: "+freeClusterMem.containerLimit);
yarnClient.stop();
System.exit(1);
}
// respect custom JVM options in the YAML file
final String javaOpts = GlobalConfiguration.getString(ConfigConstants.FLINK_JVM_OPTIONS, "");
// Set up the container launch context for the application master
ContainerLaunchContext amContainer = Records
.newRecord(ContainerLaunchContext.class);
String amCommand = "$JAVA_HOME/bin/java"
+ " -Xmx"+Utils.calculateHeapSize(jmMemory)+"M " +javaOpts;
if(hasLogback || hasLog4j) {
amCommand += " -Dlog.file=\""+ApplicationConstants.LOG_DIR_EXPANSION_VAR +"/jobmanager-main.log\"";
}
if(hasLogback) {
amCommand += " -Dlogback.configurationFile=file:logback.xml";
}
if(hasLog4j) {
amCommand += " -Dlog4j.configuration=file:log4j.properties";
}
amCommand += " "+ApplicationMaster.class.getName()+" "
+ " 1>"
+ ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/jobmanager-stdout.log"
+ " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/jobmanager-stderr.log";
amContainer.setCommands(Collections.singletonList(amCommand));
System.err.println("amCommand="+amCommand);
// Set-up ApplicationSubmissionContext for the application
ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
final ApplicationId appId = appContext.getApplicationId();
/**
* All network ports are offsetted by the application number
* to avoid version port clashes when running multiple Flink sessions
* in parallel