/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.client.program;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.net.InetSocketAddress;
import java.util.List;
import org.apache.commons.lang3.StringEscapeUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.flink.api.common.JobExecutionResult;
import org.apache.flink.api.common.Plan;
import org.apache.flink.compiler.CompilerException;
import org.apache.flink.compiler.DataStatistics;
import org.apache.flink.compiler.PactCompiler;
import org.apache.flink.compiler.contextcheck.ContextChecker;
import org.apache.flink.compiler.costs.DefaultCostEstimator;
import org.apache.flink.compiler.plan.OptimizedPlan;
import org.apache.flink.compiler.plandump.PlanJSONDumpGenerator;
import org.apache.flink.compiler.plantranslate.NepheleJobGraphGenerator;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.GlobalConfiguration;
import org.apache.flink.core.fs.Path;
import org.apache.flink.runtime.client.JobClient;
import org.apache.flink.runtime.client.JobExecutionException;
import org.apache.flink.runtime.client.JobSubmissionResult;
import org.apache.flink.runtime.client.AbstractJobResult.ReturnCode;
import org.apache.flink.runtime.jobgraph.JobGraph;
import com.google.common.base.Preconditions;
import org.apache.flink.api.java.ExecutionEnvironment;
/**
* Encapsulates the functionality necessary to submit a program to a remote cluster.
*/
public class Client {
private static final Logger LOG = LoggerFactory.getLogger(Client.class);
private final Configuration configuration; // the configuration describing the job manager address
private final PactCompiler compiler; // the compiler to compile the jobs
private final ClassLoader userCodeClassLoader;
private boolean printStatusDuringExecution;
// ------------------------------------------------------------------------
// Construction
// ------------------------------------------------------------------------
/**
* Creates a new instance of the class that submits the jobs to a job-manager.
* at the given address using the default port.
*
* @param jobManagerAddress Address and port of the job-manager.
*/
public Client(InetSocketAddress jobManagerAddress, Configuration config, ClassLoader userCodeClassLoader) {
Preconditions.checkNotNull(config, "Configuration is null");
this.configuration = config;
configuration.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, jobManagerAddress.getAddress().getHostAddress());
configuration.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, jobManagerAddress.getPort());
this.userCodeClassLoader = userCodeClassLoader;
this.compiler = new PactCompiler(new DataStatistics(), new DefaultCostEstimator());
}
/**
* Creates a instance that submits the programs to the job-manager defined in the
* configuration.
*
* @param config The config used to obtain the job-manager's address.
*/
public Client(Configuration config, ClassLoader userCodeClassLoader) {
Preconditions.checkNotNull(config, "Configuration is null");
this.configuration = config;
// instantiate the address to the job manager
final String address = config.getString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, null);
if (address == null) {
throw new CompilerException("Cannot find address to job manager's RPC service in the global configuration.");
}
final int port = GlobalConfiguration.getInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, ConfigConstants.DEFAULT_JOB_MANAGER_IPC_PORT);
if (port < 0) {
throw new CompilerException("Cannot find port to job manager's RPC service in the global configuration.");
}
this.userCodeClassLoader = userCodeClassLoader;
this.compiler = new PactCompiler(new DataStatistics(), new DefaultCostEstimator());
}
public void setPrintStatusDuringExecution(boolean print) {
this.printStatusDuringExecution = print;
}
public String getJobManagerAddress() {
return this.configuration.getString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, null);
}
public int getJobManagerPort() {
return this.configuration.getInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, -1);
}
// ------------------------------------------------------------------------
// Compilation and Submission
// ------------------------------------------------------------------------
public String getOptimizedPlanAsJson(PackagedProgram prog, int parallelism) throws CompilerException, ProgramInvocationException {
PlanJSONDumpGenerator jsonGen = new PlanJSONDumpGenerator();
return jsonGen.getOptimizerPlanAsJSON(getOptimizedPlan(prog, parallelism));
}
public OptimizedPlan getOptimizedPlan(PackagedProgram prog, int parallelism) throws CompilerException, ProgramInvocationException {
Thread.currentThread().setContextClassLoader(prog.getUserCodeClassLoader());
if (prog.isUsingProgramEntryPoint()) {
return getOptimizedPlan(prog.getPlanWithJars(), parallelism);
}
else if (prog.isUsingInteractiveMode()) {
// temporary hack to support the optimizer plan preview
OptimizerPlanEnvironment env = new OptimizerPlanEnvironment(this.compiler);
if (parallelism > 0) {
env.setDegreeOfParallelism(parallelism);
}
env.setAsContext();
// temporarily write syser and sysout to bytearray.
PrintStream originalOut = System.out;
PrintStream originalErr = System.err;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
System.setOut(new PrintStream(baos));
ByteArrayOutputStream baes = new ByteArrayOutputStream();
System.setErr(new PrintStream(baes));
try {
ContextEnvironment.disableLocalExecution();
prog.invokeInteractiveModeForExecution();
}
catch (ProgramInvocationException e) {
throw e;
}
catch (Throwable t) {
// the invocation gets aborted with the preview plan
if (env.optimizerPlan != null) {
return env.optimizerPlan;
} else {
throw new ProgramInvocationException("The program caused an error: ", t);
}
} finally {
System.setOut(originalOut);
System.setErr(originalErr);
System.err.println(baes);
System.out.println(baos);
}
throw new ProgramInvocationException(
"The program plan could not be fetched - the program aborted pre-maturely. <br/><br/>"
+ "System.err: "+StringEscapeUtils.escapeHtml4(baes.toString())+" <br/>"
+ "System.out: "+StringEscapeUtils.escapeHtml4(baos.toString())+" <br/>" );
}
else {
throw new RuntimeException();
}
}
public OptimizedPlan getOptimizedPlan(Plan p, int parallelism) throws CompilerException {
if (parallelism > 0 && p.getDefaultParallelism() <= 0) {
p.setDefaultParallelism(parallelism);
}
ContextChecker checker = new ContextChecker();
checker.check(p);
return this.compiler.compile(p);
}
/**
* Creates the optimized plan for a given program, using this client's compiler.
*
* @param prog The program to be compiled.
* @return The compiled and optimized plan, as returned by the compiler.
* @throws CompilerException Thrown, if the compiler encounters an illegal situation.
* @throws ProgramInvocationException Thrown, if the program could not be instantiated from its jar file.
*/
public OptimizedPlan getOptimizedPlan(JobWithJars prog, int parallelism) throws CompilerException, ProgramInvocationException {
return getOptimizedPlan(prog.getPlan(), parallelism);
}
public JobGraph getJobGraph(PackagedProgram prog, OptimizedPlan optPlan) throws ProgramInvocationException {
return getJobGraph(optPlan, prog.getAllLibraries());
}
private JobGraph getJobGraph(OptimizedPlan optPlan, List<File> jarFiles) {
NepheleJobGraphGenerator gen = new NepheleJobGraphGenerator();
JobGraph job = gen.compileJobGraph(optPlan);
for (File jar : jarFiles) {
job.addJar(new Path(jar.getAbsolutePath()));
}
return job;
}
public JobExecutionResult run(final PackagedProgram prog, int parallelism, boolean wait) throws ProgramInvocationException {
Thread.currentThread().setContextClassLoader(prog.getUserCodeClassLoader());
if (prog.isUsingProgramEntryPoint()) {
return run(prog.getPlanWithJars(), parallelism, wait);
}
else if (prog.isUsingInteractiveMode()) {
ContextEnvironment env = new ContextEnvironment(this, prog.getAllLibraries(), prog.getUserCodeClassLoader());
if (parallelism > 0) {
env.setDegreeOfParallelism(parallelism);
}
env.setAsContext();
ContextEnvironment.disableLocalExecution();
if (wait) {
// invoke here
prog.invokeInteractiveModeForExecution();
}
else {
// invoke in the background
Thread backGroundRunner = new Thread("Program Runner") {
public void run() {
try {
prog.invokeInteractiveModeForExecution();
}
catch (Throwable t) {
LOG.error("The program execution failed.", t);
}
}
};
backGroundRunner.start();
}
return null;
}
else {
throw new RuntimeException();
}
}
public JobExecutionResult run(PackagedProgram prog, OptimizedPlan optimizedPlan, boolean wait) throws ProgramInvocationException {
return run(optimizedPlan, prog.getAllLibraries(), wait);
}
/**
* Runs a program on the nephele system whose job-manager is configured in this client's configuration.
* This method involves all steps, from compiling, job-graph generation to submission.
*
* @param prog The program to be executed.
* @param wait A flag that indicates whether this function call should block until the program execution is done.
* @throws CompilerException Thrown, if the compiler encounters an illegal situation.
* @throws ProgramInvocationException Thrown, if the program could not be instantiated from its jar file,
* or if the submission failed. That might be either due to an I/O problem,
* i.e. the job-manager is unreachable, or due to the fact that the execution
* on the nephele system failed.
*/
public JobExecutionResult run(JobWithJars prog, int parallelism, boolean wait) throws CompilerException, ProgramInvocationException {
return run(getOptimizedPlan(prog, parallelism), prog.getJarFiles(), wait);
}
public JobExecutionResult run(OptimizedPlan compiledPlan, List<File> libraries, boolean wait) throws ProgramInvocationException {
JobGraph job = getJobGraph(compiledPlan, libraries);
return run(job, wait);
}
public JobExecutionResult run(JobGraph jobGraph, boolean wait) throws ProgramInvocationException {
JobClient client;
try {
client = new JobClient(jobGraph, configuration, this.userCodeClassLoader);
}
catch (IOException e) {
throw new ProgramInvocationException("Could not open job manager: " + e.getMessage());
}
client.setConsoleStreamForReporting(this.printStatusDuringExecution ? System.out : null);
try {
if (wait) {
return client.submitJobAndWait();
}
else {
JobSubmissionResult result = client.submitJob();
if (result.getReturnCode() != ReturnCode.SUCCESS) {
throw new ProgramInvocationException("The job was not successfully submitted to the nephele job manager"
+ (result.getDescription() == null ? "." : ": " + result.getDescription()));
}
}
}
catch (IOException e) {
throw new ProgramInvocationException("Could not submit job to job manager: " + e.getMessage());
}
catch (JobExecutionException jex) {
if(jex.isJobCanceledByUser()) {
throw new ProgramInvocationException("The program has been canceled");
} else {
throw new ProgramInvocationException("The program execution failed: " + jex.getMessage());
}
}
return new JobExecutionResult(-1, null);
}
// --------------------------------------------------------------------------------------------
private static final class OptimizerPlanEnvironment extends ExecutionEnvironment {
private final PactCompiler compiler;
private OptimizedPlan optimizerPlan;
private OptimizerPlanEnvironment(PactCompiler compiler) {
this.compiler = compiler;
}
@Override
public JobExecutionResult execute(String jobName) throws Exception {
Plan plan = createProgramPlan(jobName);
this.optimizerPlan = compiler.compile(plan);
// do not go on with anything now!
throw new ProgramAbortException();
}
@Override
public String getExecutionPlan() throws Exception {
Plan plan = createProgramPlan();
this.optimizerPlan = compiler.compile(plan);
// do not go on with anything now!
throw new ProgramAbortException();
}
private void setAsContext() {
initializeContextEnvironment(this);
}
}
public static final class ProgramAbortException extends Error {
private static final long serialVersionUID = 1L;
}
}