return new URL(url.toString());
}
public static HttpConnectionParams constructHttpShuffleConnectionParams(
Configuration conf) {
HttpConnectionParamsBuilder builder = new HttpConnectionParamsBuilder();
int connectionTimeout =
conf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_CONNECT_TIMEOUT,
TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_STALLED_COPY_TIMEOUT_DEFAULT);
int readTimeout =
conf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_READ_TIMEOUT,
TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_READ_TIMEOUT_DEFAULT);
int bufferSize =
conf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_BUFFER_SIZE,
TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_BUFFER_SIZE_DEFAULT);
boolean keepAlive =
conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_KEEP_ALIVE_ENABLED,
TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_KEEP_ALIVE_ENABLED_DEFAULT);
int keepAliveMaxConnections =
conf.getInt(
TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_KEEP_ALIVE_MAX_CONNECTIONS,
TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_KEEP_ALIVE_MAX_CONNECTIONS_DEFAULT);
if (keepAlive) {
System.setProperty("sun.net.http.errorstream.enableBuffering", "true");
System.setProperty("http.maxConnections",
String.valueOf(keepAliveMaxConnections));
LOG.info("Set keepAlive max connections: " + keepAliveMaxConnections);
}
builder.setTimeout(connectionTimeout, readTimeout)
.setBufferSize(bufferSize)
.setKeepAlive(keepAlive, keepAliveMaxConnections);
boolean sslShuffle = conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_ENABLE_SSL,
TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_ENABLE_SSL_DEFAULT);
builder.setSSL(sslShuffle, conf);
return builder.build();
}