/*
* Copyright (c) 2008-2014 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.cometd.benchmark.server;
import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.lang.management.ManagementFactory;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collections;
import java.util.Enumeration;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import javax.servlet.ServletContext;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.cometd.bayeux.server.BayeuxServer;
import org.cometd.bayeux.server.ServerMessage;
import org.cometd.bayeux.server.ServerSession;
import org.cometd.benchmark.Config;
import org.cometd.benchmark.MonitoringQueuedThreadPool;
import org.cometd.benchmark.MonitoringThreadPoolExecutor;
import org.cometd.server.AbstractServerTransport;
import org.cometd.server.AbstractService;
import org.cometd.server.BayeuxServerImpl;
import org.cometd.server.CometDServlet;
import org.cometd.server.Jackson1JSONContextServer;
import org.cometd.server.ext.AcknowledgedMessagesExtension;
import org.cometd.server.transport.AsyncJSONTransport;
import org.cometd.server.transport.JSONTransport;
import org.cometd.websocket.server.JettyWebSocketTransport;
import org.cometd.websocket.server.WebSocketTransport;
import org.eclipse.jetty.jmx.MBeanContainer;
import org.eclipse.jetty.server.AbstractConnectionFactory;
import org.eclipse.jetty.server.ConnectionFactory;
import org.eclipse.jetty.server.HttpConnectionFactory;
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.handler.HandlerWrapper;
import org.eclipse.jetty.server.handler.StatisticsHandler;
import org.eclipse.jetty.servlet.DefaultServlet;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.eclipse.jetty.toolchain.perf.MeasureRecorder;
import org.eclipse.jetty.toolchain.perf.PlatformMonitor;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.eclipse.jetty.websocket.jsr356.server.deploy.WebSocketServerContainerInitializer;
public class BayeuxLoadServer
{
public static void main(String[] args) throws Exception
{
BayeuxLoadServer server = new BayeuxLoadServer();
server.run();
}
public void run() throws Exception
{
BufferedReader console = new BufferedReader(new InputStreamReader(System.in));
int port = 8080;
System.err.printf("listen port [%d]: ", port);
String value = console.readLine().trim();
if (value.length() == 0)
value = String.valueOf(port);
port = Integer.parseInt(value);
boolean ssl = false;
System.err.printf("use ssl [%b]: ", ssl);
value = console.readLine().trim();
if (value.length() == 0)
value = String.valueOf(ssl);
ssl = Boolean.parseBoolean(value);
int selectors = Runtime.getRuntime().availableProcessors();
System.err.printf("selectors [%d]: ", selectors);
value = console.readLine().trim();
if (value.length() == 0)
value = String.valueOf(selectors);
selectors = Integer.parseInt(value);
int maxThreads = Integer.parseInt(System.getProperty("cometd.threads", "256"));
System.err.printf("max threads [%d]: ", maxThreads);
value = console.readLine().trim();
if (value.length() == 0)
value = String.valueOf(maxThreads);
maxThreads = Integer.parseInt(value);
BayeuxServerImpl bayeuxServer = new BayeuxServerImpl();
bayeuxServer.addExtension(new AcknowledgedMessagesExtension());
MonitoringQueuedThreadPool jettyThreadPool = new MonitoringQueuedThreadPool(maxThreads);
MonitoringThreadPoolExecutor websocketThreadPool = new MonitoringThreadPoolExecutor(maxThreads, jettyThreadPool.getIdleTimeout(), TimeUnit.MILLISECONDS, new ThreadPoolExecutor.AbortPolicy());
String availableTransports = "jsrws,jettyws,http,asynchttp";
String transports = "jsrws,http";
System.err.printf("transports (%s) [%s]: ", availableTransports, transports);
value = console.readLine().trim();
if (value.length() == 0)
value = transports;
for (String token : value.split(","))
{
switch (token.trim())
{
case "jsrws":
bayeuxServer.addTransport(new LoadWebSocketTransport(bayeuxServer, websocketThreadPool));
break;
case "jettyws":
bayeuxServer.addTransport(new LoadJettyWebSocketTransport(bayeuxServer, websocketThreadPool));
break;
case "http":
bayeuxServer.addTransport(new JSONTransport(bayeuxServer));
break;
case "asynchttp":
bayeuxServer.addTransport(new AsyncJSONTransport(bayeuxServer));
break;
default:
throw new IllegalArgumentException("Invalid transport: " + token);
}
}
boolean stats = true;
System.err.printf("record statistics [%b]: ", stats);
value = console.readLine().trim();
if (value.length() == 0)
value = String.valueOf(stats);
stats = Boolean.parseBoolean(value);
boolean reqs = true;
System.err.printf("record latencies [%b]: ", reqs);
value = console.readLine().trim();
if (value.length() == 0)
value = String.valueOf(reqs);
reqs = Boolean.parseBoolean(value);
boolean qos = false;
System.err.printf("detect long requests [%b]: ", qos);
value = console.readLine().trim();
if (value.length() == 0)
value = String.valueOf(qos);
qos = Boolean.parseBoolean(value);
Server server = new Server(jettyThreadPool);
// Setup JMX
MBeanContainer mbeanContainer = new MBeanContainer(ManagementFactory.getPlatformMBeanServer());
server.addBean(mbeanContainer);
SslContextFactory sslContextFactory = null;
if (ssl)
{
Path keyStoreFile = Paths.get("src/main/resources/keystore.jks");
if (Files.exists(keyStoreFile))
throw new FileNotFoundException(keyStoreFile.toString());
sslContextFactory = new SslContextFactory();
sslContextFactory.setKeyStorePath(keyStoreFile.toString());
sslContextFactory.setKeyStorePassword("storepwd");
sslContextFactory.setKeyManagerPassword("keypwd");
}
ConnectionFactory[] factories = AbstractConnectionFactory.getFactories(sslContextFactory, new HttpConnectionFactory());
ServerConnector connector = new ServerConnector(server, null, null, null, 1, selectors, factories);
// Make sure the OS is configured properly for load testing;
// see http://cometd.org/documentation/howtos/loadtesting
connector.setAcceptQueueSize(2048);
// Make sure the server timeout on a TCP connection is large
connector.setIdleTimeout(50 * Config.MAX_NETWORK_DELAY);
connector.setPort(port);
server.addConnector(connector);
HandlerWrapper handler = server;
RequestLatencyHandler requestLatencyHandler = null;
if (reqs)
{
requestLatencyHandler = new RequestLatencyHandler();
handler.setHandler(requestLatencyHandler);
handler = requestLatencyHandler;
}
if (qos)
{
RequestQoSHandler requestQoSHandler = new RequestQoSHandler();
handler.setHandler(requestQoSHandler);
handler = requestQoSHandler;
}
StatisticsHandler statisticsHandler = null;
if (stats)
{
statisticsHandler = new StatisticsHandler();
handler.setHandler(statisticsHandler);
handler = statisticsHandler;
}
// Add more handlers if needed
ServletContextHandler context = new ServletContextHandler(handler, Config.CONTEXT_PATH, ServletContextHandler.SESSIONS);
context.setAttribute(BayeuxServer.ATTRIBUTE, bayeuxServer);
context.setInitParameter(ServletContextHandler.MANAGED_ATTRIBUTES, BayeuxServer.ATTRIBUTE);
WebSocketServerContainerInitializer.configureContext(context);
// Setup default servlet to serve static files
context.addServlet(DefaultServlet.class, "/");
// Setup comet servlet
String cometdURLMapping = Config.SERVLET_PATH + "/*";
CometDServlet cometServlet = new CometDServlet();
ServletHolder cometdServletHolder = new ServletHolder(cometServlet);
context.addServlet(cometdServletHolder, cometdURLMapping);
// Make sure the expiration timeout is large to avoid clients to timeout
// This value must be several times larger than the client value
// (e.g. 60 s on server vs 5 s on client) so that it's guaranteed that
// it will be the client to dispose idle connections.
bayeuxServer.setOption(AbstractServerTransport.MAX_INTERVAL_OPTION, String.valueOf(10 * Config.MAX_NETWORK_DELAY));
// Explicitly set the timeout value
bayeuxServer.setOption(AbstractServerTransport.TIMEOUT_OPTION, String.valueOf(Config.META_CONNECT_TIMEOUT));
// Use the faster JSON parser/generator
bayeuxServer.setOption(AbstractServerTransport.JSON_CONTEXT_OPTION, Jackson1JSONContextServer.class.getName());
bayeuxServer.setOption("ws.cometdURLMapping", cometdURLMapping);
bayeuxServer.setOption(ServletContext.class.getName(), context.getServletContext());
server.start();
new StatisticsService(bayeuxServer, jettyThreadPool, websocketThreadPool, statisticsHandler, requestLatencyHandler);
}
public static class StatisticsService extends AbstractService
{
private final PlatformMonitor monitor = new PlatformMonitor();
private final MonitoringQueuedThreadPool jettyThreadPool;
private final MonitoringThreadPoolExecutor websocketThreadPool;
private final StatisticsHandler statisticsHandler;
private final RequestLatencyHandler requestLatencyHandler;
private StatisticsService(BayeuxServer bayeux, MonitoringQueuedThreadPool jettyThreadPool, MonitoringThreadPoolExecutor websocketThreadPool, StatisticsHandler statisticsHandler, RequestLatencyHandler requestLatencyHandler)
{
super(bayeux, "statistics-service");
this.jettyThreadPool = jettyThreadPool;
this.websocketThreadPool = websocketThreadPool;
this.statisticsHandler = statisticsHandler;
this.requestLatencyHandler = requestLatencyHandler;
addService("/service/statistics/start", "startStatistics");
addService("/service/statistics/stop", "stopStatistics");
}
public void startStatistics(ServerSession remote, ServerMessage message)
{
// Multiple nodes must wait that initialization is completed
synchronized (this)
{
PlatformMonitor.Start start = monitor.start();
if (start != null)
{
System.err.println();
System.err.println(start);
if (jettyThreadPool != null)
jettyThreadPool.reset();
if (websocketThreadPool != null)
websocketThreadPool.reset();
if (statisticsHandler != null)
statisticsHandler.statsReset();
if (requestLatencyHandler != null)
{
requestLatencyHandler.reset();
requestLatencyHandler.doNotTrackCurrentRequest();
}
}
}
}
public void stopStatistics(ServerSession remote, ServerMessage message) throws Exception
{
synchronized (this)
{
PlatformMonitor.Stop stop = monitor.stop();
if (stop != null)
{
System.err.println(stop);
if (requestLatencyHandler != null)
{
requestLatencyHandler.print();
requestLatencyHandler.doNotTrackCurrentRequest();
}
if (statisticsHandler != null)
{
System.err.printf("Requests times (total/avg/max - stddev): %d/%d/%d ms - %d%n",
statisticsHandler.getDispatchedTimeTotal(),
((Double)statisticsHandler.getDispatchedTimeMean()).longValue(),
statisticsHandler.getDispatchedTimeMax(),
((Double)statisticsHandler.getDispatchedTimeStdDev()).longValue());
System.err.printf("Requests (total/failed/max - rate): %d/%d/%d - %d requests/s%n",
statisticsHandler.getDispatched(),
statisticsHandler.getResponses4xx() + statisticsHandler.getResponses5xx(),
statisticsHandler.getDispatchedActiveMax(),
statisticsHandler.getStatsOnMs() == 0 ? -1 : statisticsHandler.getDispatched() * 1000L / statisticsHandler.getStatsOnMs());
}
if (jettyThreadPool != null)
{
System.err.printf("Jetty Thread Pool - Tasks = %d | Concurrent Threads max = %d | Queue Size max = %d | Queue Latency avg/max = %d/%d ms | Task Latency avg/max = %d/%d ms%n",
jettyThreadPool.getTasks(),
jettyThreadPool.getMaxActiveThreads(),
jettyThreadPool.getMaxQueueSize(),
TimeUnit.NANOSECONDS.toMillis(jettyThreadPool.getAverageQueueLatency()),
TimeUnit.NANOSECONDS.toMillis(jettyThreadPool.getMaxQueueLatency()),
TimeUnit.NANOSECONDS.toMillis(jettyThreadPool.getAverageTaskLatency()),
TimeUnit.NANOSECONDS.toMillis(jettyThreadPool.getMaxTaskLatency()));
}
if (websocketThreadPool != null)
{
System.err.printf("WebSocket Thread Pool - Tasks = %d | Concurrent Threads max = %d | Queue Size max = %d | Queue Latency avg/max = %d/%d ms | Task Latency avg/max = %d/%d ms%n",
websocketThreadPool.getTasks(),
websocketThreadPool.getMaxActiveThreads(),
websocketThreadPool.getMaxQueueSize(),
TimeUnit.NANOSECONDS.toMillis(websocketThreadPool.getAverageQueueLatency()),
TimeUnit.NANOSECONDS.toMillis(websocketThreadPool.getMaxQueueLatency()),
TimeUnit.NANOSECONDS.toMillis(websocketThreadPool.getAverageTaskLatency()),
TimeUnit.NANOSECONDS.toMillis(websocketThreadPool.getMaxTaskLatency()));
}
System.err.println();
}
}
}
}
private static class RequestQoSHandler extends HandlerWrapper
{
private final long maxRequestTime = 500;
private final AtomicLong requestIds = new AtomicLong();
private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(50);
@Override
protected void doStop() throws Exception
{
super.doStop();
scheduler.shutdown();
}
@Override
public void handle(String target, Request request, final HttpServletRequest httpRequest, HttpServletResponse httpResponse) throws IOException, ServletException
{
final long requestId = requestIds.incrementAndGet();
final AtomicBoolean longRequest = new AtomicBoolean(false);
final Thread thread = Thread.currentThread();
ScheduledFuture<?> task = scheduler.scheduleWithFixedDelay(new Runnable()
{
public void run()
{
longRequest.set(true);
onLongRequestDetected(requestId, httpRequest, thread);
}
}, maxRequestTime, maxRequestTime, TimeUnit.MILLISECONDS);
long start = System.nanoTime();
try
{
super.handle(target, request, httpRequest, httpResponse);
}
finally
{
long end = System.nanoTime();
task.cancel(false);
if (longRequest.get())
{
onLongRequestEnded(requestId, end - start);
}
}
}
private void onLongRequestDetected(long requestId, HttpServletRequest request, Thread thread)
{
try
{
long begin = System.nanoTime();
StackTraceElement[] stackFrames = thread.getStackTrace();
StringBuilder builder = new StringBuilder();
formatRequest(request, builder);
builder.append(thread).append("\n");
formatStackFrames(stackFrames, builder);
System.err.println("Request #" + requestId + " is too slow (> " + maxRequestTime + " ms)\n" + builder);
long end = System.nanoTime();
System.err.println("Request #" + requestId + " printed in " + TimeUnit.NANOSECONDS.toMicros(end - begin) + " \u00B5s");
}
catch (Exception x)
{
x.printStackTrace();
}
}
private void formatRequest(HttpServletRequest request, StringBuilder builder)
{
builder.append(request.getRequestURI()).append("\n");
for (Enumeration<String> headers = request.getHeaderNames(); headers.hasMoreElements(); )
{
String name = headers.nextElement();
builder.append(name).append("=").append(Collections.list(request.getHeaders(name))).append("\n");
}
builder.append(request.getRemoteAddr()).append(":").append(request.getRemotePort()).append(" => ");
builder.append(request.getLocalAddr()).append(":").append(request.getLocalPort()).append("\n");
}
private void onLongRequestEnded(long requestId, long time)
{
System.err.println("Request #" + requestId + " lasted " + TimeUnit.NANOSECONDS.toMillis(time) + " ms");
}
private void formatStackFrames(StackTraceElement[] stackFrames, StringBuilder builder)
{
for (int i = 0; i < stackFrames.length; ++i)
{
StackTraceElement stackFrame = stackFrames[i];
for (int j = 0; j < i; ++j)
builder.append(" ");
builder.append(stackFrame).append("\n");
}
}
}
private static class RequestLatencyHandler extends HandlerWrapper implements MeasureRecorder.Converter
{
private final MeasureRecorder recorder = new MeasureRecorder(this, "Requests - Latency", "ms");
private final ThreadLocal<Boolean> currentEnabled = new ThreadLocal<Boolean>()
{
@Override
protected Boolean initialValue()
{
return Boolean.TRUE;
}
};
@Override
public void handle(String target, Request request, final HttpServletRequest httpRequest, HttpServletResponse httpResponse) throws IOException, ServletException
{
long begin = System.nanoTime();
try
{
super.handle(target, request, httpRequest, httpResponse);
}
finally
{
long end = System.nanoTime();
if (currentEnabled.get())
{
updateLatencies(begin, end);
}
else
{
currentEnabled.set(true);
}
}
}
@Override
public long convert(long measure)
{
return TimeUnit.NANOSECONDS.toMillis(measure);
}
private void reset()
{
recorder.reset();
}
private void updateLatencies(long begin, long end)
{
recorder.record(end - begin, true);
}
private void print()
{
System.err.println(recorder.snapshot());
}
public void doNotTrackCurrentRequest()
{
currentEnabled.set(false);
}
}
public static class LoadJettyWebSocketTransport extends JettyWebSocketTransport
{
private final Executor executor;
public LoadJettyWebSocketTransport(BayeuxServerImpl bayeux, Executor executor)
{
super(bayeux);
this.executor = executor;
}
@Override
protected Executor newExecutor()
{
return executor;
}
}
public static class LoadWebSocketTransport extends WebSocketTransport
{
private final Executor executor;
public LoadWebSocketTransport(BayeuxServerImpl bayeux, Executor executor)
{
super(bayeux);
this.executor = executor;
}
@Override
protected Executor newExecutor()
{
return executor;
}
}
}