package com.cloudhopper.mq.broker;
/*
* #%L
* ch-mq-remote
* %%
* Copyright (C) 2009 - 2012 Cloudhopper by Twitter
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import com.cloudhopper.commons.util.URL;
import com.cloudhopper.mq.queue.Queue;
import com.cloudhopper.mq.queue.QueueInvalidStateException;
import com.cloudhopper.mq.queue.QueueManager;
import com.cloudhopper.mq.queue.QueueManagerListener;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.DelayQueue;
import java.util.concurrent.Executors;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.RejectedExecutionHandler;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import javax.management.ObjectName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
*
* @author joelauer
*/
public class DistributedQueueManager implements QueueManagerListener, DistributedQueueStateListener, DistributedQueueManagerMBean {
private static final Logger logger = LoggerFactory.getLogger(DistributedQueueManager.class);
public enum Event {
REMOTE_QUEUE_AVAILABLE,
REMOTE_QUEUE_NOT_AVAILABLE,
LOCAL_CONSUMER_STARTED,
LOCAL_CONSUMER_STOPPED,
LOCAL_QUEUE_CREATED,
LOCAL_QUEUE_DESTROYED
};
protected final AtomicBoolean started;
// configuration for the distributed queue
protected final DistributedQueueConfiguration configuration;
// local queue manager
protected final QueueManager queueManager;
// remote queue states
protected final DistributedQueueState dqs;
// delayed queue containing events
protected final DelayQueue<DistributedQueueEvent> events;
// pool of threads to execute remote queue transfers
protected ExecutorService remotingExecutorService;
// scheduler for remote queue transfers
protected RemoteQueueTransferScheduler remotingScheduler;
// pool of threads to monitor remote brokers
protected ScheduledThreadPoolExecutor monitorExecutorService;
// array of remote broker monitors (almost never changes)
protected final ConcurrentHashMap<String,RemoteBrokerMonitor> remoteBrokerMonitors;
// map of queue names to processors
protected final ConcurrentHashMap<String,LocalToRemoteQueueProcessor> queueProcessors;
// thread that processes events
private EventProcessor eventProcessor;
// http client factory
private final AsyncHttpClientFactory httpFactory;
// remoting take timeouts
private final AtomicLong remotingTakeTimeouts;
// remoting take nulls
private final AtomicLong remotingTakeNulls;
public DistributedQueueManager(DistributedQueueConfiguration configuration, QueueManager queueManager) {
this.configuration = configuration;
this.queueManager = queueManager;
this.dqs = new DistributedQueueState(configuration);
this.events = new DelayQueue<DistributedQueueEvent>();
this.remoteBrokerMonitors = new ConcurrentHashMap<String,RemoteBrokerMonitor>();
this.queueProcessors = new ConcurrentHashMap<String,LocalToRemoteQueueProcessor>();
this.httpFactory = new AsyncHttpClientFactory((int)configuration.getConnectionTimeout(),
(int)configuration.getConnectionTimeout());
this.remotingTakeTimeouts = new AtomicLong(0l);
this.remotingTakeNulls = new AtomicLong(0l);
this.started = new AtomicBoolean(false);
registerMBean();
}
protected void registerMBean() {
if (queueManager.getConfiguration().isJmxEnabled()) {
// register the this queue manager as an mbean
try {
ObjectName name = new ObjectName(queueManager.getConfiguration().getJmxDomain() + ":name=DistributedQueueManager");
queueManager.getConfiguration().getMBeanServer().registerMBean(this, name);
} catch (Exception e) {
// log the error, but don't throw an exception for this datasource
logger.warn("Error while attempting to register DistributedQueueManager as an MBean: {}", e.toString());
}
}
}
public DistributedQueueConfiguration getConfiguration() {
return this.configuration;
}
public QueueManager getQueueManager() {
return this.queueManager;
}
public DistributedQueueState getDistributedQueueState() {
return this.dqs;
}
public DelayQueue getEventQueue() {
return this.events;
}
public ConcurrentHashMap<String,RemoteBrokerMonitor> getRemoteBrokerMonitors() {
return this.remoteBrokerMonitors;
}
public ConcurrentHashMap<String,LocalToRemoteQueueProcessor> getQueueProcessors() {
return this.queueProcessors;
}
public Integer getAreaId() {
return configuration.getAreaId();
}
public String getGroupName() {
return configuration.getGroupName();
}
public String[] getRemoteBrokerMonitorNames() {
String[] monitorNames = this.remoteBrokerMonitors.keySet().toArray(new String[0]);
return monitorNames;
}
public int getEventQueueSize() {
return this.events.size();
}
public int getLocalToRemoteQueueProcessorSize() {
return this.queueProcessors.size();
}
public String[] getLocalToRemoteQueueProcessorNames() {
String[] queueNames = this.queueProcessors.keySet().toArray(new String[0]);
return queueNames;
}
public boolean isStarted() {
return this.started.get();
}
public boolean isStopped() {
return !this.started.get();
}
public boolean isEventProcessorAlive() {
return eventProcessor.isAlive();
}
public boolean isMonitorThreadPoolAlive() {
return !(monitorExecutorService.isTerminated() || monitorExecutorService.isShutdown());
}
public boolean isTransferThreadPoolAlive() {
return !(remotingExecutorService.isTerminated() || remotingExecutorService.isShutdown());
}
public int getMonitorThreadsSize() {
return ((ScheduledThreadPoolExecutor)monitorExecutorService).getPoolSize();
}
public int getTransferThreadsSize() {
return ((ThreadPoolExecutor)remotingExecutorService).getPoolSize();
}
public long getRemotingTakeTimeouts() {
return remotingTakeTimeouts.get();
}
public long incrementRemotingTakeTimeouts() {
return remotingTakeTimeouts.incrementAndGet();
}
public long getRemotingTakeNulls() {
return remotingTakeNulls.get();
}
public long incrementRemotingTakeNulls() {
return remotingTakeNulls.incrementAndGet();
}
synchronized public void start() {
if (isStarted()) {
logger.warn("Ignoring duplicate start() request");
return;
}
// add this to listeners
this.queueManager.addListener(this);
this.dqs.addListener(this);
// start monitoring of remote brokers - create pool first
logger.debug("Starting monitoring ExecutorService with {} threads.", configuration.getMonitorThreads());
monitorExecutorService = new ScheduledThreadPoolExecutor(configuration.getMonitorThreads(), new ThreadFactory() {
AtomicInteger sequence = new AtomicInteger();
public Thread newThread(Runnable r) {
Thread t = new Thread(r);
t.setName("CHMQ-RemoteBrokerMonitor-" + sequence.getAndIncrement());
t.setDaemon(true);
return t;
}
});
monitorExecutorService.prestartAllCoreThreads();
// create array of broker monitors
remoteBrokerMonitors.clear();
// create the remoting scheduler
this.remotingScheduler = new FairRemoteQueueTransferScheduler(configuration.getMaxConcurrentRequests(),
queueManager);
// start remoting executor service
logger.debug("Starting remoting ExecutorService with {} request and {} response threads.", configuration.getRemotingRequestThreads(), configuration.getRemotingResponseThreads());
ThreadPoolExecutor tpRemotingExecutorService = new ThreadPoolExecutor(configuration.getRemotingRequestThreads(), configuration.getRemotingRequestThreads(), 5000, TimeUnit.MILLISECONDS, remotingScheduler, new ThreadFactory() {
AtomicInteger sequence = new AtomicInteger();
public Thread newThread(Runnable r) {
Thread t = new Thread(r);
t.setName("CHMQ-RemoteQueueTransfer-" + sequence.getAndIncrement());
t.setDaemon(true);
return t;
}
}, new RejectedExecutionHandler() {
public void rejectedExecution(Runnable r,
ThreadPoolExecutor executor) {
logger.warn("Rejected execution of {} by executor with {} current threads in the pool", r.getClass().getName(), executor.getPoolSize());
}
});
tpRemotingExecutorService.prestartAllCoreThreads();
remotingExecutorService = tpRemotingExecutorService;
eventProcessor = new EventProcessor();
eventProcessor.start();
this.started.set(true);
// create all remote brokers based on configuration
for (URL url : configuration.getRemoteBrokers()) {
RemoteBrokerInfo bi = new RemoteBrokerInfo(url.toString());
dqs.addRemoteBroker(bi);
}
}
synchronized public void stop() {
if (isStopped()) {
logger.warn("Ignoring duplicate stop() request");
return;
}
// kill all local processors first...
killAllLocalToRemoteQueueProcessors();
// remove this from listeners
this.queueManager.removeListener(this);
this.dqs.removeListener(this);
// stop monitoring remote brokers
monitorExecutorService.shutdownNow();
remoteBrokerMonitors.clear();
// stop processing events
eventProcessor.interrupt();
// stop remoting
remotingExecutorService.shutdownNow();
remotingScheduler = null;
// clean up the distributed state
dqs.clear();
this.started.set(false);
}
private class EventProcessor extends Thread {
public EventProcessor() {
super();
this.setDaemon(true);
this.setName("CHMQ-DistributedQueueManager-EventProcessor");
}
@Override
public void run() {
logger.info("Event processing thread started");
// continue running while we haven't been interrupted
while (!Thread.interrupted()) {
try {
// get the next event (can be interrupted
DistributedQueueEvent event = events.take();
// process event and make a decision
logger.info("Processing event " + event);
if (event.getEvent() == Event.REMOTE_QUEUE_AVAILABLE) {
startLocalToRemoteQueueProcessor(event.getQueueName());
} else if (event.getEvent() == Event.REMOTE_QUEUE_NOT_AVAILABLE) {
killLocalToRemoteQueueProcessor(event.getQueueName());
} else if (event.getEvent() == Event.LOCAL_CONSUMER_STARTED) {
killLocalToRemoteQueueProcessor(event.getQueueName());
} else if (event.getEvent() == Event.LOCAL_CONSUMER_STOPPED) {
startLocalToRemoteQueueProcessor(event.getQueueName());
} else if (event.getEvent() == Event.LOCAL_QUEUE_CREATED) {
startLocalToRemoteQueueProcessor(event.getQueueName());
} else if (event.getEvent() == Event.LOCAL_QUEUE_DESTROYED) {
killLocalToRemoteQueueProcessor(event.getQueueName());
}
} catch (InterruptedException e) {
// correct behavior, exit
break;
}
}
logger.info("Event processing thread ending");
}
}
synchronized private void startLocalToRemoteQueueProcessor(String queueName) {
// get the local queue
Queue localQueue = null;
try {
localQueue = queueManager.getQueue(queueName);
} catch (QueueInvalidStateException e) {
logger.error("Unable to complete startLocalToRemoteQueueProcessor since QueueManager is in an invalid state", e);
return;
}
// if no local queue, silently ignore this event?
if (localQueue == null) {
logger.warn("Unable to complete startLocalToRemoteQueueProcessor since local queue [" + queueName + "] does not exist - perhaps a delayed event?");
return;
}
// if there is a local consumer for this queue, we need to ignore this
if (localQueue.getConsumerCount() > 0) {
logger.warn("Ignoring request to startLocalToRemoteQueueProcessor for queue [" + queueName + "]: there are currently [" + localQueue.getConsumerCount() + "] local consumers");
return;
}
// make sure there is a remote queue
RemoteQueueInfo remoteQueue = dqs.getRemoteQueue(queueName);
if (remoteQueue == null) {
logger.error("Ignoring request to startLocalToRemoteQueueProcessor for queue [" + queueName + "]: there is no remote queue for it in DistributedQueueState");
return;
}
if (remoteQueue.isNotAvailable()) {
logger.error("Ignoring request to startLocalToRemoteQueueProcessor for queue [" + queueName + "]: the remote queue is not available (perhaps it flapped?)");
return;
}
// is there already a processor?
LocalToRemoteQueueProcessor queueProcessor = this.queueProcessors.get(queueName);
if (queueProcessor != null) {
// is it still alive?
if (!queueProcessor.isKilled() && queueProcessor.isAlive()) {
logger.error("Ignoring request to startLocalToRemoteQueueProcessor for queue [" + queueName + "]: the processor already exists and is still alive");
return;
}
}
logger.debug("Creating and starting LocalToRemoteQueueProcessor for queue [" + localQueue.getName() + "]");
try {
// create a new processor
queueProcessor = new LocalToRemoteQueueProcessor(this, localQueue, remoteQueue, httpFactory);
// try to add it to our map
this.queueProcessors.put(queueName, queueProcessor);
// start it up!
queueProcessor.start(); // This does nothing now
remotingScheduler.addLocalToRemoteQueueProcessor(localQueue, queueProcessor); // Adds it to the scheduler
} catch (Exception e) {
logger.error("Error while starting and adding new LocalToRemoteQueueProcessor.", e);
}
}
synchronized private void killLocalToRemoteQueueProcessor(String queueName) {
// remove the processor from our internal map
LocalToRemoteQueueProcessor queueProcessor = queueProcessors.remove(queueName);
if (queueProcessor == null) {
logger.warn("Ignoring request to killLocalToRemoteQueueProcessor for queue [" + queueName + "]: no processor currently exists");
return;
}
logger.debug("Removing and destroying LocalToRemoteQueueProcessor for queue [" + queueName + "]");
try {
// kill it
queueProcessor.kill(); // This does nothing now
remotingScheduler.removeLocalToRemoteQueueProcessor(queueProcessor.getLocalQueue()); // Removes it from the scheduler
} catch (Exception e) {
logger.error("Error while stopping and removing new LocalToRemoteQueueProcessor.", e);
}
}
synchronized private void killAllLocalToRemoteQueueProcessors() {
// kill all of 'em
for (String queueName : queueProcessors.keySet()) {
this.killLocalToRemoteQueueProcessor(queueName);
}
}
//
// local queues
//
public void notifyQueueCreated(Queue queue) {
logger.debug("Received notification that queue [" + queue.getName() + "] created, going to wait " + configuration.getLocalQueueCreateDelay() + " ms before checking if a remote processor should start");
this.events.put(new DistributedQueueEvent(queue.getName(), Event.LOCAL_QUEUE_CREATED, configuration.getLocalQueueCreateDelay()));
}
public void notifyQueueDestroyed(Queue queue) {
logger.debug("Received notification that queue [" + queue.getName() + "] destroyed");
this.events.put(new DistributedQueueEvent(queue.getName(), Event.LOCAL_QUEUE_DESTROYED));
}
public void notifyAtLeastOneQueueConsumerStarted(Queue queue) {
logger.debug("Received notification that queue [" + queue.getName() + "] now has at least one local consumer");
// this needs to be processed asap
this.events.put(new DistributedQueueEvent(queue.getName(), Event.LOCAL_CONSUMER_STARTED));
}
public void notifyAllQueueConsumersStopped(Queue queue) {
logger.debug("Received notification that queue [" + queue.getName() + "] now has no local consumers");
// in order to prevent "flapping" and unnecessarily forwarding items
// to remote queues when a local consumer may start up again...
// we'll delay processing this event for a period of time
logger.debug("To prevent possible flapping, will delay processing event that all local consumers stopped on queue [" + queue.getName() + "] for " + configuration.getLocalConsumerFlappingDelay() + " ms");
this.events.put(new DistributedQueueEvent(queue.getName(), Event.LOCAL_CONSUMER_STOPPED, configuration.getLocalConsumerFlappingDelay()));
}
//
// remote brokers
//
public void notifyRemoteBrokerAdded(RemoteBrokerInfo bi) {
logger.debug("Received notification that remote broker [" + bi.getUrl() + "] was added");
if (logger.isTraceEnabled()) {
logger.trace(dqs.toDebugString());
}
// need to add a monitor thread if it doesn't exist
if (remoteBrokerMonitors.get(bi.getUrl()) == null) { //the monitor isn't present
// create a monitor for it
RemoteBrokerMonitor monitor = new RemoteBrokerMonitor(configuration, dqs, bi, httpFactory);
// add this monitor to our array
remoteBrokerMonitors.put(bi.getUrl(), monitor);
// schedule it to run every interval
logger.debug("Scheduling monitor for RemoteBroker [" + bi.getUrl() + "] to run every [" + configuration.getMonitorInterval() + " ms]");
monitorExecutorService.scheduleWithFixedDelay(monitor, 1000, configuration.getMonitorInterval(), TimeUnit.MILLISECONDS);
}
}
public void notifyRemoteBrokerRemoved(RemoteBrokerInfo bi) {
logger.debug("Received notification that remote broker [" + bi.getUrl() + "] was removed");
if (logger.isTraceEnabled()) {
logger.trace(dqs.toDebugString());
}
RemoteBrokerMonitor monitor = remoteBrokerMonitors.get(bi.getUrl());
if (monitor != null) {
logger.debug("Removing monitor for RemoteBroker [" + bi.getUrl() + "]");
monitorExecutorService.remove(monitor);
}
}
public void notifyRemoteBrokerStateChanged(RemoteBrokerInfo bi, int state) {
logger.debug("Received notification that remote broker [" + bi.getUrl() + "] had a state change");
if (logger.isTraceEnabled()) {
logger.trace(dqs.toDebugString());
}
// don't really need to do anything with this event
// any remote queues associated with this remote broker would have
// had the specific event for the remote queue triggered below
}
//
// remote queues
//
public void notifyRemoteQueueAdded(RemoteQueueInfo qi) {
logger.debug("Received notification that remote queue [" + qi.getName() + "] was added");
if (logger.isTraceEnabled()) {
logger.trace(dqs.toDebugString());
}
// this event is triggered prior to its state being changed below
// we don't really need to process this event...
}
public void notifyRemoteQueueRemoved(RemoteQueueInfo qi) {
logger.debug("Received notification that remote queue [" + qi.getName() + "] was removed");
if (logger.isTraceEnabled()) {
logger.trace(dqs.toDebugString());
}
// hmmm... safe to just make sure we remove anything?
}
public void notifyRemoteQueueStateChanged(RemoteQueueInfo qi, int state) {
logger.debug("Received notification that remote queue [" + qi.getName() + "] has a new state [" + RemoteQueueInfo.STATES[state] + "]");
if (logger.isTraceEnabled()) {
logger.trace(dqs.toDebugString());
}
// most important event, remote queue available/not available
if (qi.isAvailable()) {
this.events.put(new DistributedQueueEvent(qi.getName(), Event.REMOTE_QUEUE_AVAILABLE));
} else if (qi.isNotAvailable()) {
this.events.put(new DistributedQueueEvent(qi.getName(), Event.REMOTE_QUEUE_NOT_AVAILABLE));
}
}
public void notifyRemoteQueueAttributesChanged(RemoteQueueInfo qi) {
logger.debug("Received notification that remote queue [" + qi.getName() + "] had attributes change, but not its overall state");
if (logger.isTraceEnabled()) {
logger.trace(dqs.toDebugString());
}
// just for informational purposes
}
}