/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.activemq.usecases;
import java.net.URI;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Random;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import javax.jms.Connection;
import javax.jms.DeliveryMode;
import javax.jms.Destination;
import javax.jms.JMSException;
import javax.jms.Message;
import javax.jms.MessageConsumer;
import javax.jms.MessageListener;
import javax.jms.MessageProducer;
import javax.jms.Session;
import javax.jms.TextMessage;
import junit.framework.Test;
import org.apache.activemq.ActiveMQConnectionFactory;
import org.apache.activemq.JmsMultipleBrokersTestSupport;
import org.apache.activemq.advisory.AdvisorySupport;
import org.apache.activemq.broker.region.policy.DeadLetterStrategy;
import org.apache.activemq.broker.region.policy.PolicyEntry;
import org.apache.activemq.broker.region.policy.PolicyMap;
import org.apache.activemq.broker.region.policy.PriorityDispatchPolicy;
import org.apache.activemq.broker.region.policy.SharedDeadLetterStrategy;
import org.apache.activemq.command.ActiveMQQueue;
import org.apache.activemq.command.ActiveMQTempTopic;
import org.apache.activemq.command.ActiveMQTextMessage;
import org.apache.activemq.network.DemandForwardingBridgeSupport;
import org.apache.activemq.network.NetworkBridge;
import org.apache.activemq.network.NetworkConnector;
import org.apache.activemq.util.DefaultTestAppender;
import org.apache.log4j.Appender;
import org.apache.log4j.Level;
import org.apache.log4j.spi.LoggingEvent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @author <a href="http://www.christianposta.com/blog">Christian Posta</a>
*/
public class RequestReplyTempDestRemovalAdvisoryRaceTest extends JmsMultipleBrokersTestSupport {
private static final Logger LOG = LoggerFactory.getLogger(RequestReplyTempDestRemovalAdvisoryRaceTest.class);
private static final String BROKER_A = "BrokerA";
private static final String BROKER_B = "BrokerB";
private static final String BROKER_C = "BrokerC";
private static final int NUM_RESPONDENTS = 1;
private static final int NUM_SENDS = 1;
private static final int RANDOM_SLEEP_FOR_RESPONDENT_MS = 0;
private static final int RANDOM_SLEEP_FOR_SENDER_MS = 1;
private static final String QUEUE_NAME = "foo.queue";
private static String[] TEST_ITERATIONS = new String[]{QUEUE_NAME+"0", QUEUE_NAME+"1", QUEUE_NAME+"2", QUEUE_NAME+"3"};
final AtomicLong messageCount = new AtomicLong(0);
final AtomicLong respondentSendError = new AtomicLong(0);
final AtomicLong responseReceived = new AtomicLong(0);
final AtomicLong sendsWithNoConsumers = new AtomicLong(0);
final AtomicLong forwardFailures = new AtomicLong(0);
protected final AtomicBoolean shutdown = new AtomicBoolean(false);
HashSet<NetworkConnector> networkConnectors = new HashSet<NetworkConnector>();
HashSet<Connection> advisoryConsumerConnections = new HashSet<Connection>();
Appender slowDownAppender;
CountDownLatch consumerDemandExists;
protected boolean useDuplex = false;
public static Test suite() {
return suite(RequestReplyTempDestRemovalAdvisoryRaceTest.class);
}
/**
* Notes: to reliably reproduce use debugger... set a "thread" breakpoint at line 679 in DemandForwardingBridgeSupport,
* and only break on the "2nd" pass (broker C's bridge). Allow debugging to continue shortly after hitting
* the breakpoint, for this test we use a logging appender to implement the pause,
* it fails most of the time, hence the combos
*/
public void initCombos() {
addCombinationValues("QUEUE_NAME", TEST_ITERATIONS);
}
public void testTempDestRaceDuplex() throws Exception {
// duplex
useDuplex = true;
bridgeBrokers(BROKER_A, BROKER_B, false, 3);
bridgeBrokers(BROKER_B, BROKER_C, false, 3);
startAllBrokers();
waitForBridgeFormation(1);
HashSet<NetworkBridge> bridgesStart = new HashSet<NetworkBridge>();
for (NetworkConnector networkConnector : networkConnectors) {
bridgesStart.addAll(networkConnector.activeBridges());
}
LOG.info("Bridges start:" + bridgesStart);
slowDownAdvisoryDispatch();
noConsumerAdvisory();
forwardFailureAdvisory();
// set up respondents
ExecutorService respondentThreadPool = Executors.newFixedThreadPool(50);
BrokerItem brokerA = brokers.get(BROKER_A);
ActiveMQConnectionFactory brokerAFactory = new ActiveMQConnectionFactory(brokerA.broker.getTransportConnectorByScheme("tcp").getName()
+ "?jms.watchTopicAdvisories=false");
brokerAFactory.setAlwaysSyncSend(true);
for (int i = 0; i < NUM_RESPONDENTS; i++) {
respondentThreadPool.execute(new EchoRespondent(brokerAFactory));
}
// fire off sends
ExecutorService senderThreadPool = Executors.newCachedThreadPool();
BrokerItem brokerC = brokers.get(BROKER_C);
ActiveMQConnectionFactory brokerCFactory = new ActiveMQConnectionFactory(brokerC.broker.getTransportConnectorByScheme("tcp").getName()
+ "?jms.watchTopicAdvisories=false");
for (int i = 0; i < NUM_SENDS; i++) {
senderThreadPool.execute(new MessageSender(brokerCFactory));
}
senderThreadPool.shutdown();
senderThreadPool.awaitTermination(30, TimeUnit.SECONDS);
TimeUnit.SECONDS.sleep(15);
LOG.info("shutting down");
shutdown.compareAndSet(false, true);
HashSet<NetworkBridge> bridgesEnd = new HashSet<NetworkBridge>();
for (NetworkConnector networkConnector : networkConnectors) {
bridgesEnd.addAll(networkConnector.activeBridges());
}
LOG.info("Bridges end:" + bridgesEnd);
assertEquals("no new bridges created", bridgesStart, bridgesEnd);
// validate success or error of dlq
LOG.info("received: " + responseReceived.get() + ", respondent error: " + respondentSendError.get()
+ ", noConsumerCount: " + sendsWithNoConsumers.get()
+ ", forwardFailures: " + forwardFailures.get());
assertEquals("success or error", NUM_SENDS, respondentSendError.get() + forwardFailures.get()
+ responseReceived.get() + sendsWithNoConsumers.get());
}
private void forwardFailureAdvisory() throws JMSException {
for (BrokerItem item : brokers.values()) {
ActiveMQConnectionFactory brokerAFactory = new ActiveMQConnectionFactory(item.broker.getTransportConnectorByScheme("tcp").getName()
+ "?jms.watchTopicAdvisories=false");
Connection connection = brokerAFactory.createConnection();
connection.start();
connection.createSession(false, Session.AUTO_ACKNOWLEDGE).createConsumer(
AdvisorySupport.getNetworkBridgeForwardFailureAdvisoryTopic()).setMessageListener(new MessageListener() {
@Override
public void onMessage(Message message) {
forwardFailures.incrementAndGet();
}
});
}
}
private void noConsumerAdvisory() throws JMSException {
for (BrokerItem item : brokers.values()) {
ActiveMQConnectionFactory brokerAFactory = new ActiveMQConnectionFactory(item.broker.getTransportConnectorByScheme("tcp").getName()
+ "?jms.watchTopicAdvisories=false");
Connection connection = brokerAFactory.createConnection();
connection.start();
connection.createSession(false, Session.AUTO_ACKNOWLEDGE).createConsumer(
AdvisorySupport.getNoTopicConsumersAdvisoryTopic(new ActiveMQTempTopic(">"))).setMessageListener(new MessageListener() {
@Override
public void onMessage(Message message) {
sendsWithNoConsumers.incrementAndGet();
}
});
}
}
public void testTempDestRace() throws Exception {
// non duplex
bridgeBrokers(BROKER_A, BROKER_B, false, 3);
bridgeBrokers(BROKER_B, BROKER_A, false, 3);
bridgeBrokers(BROKER_B, BROKER_C, false, 3);
bridgeBrokers(BROKER_C, BROKER_B, false, 3);
startAllBrokers();
waitForBridgeFormation(1);
HashSet<NetworkBridge> bridgesStart = new HashSet<NetworkBridge>();
for (NetworkConnector networkConnector : networkConnectors) {
bridgesStart.addAll(networkConnector.activeBridges());
}
slowDownAdvisoryDispatch();
noConsumerAdvisory();
forwardFailureAdvisory();
// set up respondents
ExecutorService respondentThreadPool = Executors.newFixedThreadPool(50);
BrokerItem brokerA = brokers.get(BROKER_A);
ActiveMQConnectionFactory brokerAFactory = new ActiveMQConnectionFactory(brokerA.broker.getTransportConnectorByScheme("tcp").getName()
+ "?jms.watchTopicAdvisories=false");
brokerAFactory.setAlwaysSyncSend(true);
for (int i = 0; i < NUM_RESPONDENTS; i++) {
respondentThreadPool.execute(new EchoRespondent(brokerAFactory));
}
// fire off sends
ExecutorService senderThreadPool = Executors.newCachedThreadPool();
BrokerItem brokerC = brokers.get(BROKER_C);
ActiveMQConnectionFactory brokerCFactory = new ActiveMQConnectionFactory(brokerC.broker.getTransportConnectorByScheme("tcp").getName()
+ "?jms.watchTopicAdvisories=false");
for (int i = 0; i < NUM_SENDS; i++) {
senderThreadPool.execute(new MessageSender(brokerCFactory));
}
senderThreadPool.shutdown();
senderThreadPool.awaitTermination(30, TimeUnit.SECONDS);
TimeUnit.SECONDS.sleep(10);
LOG.info("shutting down");
shutdown.compareAndSet(false, true);
HashSet<NetworkBridge> bridgesEnd = new HashSet<NetworkBridge>();
for (NetworkConnector networkConnector : networkConnectors) {
bridgesEnd.addAll(networkConnector.activeBridges());
}
assertEquals("no new bridges created", bridgesStart, bridgesEnd);
// validate success or error or dlq
LOG.info("received: " + responseReceived.get() + ", respondent error: " + respondentSendError.get()
+ ", noConsumerCount: " + sendsWithNoConsumers.get()
+ ", forwardFailures: " + forwardFailures.get());
assertEquals("success or error", NUM_SENDS, respondentSendError.get() + forwardFailures.get()
+ responseReceived.get() + sendsWithNoConsumers.get());
}
private void slowDownAdvisoryDispatch() throws Exception {
org.apache.log4j.Logger.getLogger(DemandForwardingBridgeSupport.class).setLevel(Level.DEBUG);
// instrument a logger to block the processing of a remove sub advisory
// simulate a slow thread
slowDownAppender = new DefaultTestAppender() {
@Override
public void doAppend(LoggingEvent loggingEvent) {
if (Level.DEBUG.equals(loggingEvent.getLevel())) {
String message = loggingEvent.getMessage().toString();
if (message.startsWith("BrokerB") && message.contains("remove local subscription")) {
// sleep for a bit
try {
consumerDemandExists.countDown();
System.err.println("Sleeping on receipt of remove info debug message: " + message);
TimeUnit.SECONDS.sleep(2);
} catch (Exception ignored) {
}
}
}
}
};
org.apache.log4j.Logger.getRootLogger().addAppender(slowDownAppender);
}
@Override
protected void setUp() throws Exception {
super.setUp();
responseReceived.set(0);
respondentSendError.set(0);
forwardFailures.set(0);
sendsWithNoConsumers.set(0);
networkConnectors.clear();
advisoryConsumerConnections.clear();
consumerDemandExists = new CountDownLatch(1);
createBroker(new URI("broker:(tcp://localhost:0)/" + BROKER_A + "?persistent=false&useJmx=false")).setDedicatedTaskRunner(false);
createBroker(new URI("broker:(tcp://localhost:0)/" + BROKER_B + "?persistent=false&useJmx=false")).setDedicatedTaskRunner(false);
createBroker(new URI("broker:(tcp://localhost:0)/" + BROKER_C + "?persistent=false&useJmx=false")).setDedicatedTaskRunner(false);
PolicyMap map = new PolicyMap();
PolicyEntry defaultEntry = new PolicyEntry();
defaultEntry.setSendAdvisoryIfNoConsumers(true);
DeadLetterStrategy deadletterStrategy = new SharedDeadLetterStrategy();
deadletterStrategy.setProcessNonPersistent(true);
defaultEntry.setDeadLetterStrategy(deadletterStrategy);
defaultEntry.setDispatchPolicy(new PriorityDispatchPolicy());
map.put(new ActiveMQTempTopic(">"), defaultEntry);
for (BrokerItem item : brokers.values()) {
item.broker.setDestinationPolicy(map);
}
}
@Override
protected void tearDown() throws Exception {
if (slowDownAppender != null) {
org.apache.log4j.Logger.getRootLogger().removeAppender(slowDownAppender);
}
for (Connection connection : advisoryConsumerConnections) {
connection.close();
}
super.tearDown();
}
protected NetworkConnector bridgeBrokers(String localBrokerName, String remoteBrokerName, boolean dynamicOnly, int networkTTL) throws Exception {
NetworkConnector connector = super.bridgeBrokers(localBrokerName, remoteBrokerName, dynamicOnly, networkTTL, true);
connector.setBridgeTempDestinations(true);
connector.setAdvisoryForFailedForward(true);
connector.setDuplex(useDuplex);
connector.setAlwaysSyncSend(true);
networkConnectors.add(connector);
return connector;
}
abstract class MessageClient {
protected Connection connection;
protected Session session;
protected MessageConsumer consumer;
protected MessageProducer producer;
protected Random random;
protected int timeToSleep;
// set up the connection and session
public MessageClient(ActiveMQConnectionFactory factory, int timeToSleep) throws Exception {
this.connection = factory.createConnection();
this.session = connection.createSession(false, Session.AUTO_ACKNOWLEDGE);
this.timeToSleep = timeToSleep;
this.random = new Random(System.currentTimeMillis());
preInit();
initProducer();
initConsumer();
this.connection.start();
}
protected void preInit() throws JMSException {
}
abstract protected void initProducer() throws JMSException;
abstract protected void initConsumer() throws JMSException;
}
class MessageSender extends MessageClient implements Runnable {
protected Destination tempDest;
public MessageSender(ActiveMQConnectionFactory factory) throws Exception {
super(factory, RANDOM_SLEEP_FOR_SENDER_MS);
}
@Override
public void run() {
// create a message
try {
TextMessage message = session.createTextMessage("request: message #" + messageCount.getAndIncrement());
message.setJMSReplyTo(tempDest);
producer.send(message);
LOG.info("SENDER: Message [" + message.getText() + "] has been sent.");
Message incomingMessage = consumer.receive(timeToSleep);
if (incomingMessage instanceof TextMessage) {
try {
LOG.info("SENDER: Got a response from echo service!" + ((TextMessage) incomingMessage).getText());
responseReceived.incrementAndGet();
} catch (JMSException e) {
LOG.error("SENDER: might want to see why i'm getting non-text messages..." + incomingMessage, e);
}
} else {
LOG.info("SENDER: Did not get a response this time");
}
} catch (JMSException e) {
LOG.error("SENDER: Could not complete message sending properly: " + e.getMessage());
} finally {
try {
producer.close();
consumer.close();
session.close();
connection.close();
} catch (JMSException e) {
e.printStackTrace();
}
}
}
@Override
protected void preInit() throws JMSException {
this.tempDest = session.createTemporaryTopic();
}
@Override
protected void initProducer() throws JMSException {
this.producer = session.createProducer(new ActiveMQQueue(QUEUE_NAME));
}
@Override
protected void initConsumer() throws JMSException {
this.consumer = session.createConsumer(tempDest);
LOG.info("consumer for: " + tempDest + ", " + consumer);
}
}
class EchoRespondent extends MessageClient implements Runnable {
public EchoRespondent(ActiveMQConnectionFactory factory) throws Exception {
super(factory, RANDOM_SLEEP_FOR_RESPONDENT_MS);
}
@Override
public void run() {
try {
LOG.info("RESPONDENT LISTENING");
while (!shutdown.get()) {
Message incomingMessage = consumer.receive(1000);
if (incomingMessage instanceof TextMessage) {
ActiveMQTextMessage textMessage = (ActiveMQTextMessage) incomingMessage;
try {
LOG.info("RESPONDENT: Received a message: [" + textMessage.getText() + "]" + Arrays.asList(textMessage.getBrokerPath()));
Message message = session.createTextMessage("reply: " + textMessage.getText());
Destination replyTo = incomingMessage.getJMSReplyTo();
TimeUnit.MILLISECONDS.sleep(timeToSleep);
consumerDemandExists.await(5, TimeUnit.SECONDS);
try {
producer.send(replyTo, message);
LOG.info("RESPONDENT: sent reply:" + message.getJMSMessageID() + " back to: " + replyTo);
} catch (JMSException e) {
LOG.error("RESPONDENT: could not send reply message: " + e.getLocalizedMessage(), e);
respondentSendError.incrementAndGet();
}
} catch (JMSException e) {
LOG.error("RESPONDENT: could not create the reply message: " + e.getLocalizedMessage(), e);
} catch (InterruptedException e) {
LOG.info("RESPONDENT could not generate a random number");
}
}
}
} catch (JMSException e) {
LOG.info("RESPONDENT: Could not set the message listener on the respondent");
}
}
@Override
protected void initProducer() throws JMSException {
this.producer = session.createProducer(null);
// so that we can get an advisory on sending with no consumers
this.producer.setDeliveryMode(DeliveryMode.NON_PERSISTENT);
}
@Override
protected void initConsumer() throws JMSException {
this.consumer = session.createConsumer(new ActiveMQQueue(QUEUE_NAME));
}
}
}