/**
* Copyright 2012 Comcast Corporation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/package com.comcast.cqs.test.unit;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.Vector;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import org.junit.Before;
import org.junit.Test;
import com.amazonaws.services.sqs.model.ChangeMessageVisibilityRequest;
import com.amazonaws.services.sqs.model.CreateQueueRequest;
import com.amazonaws.services.sqs.model.DeleteMessageRequest;
import com.amazonaws.services.sqs.model.DeleteQueueRequest;
import com.amazonaws.services.sqs.model.Message;
import com.amazonaws.services.sqs.model.ReceiveMessageRequest;
import com.amazonaws.services.sqs.model.ReceiveMessageResult;
import com.amazonaws.services.sqs.model.SendMessageRequest;
import com.amazonaws.services.sqs.model.SendMessageResult;
import com.comcast.cmb.common.controller.CMBControllerServlet;
import com.comcast.cmb.test.tools.CMBAWSBaseTest;
public class CQSScaleQueuesTest extends CMBAWSBaseTest {
private Random randomGenerator = new Random();
private final static String QUEUE_PREFIX = "TSTQ_";
private final static AtomicLong[] responseTimeDistribution100MS = new AtomicLong[100];
private final static AtomicLong[] responseTimeDistribution10MS = new AtomicLong[10];
private static String accessKey = null;
private static String accessSecret = null;
private Vector<String> report = new Vector<String>();
private static boolean stopHealthCheck = false;
private static int messageLength = 0;
private static final String ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
private static Random rand = new Random();
private static String queueName = null;
private static int numBatchReceive = 1;
private static boolean zipped = false;
private static String messageFile = null;
private static String messageFromFile = null;
//private static boolean restReceive = false;
private static int vto = 0;
private static int delaySeconds = 0;
private static AtomicLong totalMessagesFound = new AtomicLong(0);
private static AtomicLong totalMessagesSent = new AtomicLong(0);
private static AtomicLong totalMessageSendTime = new AtomicLong(0);
private static AtomicLong totalMessageReceiveTime = new AtomicLong(0);
private static AtomicLong totalMessageDeleteTime = new AtomicLong(0);
private static AtomicLong totalNumReceiveCalls = new AtomicLong(0);
private static boolean deleteQueues = true;
private static HashMap<String, String> attributeParams = new HashMap<String, String>();
public static void main(String [ ] args) throws Exception {
System.out.println("CQSScaleQueuesTest V" + CMBControllerServlet.VERSION);
long numQueuesPerThread = 10;
long numMessagesPerQueue = 10;
int numThreads = 10;
int numPartitions = 100;
int numShards = 1;
for (String arg : args) {
if (arg.startsWith("-nq")) {
numQueuesPerThread = Integer.parseInt(arg.substring(4));
} else if (arg.startsWith("-nm")) {
numMessagesPerQueue = Integer.parseInt(arg.substring(4));
} else if (arg.startsWith("-nt")) {
numThreads = Integer.parseInt(arg.substring(4));
} else if (arg.startsWith("-np")) {
numPartitions = Integer.parseInt(arg.substring(4));
} else if (arg.startsWith("-ns")) {
numShards = Integer.parseInt(arg.substring(4));
} else if (arg.startsWith("-ml")) {
messageLength = Integer.parseInt(arg.substring(4));
} else if (arg.startsWith("-qn")) {
queueName = arg.substring(4);
} else if (arg.startsWith("-ak")) {
accessKey = arg.substring(4);
} else if (arg.startsWith("-as")) {
accessSecret = arg.substring(4);
} else if (arg.startsWith("-dq")) {
deleteQueues = Boolean.parseBoolean(arg.substring(4));
} else if (arg.startsWith("-br")) {
numBatchReceive = Integer.parseInt(arg.substring(4));
} else if (arg.startsWith("-vt")) {
vto = Integer.parseInt(arg.substring(4));
} else if (arg.startsWith("-ds")) {
delaySeconds = Integer.parseInt(arg.substring(4));
} else if (arg.startsWith("-zp")) {
zipped = Boolean.parseBoolean(arg.substring(4));
//} else if (arg.startsWith("-rr")) {
// restReceive = Boolean.parseBoolean(arg.substring(4));
} else if (arg.startsWith("-mf")) {
messageFile = arg.substring(4);
StringBuffer sb = new StringBuffer("");
BufferedReader br = new BufferedReader(new FileReader(messageFile));
String line;
while ((line = br.readLine()) != null) {
sb.append(line);
}
br.close();
messageFromFile = sb.toString();
} else {
System.out.println("Usage: CQSScaleQueuesTest -Dcmb.log4j.propertyFile=config/log4j.properties -Dcmb.propertyFile=config/cmb.properties -nq=<number_queues_per_thread> -nm=<number_messages_per_queue> -nt=<number_threads> -np=<number_partitions> -ns=<number_shards> -ml=<message_length> -dq=<delete_queues_at_end_true_or_false> -qn=<queue_name_for_single_queue_tests> -ak=<access_key> -as=<access_secret> -br=<num_batch_receive> -zp=<zipped_true_or_false> -mf=<path_to_message_file> -vto=<visibility_timeout_seconds> -ds=<delay_seconds>");
System.out.println("Example: java CQSScaleQueuesTest -Dcmb.log4j.propertyFile=config/log4j.properties -Dcmb.propertyFile=config/cmb.properties -nq=10 -nm=10 -nt=10 -np=100 -ns=1 -dq=true -br=1");
System.exit(1);
}
}
System.out.println("Params for this test run:");
System.out.println("Number of queues per thread: " + numQueuesPerThread);
System.out.println("Number of messages per queue: " + numMessagesPerQueue);
System.out.println("Message length: " + messageLength);
System.out.println("Number of threads: " + numThreads);
System.out.println("Number of partitions: " + numPartitions);
System.out.println("Number of shards: " + numShards);
System.out.println("Delete queues: " + deleteQueues);
System.out.println("Number of messages received in batch: " + numBatchReceive);
System.out.println("Zipped: " + zipped);
System.out.println("Message File: " + messageFile);
//System.out.println("REST Receive: " + restReceive);
System.out.println("VTO: " + vto);
if (messageFromFile != null) {
System.out.println("Message File Size: " + messageFromFile.length());
}
if (queueName != null) {
System.out.println("Queue name: " + queueName);
System.out.println("Ignoring -nq setting above!");
numQueuesPerThread = 1;
}
if (accessKey != null) {
System.out.println("Access Key: " + accessKey);
}
if (accessSecret != null) {
System.out.println("Access Secret: " + accessSecret);
}
CQSScaleQueuesTest cqsScaleTest = new CQSScaleQueuesTest();
cqsScaleTest.setup();
cqsScaleTest.CreateQueuesConcurrent(numQueuesPerThread, numMessagesPerQueue, numThreads, numPartitions, numShards);
}
@Before
public void setup() throws Exception {
super.setup();
for (int i=0; i<responseTimeDistribution100MS.length; i++) {
responseTimeDistribution100MS[i] = new AtomicLong(0);
}
for (int i=0; i<responseTimeDistribution10MS.length; i++) {
responseTimeDistribution10MS[i] = new AtomicLong(0);
}
}
private String generateRandomMessage(int length) {
StringBuilder sb = new StringBuilder(length);
for (int i=0; i<length; i++) {
sb.append(ALPHABET.charAt(rand.nextInt(ALPHABET.length())));
}
return sb.toString();
}
private void recordResponseTime(String api, long responseTimeMS) {
// ignore api for now
long rt100 = responseTimeMS/100;
if (rt100<responseTimeDistribution100MS.length) {
responseTimeDistribution100MS[(int)rt100].incrementAndGet();
} else {
logger.warn("event=RT_OFF_THE_CHART rt=" + responseTimeMS + " api=" + api);
}
long rt10 = responseTimeMS/10;
if (rt10<responseTimeDistribution10MS.length) {
responseTimeDistribution10MS[(int)rt10].incrementAndGet();
}
}
private void printResponseTimeDistribution() {
long callCount = 0;
/*logger.warn("RT100");
for (int i=0; i<responseTimeDistribution100MS.length; i++) {
logger.warn("RT=" + (i*100) + " CT=" + responseTimeDistribution100MS[i]);
callCount += responseTimeDistribution100MS[i].longValue();
}*/
logger.warn("RT10");
for (int i=0; i<responseTimeDistribution10MS.length; i++) {
logger.warn("RT=" + (i*10) + " CT=" + responseTimeDistribution10MS[i]);
}
logger.warn("CALL_COUNT=" + callCount);
logger.warn("PCT_100MS=" + 1.0*responseTimeDistribution100MS[0].longValue()/callCount);
logger.warn("PCT_10MS=" + 1.0*responseTimeDistribution10MS[0].longValue()/callCount);
}
private void launchPeriodicHealthCheck(long delayMillis) {
final long waitPeriod = delayMillis;
new Thread (new Runnable() {
public void run() {
while (!stopHealthCheck) {
long start = System.currentTimeMillis();
String result = httpGet(cqsServiceUrl + "?Action=HealthCheck");
long end = System.currentTimeMillis();
logger.warn("HEALTH_CHECK_RT=" + (end-start));
recordResponseTime(null, end-start);
try {
Thread.sleep(waitPeriod);
} catch (InterruptedException ex) {
ex.printStackTrace();
}
}
logger.warn("HEALTH_CHECK_STOPPED");
}
private String httpGet(String api) {
URL url;
HttpURLConnection conn;
BufferedReader rd;
String line;
String result = "";
try {
url = new URL(api);
conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("GET");
rd = new BufferedReader(new InputStreamReader(conn.getInputStream()));
while ((line = rd.readLine()) != null) {
result += line;
}
rd.close();
} catch (Exception ex) {
ex.printStackTrace();
}
return result;
}
}).start();
}
@Test
public void Create1Queues() {
CreateNQueues(1, true);
}
@Test
public void Create10Queues() {
CreateNQueues(10, true);
}
@Test
public void Create10Queues10Partitions() {
CreateNQueues(10, 100, 10, 1, true);
}
@Test
public void Create10Queues1Partition() {
CreateNQueues(10, 100, 1, 1, true);
}
@Test
public void Create100Queues() {
CreateNQueues(100, true);
}
//@Test
public void Create1000Queues() {
CreateNQueues(1000, true);
}
//@Test
public void Create10000Queues() {
CreateNQueues(10000, true);
}
//@Test
public void Create100000Queues() {
CreateNQueues(100000, true);
}
@Test
public void CreateQueuesConcurrent() {
//queueName = "myqueue";
//messageLength = 100;
CreateQueuesConcurrent(2, 100, 20, 100, 1);
}
private void CreateQueuesConcurrent(long numQueuesPerThread, long numMessagesPerQueue, int numThreads, int numPartitions, int numShards) {
launchPeriodicHealthCheck(5000);
ScheduledThreadPoolExecutor ep = new ScheduledThreadPoolExecutor(numThreads + 10);
final long nqpt = numQueuesPerThread;
final long nmpq = numMessagesPerQueue;
final int np = numPartitions;
final int ns = numShards;
long start = System.currentTimeMillis();
for (int i=0; i<numThreads; i++) {
ep.submit((new Runnable() { public void run() { CreateNQueues(nqpt, nmpq, np, ns, false); }}));
}
logger.warn("ALL TEST LAUNCHED");
try {
ep.shutdown();
ep.awaitTermination(60, TimeUnit.MINUTES);
//Thread.sleep(5000);
} catch (InterruptedException ex) {
logger.error("fail", ex);
fail(ex.getMessage());
}
logger.warn("ALL TEST FINISHED");
long end = System.currentTimeMillis();
stopHealthCheck = true;
int c = 0;
for (String message : report) {
logger.warn(c + ": " + message);
c++;
}
printResponseTimeDistribution();
long apisPerSec = (numThreads*numQueuesPerThread*numMessagesPerQueue*3)/((end-start)/1000);
logger.warn("APIS_PER_SEC=" + apisPerSec);
logger.warn("TOTAL_MESSAGES_SENT=" + totalMessagesSent.longValue());
logger.warn("TOTAL_MESSAGES_FOUND=" + totalMessagesFound.longValue());
logger.warn("AVG_MSG_SEND_MS=" + totalMessageSendTime.longValue()/totalMessagesSent.longValue());
logger.warn("AVG_MSG_RECEIVE_MS=" + totalMessageReceiveTime.longValue()/totalNumReceiveCalls.longValue());
logger.warn("AVG_MSG_DELTE_MS=" + totalMessageDeleteTime.longValue()/totalMessagesFound.longValue());
logger.warn("TOTAL_DURATION_SEC=" + ((end-start)/1000));
}
private String httpGet(String urlString) {
URL url;
HttpURLConnection conn;
BufferedReader br;
String line;
String doc = "";
try {
url = new URL(urlString);
conn = (HttpURLConnection)url.openConnection();
conn.setRequestMethod("GET");
br = new BufferedReader(new InputStreamReader(conn.getInputStream()));
while ((line = br.readLine()) != null) {
doc += line;
}
br.close();
} catch (Exception ex) {
logger.error("event=http_get url=" + urlString, ex);
}
return doc;
}
/*private List<String> receiveMessageREST(String queueUrl) throws Exception {
List<String> messageBodies = new ArrayList<String>();
try {
long start = System.currentTimeMillis();
//logger.info("event=REC_START");
String receiveMessageUrl = queueUrl + "/?Action=ReceiveMessage&MaxNumberOfMessages="+numBatchReceive+"&AWSAccessKeyId=" + accessKey1;
String content = httpGet(receiveMessageUrl);
Element root = XmlUtil.buildDoc(content);
List<Element> messageElements = XmlUtil.getChildNodes(root, "ReceiveMessageResult");
//messageElements = XmlUtil.getChildNodes(root, "Message");
//logger.info("event=REC_END");
long end = System.currentTimeMillis();
recordResponseTime(null, end-start);
totalMessageReceiveTime.addAndGet(end-start);
totalNumReceiveCalls.incrementAndGet();
if (end-start>=500) {
logger.warn("RECEIVE_RT=" + (end-start));
}
for (Element element : messageElements) {
List<Element> l = XmlUtil.getChildNodes(element, "Message");
for (Element m : l) {
String messageBody = XmlUtil.getCurrentLevelTextValue(m, "Body");
String receiptHandle = XmlUtil.getCurrentLevelTextValue(m, "ReceiptHandle");
messageBodies.add(messageBody);
DeleteMessageRequest deleteMessageRequest = new DeleteMessageRequest();
deleteMessageRequest.setQueueUrl(queueUrl);
deleteMessageRequest.setReceiptHandle(receiptHandle);
start = System.currentTimeMillis();
//logger.info("event=DEL_START");
cqs1.deleteMessage(deleteMessageRequest);
//logger.info("event=DEL_END");
end = System.currentTimeMillis();
recordResponseTime(null, end-start);
totalMessageDeleteTime.addAndGet(end-start);
if (end-start>=500) {
logger.warn("DELETE_RT=" + (end-start));
}
}
}
} catch (Exception ex) {
logger.error("event=receive_error", ex);
}
return messageBodies;
}*/
/**
*
* @param queueUrl
* @return list of messages, empty list signals vto change, null signals empty read
*/
private List<String> receiveMessage(String queueUrl) {
List<String> messageBodies = new ArrayList<String>();
ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest();
receiveMessageRequest.setQueueUrl(queueUrl);
receiveMessageRequest.setMaxNumberOfMessages(numBatchReceive);
// use this to test with long poll
//receiveMessageRequest.setWaitTimeSeconds(1);
long start = System.currentTimeMillis();
//logger.info("event=REC_START");
ReceiveMessageResult receiveMessageResult = cqs1.receiveMessage(receiveMessageRequest);
//logger.info("event=REC_END");
long end = System.currentTimeMillis();
if (receiveMessageResult.getMessages().size() > 0) {
recordResponseTime(null, end-start);
totalMessageReceiveTime.addAndGet(end-start);
totalNumReceiveCalls.incrementAndGet();
} else {
return null;
}
if (end-start>=500) {
logger.warn("RECEIVE_RT=" + (end-start));
}
for (Message msg : receiveMessageResult.getMessages()) {
int receiveCount = 1;
if (vto > 0) {
receiveCount = Integer.parseInt(msg.getAttributes().get("ApproximateReceiveCount"));
}
if (vto > 0 && receiveCount == 1) {
logger.info("event=change_vto vto=4 receipt_handle=" + msg.getReceiptHandle());
ChangeMessageVisibilityRequest changeMessageVisibilityRequest = new ChangeMessageVisibilityRequest();
changeMessageVisibilityRequest.setQueueUrl(queueUrl);
changeMessageVisibilityRequest.setReceiptHandle(msg.getReceiptHandle());
changeMessageVisibilityRequest.setVisibilityTimeout(4);
cqs1.changeMessageVisibility(changeMessageVisibilityRequest);
} else {
logger.info("event=deleting_message receipt_handle=" + msg.getReceiptHandle());
messageBodies.add(msg.getBody());
DeleteMessageRequest deleteMessageRequest = new DeleteMessageRequest();
deleteMessageRequest.setQueueUrl(queueUrl);
deleteMessageRequest.setReceiptHandle(msg.getReceiptHandle());
start = System.currentTimeMillis();
//logger.info("event=DEL_START");
cqs1.deleteMessage(deleteMessageRequest);
//logger.info("event=DEL_END");
end = System.currentTimeMillis();
recordResponseTime(null, end-start);
totalMessageDeleteTime.addAndGet(end-start);
if (end-start>=500) {
logger.warn("DELETE_RT=" + (end-start));
}
}
}
return messageBodies;
}
private void CreateNQueues(long numQueues, boolean logReport) {
CreateNQueues(numQueues, 100, 100, 1, logReport);
}
private void CreateNQueues(long numQueues, long numMessages, int numPartitions, int numShards, boolean logReport) {
long testStart = System.currentTimeMillis();
Set<String> queueUrls = new HashSet<String>();
Map<String, String> messageMap = new HashMap<String, String>();
if (numPartitions != 100) {
attributeParams.put("NumberOfPartitions", numPartitions + "");
}
if (numShards != 1) {
attributeParams.put("NumberOfShards", numShards + "");
}
if (zipped) {
attributeParams.put("IsCompressed", "true");
} else {
attributeParams.put("IsCompressed", "false");
}
long counter = 0;
long createFailures = 0;
long totalTime = 0;
try {
for (int i=0; i<numQueues; i++) {
try {
String name = null;
if (queueName != null) {
name = queueName;
} else {
name = QUEUE_PREFIX + randomGenerator.nextLong();
}
CreateQueueRequest createQueueRequest = new CreateQueueRequest(name);
createQueueRequest.setAttributes(attributeParams);
long start = System.currentTimeMillis();
String queueUrl = cqs1.createQueue(createQueueRequest).getQueueUrl();
long end = System.currentTimeMillis();
recordResponseTime(null, end-start);
totalTime += end-start;
logger.info("average creation millis: " + (totalTime/(i+1)) + " last: " + (end-start));
queueUrls.add(queueUrl);
logger.info("created queue " + counter + ": " + queueUrl + " " + numPartitions + " partitions, " + numShards + " shards");
counter++;
} catch (Exception ex) {
logger.error("create failure", ex);
createFailures++;
}
}
Thread.sleep(1000);
long sendFailures = 0;
totalTime = 0;
long totalCounter = 0;
long messagesSent = 0;
for (int i=0; i<numMessages; i++) {
counter = 0;
for (String queueUrl : queueUrls) {
try {
String msg = "" + messagesSent;
if (messageLength > 0) {
msg += "-" + generateRandomMessage(messageLength);
} else if (messageFromFile != null) {
msg += "-" + messageFromFile + generateRandomMessage(100);
}
long start = System.currentTimeMillis();
SendMessageRequest sendMessageRequest = new SendMessageRequest(queueUrl, msg);
if (delaySeconds > 0) {
sendMessageRequest.setDelaySeconds(delaySeconds);
}
SendMessageResult result = cqs1.sendMessage(sendMessageRequest);
long end = System.currentTimeMillis();
recordResponseTime(null, end-start);
totalMessageSendTime.addAndGet(end-start);
totalTime += end-start;
logger.info("average send millis: " + (totalTime/(totalCounter+1)) + " last: " + (end-start));
logger.info("sent message on queue " + i + " - " + counter + ": " + queueUrl);
if (end-start>=500) {
logger.warn("SEND_RT=" + (end-start));
}
counter++;
totalCounter++;
messagesSent++;
} catch (Exception ex) {
logger.error("send failure", ex);
sendFailures++;
}
}
}
totalMessagesSent.addAndGet(totalCounter);
Thread.sleep(1000);
long readFailures = 0;
long emptyResponses = 0;
long messagesFound = 0;
long outOfOrder = 0;
long duplicates = 0;
totalTime = 0;
totalCounter = 0;
long lastReceiveTime = System.currentTimeMillis();
while (true) {
//for (int i=0; i<1.5*Math.max(numMessages/numBatchReceive,1); i++) {
if (System.currentTimeMillis() - lastReceiveTime > 1*1000) {
logger.info("NO MESSAGES FOR MORE THAN 1 SEC!");
break;
}
String lastNumericContent = null;
for (String queueUrl : queueUrls) {
try {
long start = System.currentTimeMillis();
List<String> messageBodies = null;
//if (restReceive) {
// messageBodies = receiveMessageREST(queueUrl);
//} else {
messageBodies = receiveMessage(queueUrl);
//}
long end = System.currentTimeMillis();
totalTime += end-start;
logger.info("avg_receive_ms=" + (totalTime/(totalCounter+1)) + " last_receive_ms=" + (end-start));
if (messageBodies == null) {
logger.info("event=no_message_found msg_count=" + totalCounter + " queue_url" + queueUrl);
emptyResponses++;
} else if (messageBodies.size() > 0) {
lastReceiveTime = System.currentTimeMillis();
for (String messageBody : messageBodies) {
/*String numericContent = messageBody;
if (messageBody.length()>0 && messageBody.contains("-")) {
numericContent = messageBody.substring(0, messageBody.indexOf("-"));
}
if (lastNumericContent != null && numericContent != null && Long.parseLong(lastNumericContent) > Long.parseLong(numericContent)) {
outOfOrder++;
}
if (messageMap.containsKey(messageBody)) {
logger.warn("event=DUPLICATE body=" + messageBody.substring(0, 50));
duplicates++;
} else {
messageMap.put(messageBody, null);
}
lastNumericContent = numericContent;*/
messagesFound++;
totalCounter++;
totalMessagesFound.incrementAndGet();
}
String messagePrefix = messageBodies.get(0);
if (messagePrefix.length() > 50) {
messagePrefix = messagePrefix.substring(0, 50);
}
logger.info("event=received_message batch_size=" + messageBodies.size() + " msg_count=" + totalCounter + " queue_url=" + queueUrl + " body=" + messagePrefix);
} else {
lastReceiveTime = System.currentTimeMillis();
}
} catch (Exception ex) {
logger.error("read failure, will retry: " + queueUrl, ex);
readFailures++;
}
}
}
long deleteFailures = 0;
counter = 0;
totalTime = 0;
if (deleteQueues) {
Thread.sleep(1000);
for (String queueUrl : queueUrls) {
try {
long start = System.currentTimeMillis();
cqs1.deleteQueue(new DeleteQueueRequest(queueUrl));
long end = System.currentTimeMillis();
recordResponseTime(null, end-start);
totalTime += end-start;
logger.info("average delete millis: " + (totalTime/(counter+1)) + " last: " + (end-start));
logger.info("deleted queue " + counter + ": " + queueUrl);
counter++;
} catch (Exception ex) {
logger.error("delete failure", ex);
deleteFailures++;
}
}
}
long testEnd = System.currentTimeMillis();
String message = "thread=" + Thread.currentThread().getName() + " duration_sec=" + ((testEnd-testStart)/1000) + " create failuers: " + createFailures + " delete_failures=" + deleteFailures + " send_failures=" + sendFailures + " read_failures_" + readFailures + " empty_reads=" + emptyResponses + " messages_found=" + messagesFound + " messages_sent=" + messagesSent + " out_of_order=" + outOfOrder + " duplicates=" + duplicates + " distinct_messages=" + messageMap.size() + " empty_responses=" + emptyResponses;
logger.info("event=thread_finished");
report.add(new Date() + ": " + message);
if (logReport) {
logger.info(message);
assertTrue("Create failures: " + createFailures, createFailures == 0);
assertTrue("Delete failures: " + deleteFailures, deleteFailures == 0);
assertTrue("Send failures: " + sendFailures, sendFailures == 0);
assertTrue("Read failures: " + readFailures, readFailures == 0);
//assertTrue("Empty reads: " + emptyResponses, emptyResponses == 0);
assertTrue("Wrong number of messages found!", messagesFound == messagesSent);
}
} catch (Exception ex) {
ex.printStackTrace();
fail(ex.getMessage());
}
}
}