* Copyright 2012 Comcast Corporation

*

* Licensed under the Apache License, Version 2.0 (the "License");

* you may not use this file except in compliance with the License.

* You may obtain a copy of the License at

*

*   http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

// some useful greps:

//

// grep -i responsetime /tmp/receiveMessage.log | grep ReceiveMessage | cut -f9- -d= > /tmp/redisTime.out

//

// grep -i responsetime /tmp/receiveMessage.log | grep ReceiveMessage | cut -f5-5 -d= > /tmp/responseTime.log

//

// grep -i responsetime /tmp/receiveMessage.log | grep ReceiveMessage |  cut -f6-6 -d= > /tmp/cassandraTime.log

//

// some useful splunks:

//

// response time for receive message operation:

//

// sourcetype=cqs action=ReceiveMessage | chart perc95(responseTimeMS) perc90(responseTimeMS) avg(responseTimeMS) median(responseTimeMS) max(responseTimeMS) perc95(redisTime) perc90(redisTime) avg(redisTime) median(redisTime) max(redisTime) perc95(CassandraTimeMS) perc90(CassandraTimeMS) avg(CassandraTimeMS) median(CassandraTimeMS) max(CassandraTimeMS)

//

// sourcetype=cqs action=ReceiveMessage | timechart avg(redisTime) avg(CassandraTimeMS) avg(responseTimeMS)

//

// manage changes in message visibility, performed once a second:

//

// sourcetype=cqs event=chechAndProcessRevisibleSet | chart avg(responseTimeMS) p95(responseTimeMS)

//

// manage hidden messages every 10 sec, look for hiddenSetSize parameter:

//

// sourcetype=cqs event=RevisibleProcessor | chart avg(responseTimeMS) p95(responseTimeMS)

//

// todo: check for message order

    private static Logger logger = Logger.getLogger(CqsStressTester.class);

  static ConcurrentHashMap<Integer, AtomicInteger> timeReceiveMessageCountMap = new ConcurrentHashMap<Integer, AtomicInteger>();

  static ConcurrentHashMap<Integer, AtomicInteger> timeSendMessageCountMap = new ConcurrentHashMap<Integer, AtomicInteger>();

  static Set<String> sendMessageIdSet = Collections.newSetFromMap(new ConcurrentHashMap<String,Boolean>());

    //static private ConcurrentLinkedQueue<Long> receiveLacencyMSList = new ConcurrentLinkedQueue<Long>();

    public long startTime = System.currentTimeMillis();

    final static SchemeRegistry schemeRegistry = new SchemeRegistry();

    public final static ThreadSafeClientConnManager cm;

    final static HttpClient httpClient;

    static {

       schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));

       schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory()));

       cm = new ThreadSafeClientConnManager(schemeRegistry);

       // Increase max total connection to 200

       cm.setMaxTotal(CMBProperties.getInstance().getCNSPublisherHttpEndpointConnectionPoolSize());

       // Increase default max connection per route to 20

       cm.setDefaultMaxPerRoute(CMBProperties.getInstance().getCNSPublisherHttpEndpointConnectionsPerRouteSize());

       httpClient = new DefaultHttpClient(cm);

    }

    private List<String> queueUrls = new ArrayList<String>();

    private HashMap<String, List<Receiver>> receiverMap = new HashMap<String, List<Receiver>>();

  private static final String ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";

    private static HashMap<String, String> attributeParams = new HashMap<String, String>();

    private static AtomicInteger messageCount = new AtomicInteger(0);

    static Random rand = new Random();

    static User user = null;

    final static int revisiblePercentage = CQSStressTestProperties.getInstance().getRevisiblePercentage();

    private static void setup() throws Exception {

        attributeParams.put("MessageRetentionPeriod", "6000"); // the number of seconds when a message is available in the system - 10 hours

        attributeParams.put("VisibilityTimeout", "60"); // the number of seconds a message is hidden from the next user - 1 min

        try {

          IUserPersistence userPersistence = new UserCassandraPersistence();

          user = userPersistence.getUserByName("cqs_stress_user");

          if (user == null) {

              user = userPersistence.createUser("cqs_stress_user", "cqs_stress_user");

          }

        } catch (Exception ex) {

            logger.error("Action=setup status=exception ", ex);

        }

    }

    // This creates the specified number of queues

    private ScheduledExecutorService createQueuesAndInitializePublishersAndReceivers() throws PersistenceException, NoSuchAlgorithmException, UnsupportedEncodingException, InterruptedException {

      String fixedQueueName = "testQueue";

      int totalNumberOfQueues = CQSStressTestProperties.getInstance().getNumberOfQueues();

        String[] queueNames = CQSStressTestProperties.getInstance().getQueueNames();

        if (queueNames != null) {

        totalNumberOfQueues = queueNames.length;

      }

        ICQSMessagePersistence messagePersistence = PersistenceFactory.getCQSMessagePersistence();

        for (int i=0; i<totalNumberOfQueues; i++) {

          String queueName = fixedQueueName + rand.nextInt() + "_" + i;

          if (queueNames != null) {

          queueName = queueNames[i];

        }

          String myQueueUrl = createQueue(queueName);

      queueUrls.add(myQueueUrl);

      if (CQSStressTestProperties.getInstance().getNumberOfSendersPerQueue() > 0) {

        messagePersistence.clearQueue(myQueueUrl, 0);

      }

      logger.info("QueueUrl" + i + " = " + myQueueUrl);

      //first tickle the empty queue population by calling a receive when nothing is in the queue

      receiveMessage(myQueueUrl, 1, 100);

      Thread.sleep(500);

      }

        for (String queueUrl : queueUrls) {

      createReceivers(queueUrl);

        }

        return createSenders(queueUrls);

   }

    private ScheduledExecutorService createSenders(List<String> queueUrls) {

      int senderCount = CQSStressTestProperties.getInstance().getNumberOfSendersPerQueue();

      if (senderCount == 0) {

        return null;

      }

      int numberOfMessagesPerSec = CQSStressTestProperties.getInstance().getMessagesPerQueuePerSecond();

    ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(queueUrls.size()*senderCount);

    for (String queueUrl : queueUrls) {

      for (int i=0; i<senderCount; i ++) {

        scheduledExecutorService.scheduleWithFixedDelay(new MessageSender(queueUrl, i), rand.nextInt(100), 1000*senderCount/numberOfMessagesPerSec, TimeUnit.MILLISECONDS);

      }

    }

    return scheduledExecutorService;

    }

  private String generateRandomMessage(int length) {

    StringBuilder sb = new StringBuilder(length);

    Date now = new Date();

    sb.append("@").append(now.getTime()).append("*");

    sb.append(now).append("%");

    for (int i=sb.length(); i<length; i++) {

      sb.append(ALPHABET.charAt(rand.nextInt(ALPHABET.length())));

    }

    return sb.toString();

  }

  private void createReceivers(String queueUrl) {

      AbstractDurablePersistence persistence = DurablePersistenceFactory.getInstance();

      long receiverCount = CQSStressTestProperties.getInstance().getNumberOfReceiversPerQueue();

      List<Receiver> receiverListForQueue = new ArrayList<Receiver>();

      for (int i=0; i<receiverCount; i++) {

        Receiver receiver = new Receiver(queueUrl, i, persistence);

        receiver.start();

        receiverListForQueue.add(receiver);

      }

      receiverMap.put(queueUrl, receiverListForQueue);

    }

    protected void doChart(int totalMessagesSent, int totalMessagesReceived) throws IOException {

      generateChart("Send message count: Total Messages Sent=" + totalMessagesSent,  timeSendMessageCountMap, "SendMessage_Run_" + startTime, new Date(startTime), new Date());

        generateChart("Receive message count: Total Messages Received=" + totalMessagesReceived,  timeReceiveMessageCountMap, "ReceiveMessage_Run_" + startTime, new Date(startTime), new Date());

    }

    private byte[] generateChart(String title, ConcurrentHashMap<Integer, AtomicInteger> metric, String id, Date startTime, Date endTime) throws IOException {

      XYSeries series = new XYSeries(title);

      for (Entry<Integer, AtomicInteger> entry : metric.entrySet()) {

        series.add(entry.getKey().intValue(), entry.getValue().intValue());

      }

      XYSeriesCollection dataset = new XYSeriesCollection(series);

      JFreeChart chart = ChartFactory.createXYBarChart(

                "Start: " + startTime + " End: " + endTime + " Message Count: " + messageCount,

                "Test Second",

                false,

                "Number of Messages",

                dataset,

                PlotOrientation.VERTICAL,

                true,

                true,

                false

            );

      File file = new File("/tmp/" + id + ".jpeg");

      //File file = new File(getServletContext().getRealPath("WEB-INF" + "/" + id + ".jpeg"));

      ChartUtilities.saveChartAsJPEG(file, chart, 1600, 400);

      //byte b[] = Files.toByteArray(file);

      //return b;

      ByteArrayOutputStream bos = new ByteArrayOutputStream();

      ChartUtilities.writeChartAsJPEG(bos, chart, 2400, 400);

      return bos.toByteArray();

    }

    /**

   * @param args

   */

  public static void main(String[] args) {

    try {

      CMBControllerServlet.valueAccumulator.initializeAllCounters();

          setup();

      Util.initLog4j();

      CqsStressTester tester = new CqsStressTester();

      ScheduledExecutorService scheduledExecutorService = tester.createQueuesAndInitializePublishersAndReceivers(); //will clear queues

      long totalMessagesReceived = 0;

      long totalMessagesDeleted = 0;

      long totalMessagesRevisibled = 0;

      long totalEmptyResponses = 0;

      long totalDuplicates = 0;

      long totalOutOfOrderMessages = 0;

      int testDurationSeconds = CQSStressTestProperties.getInstance().getTestDurationSeconds();

      Thread.sleep(testDurationSeconds*1000);

      if (scheduledExecutorService != null) {

        scheduledExecutorService.shutdown();

      }

      logger.info("===Sender Shutdown Triggered==");

      for (String queueUrl : tester.receiverMap.keySet()) {

          for (Receiver receiver : tester.receiverMap.get(queueUrl)) {

            receiver.setContinueThread(false);

          }

      }

      for (String queueUrl : tester.receiverMap.keySet()) {

        Set<String> messageIdMaster = new HashSet<String>();

          List<Integer> deleteTimesMaster = new ArrayList<Integer>();

          List<Long> flightTimesMaster = new ArrayList<Long>();

          List<Long> receiveTimesMaster = new ArrayList<Long>();

          for (Receiver receiver : tester.receiverMap.get(queueUrl)) {

          receiver.join();

          logger.warn("===================================================================================================================");

          logger.warn("TheadId=" + receiver.getThreadId() + " receiveMessageCount=" + receiver.getTotalMessagesReceived() + " deletedMessageCount=" + receiver.getTotalMessagesDeleted() + " revisibledMessageCount=" + receiver.getTotalMessagesRevisibled());

          logger.warn("===================================================================================================================");

          totalMessagesReceived += receiver.getTotalMessagesReceived();

          totalMessagesDeleted += receiver.getTotalMessagesDeleted();

          totalMessagesRevisibled += receiver.getTotalMessagesRevisibled();

          totalOutOfOrderMessages += receiver.getTotalOutOfOrderMessages();

          totalDuplicates += checkAndCombine(messageIdMaster, receiver.messageIds);

          //deleteTimesMaster.addAll(receiver.deleteLatencyMSList);

          //flightTimesMaster.addAll(receiver.flightTimeList);

          totalEmptyResponses += receiver.emptyResponseCount;

        }

        logger.warn("===================================================================================================================");

        Iterator<String> iter = sendMessageIdSet.iterator();

        while (iter.hasNext()) {

          logger.error("Missed message:" + iter.next());

        }

        Collections.sort(deleteTimesMaster);

        Collections.sort(flightTimesMaster);

                //receiveTimesMaster.addAll(receiveLacencyMSList);

        Collections.sort(receiveTimesMaster);

        /*logger.warn("Receive message latencies");

        if (receiveTimesMaster.size() > 0) {

          for (int i=5; i<=100; i+=5) {

            int percentileIndex = receiveTimesMaster.size()*i/100 - 1;

            if (percentileIndex < 0) {

              percentileIndex = 0;

            }

            logger.warn("" + i + "th percentile=" + receiveTimesMaster.get(percentileIndex));

          }

        }

        logger.warn("===================================================================================================================");

        logger.warn("Message flight time latencies");

        if (flightTimesMaster.size() > 0) {

          for (int i=5; i<=100; i+=5) {

            int percentileIndex = flightTimesMaster.size()*i/100 - 1;

            if (percentileIndex < 0) {

              percentileIndex = 0;

            }

            logger.warn("" + i + "th percentile=" + flightTimesMaster.get(percentileIndex));

          }

        }

        logger.warn("===================================================================================================================");

        logger.warn("Delete message latencies");

        if (deleteTimesMaster.size() > 0) {

          for (int i=5; i<=100; i+=5) {

            int percentileIndex = deleteTimesMaster.size()*i/100 - 1;

            if (percentileIndex < 0) {

              percentileIndex = 0;

            }

            logger.warn("" + i + "th percentile=" + deleteTimesMaster.get(percentileIndex));

          }

        }*/

      }

      logger.warn("===================================================================================================================");

      logger.warn("===================================================================================================================");

      logger.warn("totalMessagesSent=" + tester.messageCount.get() + " totalMessagesReceived=" + totalMessagesReceived + " totalMessagesDeleted=" + totalMessagesDeleted + " totalMessagesRevisibled=" + totalMessagesRevisibled);

      logger.warn("===================================================================================================================");

      logger.warn("totalEmptyResponses=" + totalEmptyResponses);

      logger.warn("totalDuplicates=" + totalDuplicates);

      logger.warn("totalOutOfOrderMessages=" + totalOutOfOrderMessages);

      logger.warn("totalMessagesLost=" + sendMessageIdSet.size());

      logger.warn("===================================================================================================================");

      logger.warn("totalRunTimeMillis=" + (System.currentTimeMillis()-tester.startTime) + " status=Exit");

      logger.warn("===================================================================================================================");

      logger.warn("===================================================================================================================");

    } catch (Exception e) {

      logger.error("Thread=main status=exception message=setup_failure ", e);

    } finally {

      CMBControllerServlet.valueAccumulator.deleteAllCounters();

    }

  }

  private static long checkAndCombine(Set<String> masterSet, Set<String> receivedMessageIds) {

    long totalDuplicates = 0;

    if (masterSet == null || receivedMessageIds == null) {

      return 0;

    }

    for (String messageId : receivedMessageIds) {

      if (masterSet.contains(messageId)) {

        logger.error("Action=checkAndCombine status=error message=duplicateId:" + messageId);

        totalDuplicates++;

      } else {

        masterSet.add(messageId);

      }

    }

    return totalDuplicates;

  }

    private class MessageSender implements Runnable {

    private String queueUrl;

    private String threadId;

      public MessageSender(String queueUrl, int index) {

      setQueueUrl(queueUrl);

      setThreadId(queueUrl, index);

    }

    public String getQueueUrl() {

      return queueUrl;

    }

    public void setQueueUrl(String queueUrl) {

      this.queueUrl = queueUrl;

    }

    public void setThreadId(String queueUrl, int index) {

      this.queueUrl = queueUrl;

      if (queueUrl == null || queueUrl.length() == 0) {

        return;

      }

      String queueName = queueUrl.substring(queueUrl.lastIndexOf('/') + 1);

      this.threadId = "Sender_" + queueName + "_" + index;

    }

    @Override

    public void run() {

      try {

            CMBControllerServlet.valueAccumulator.initializeAllCounters();

      } catch (Exception ex) {

              logger.error("Thread=" + threadId + " Action=setup status=exception ", ex);

      }

      int maxSendBatchSize = CQSStressTestProperties.getInstance().getSendMessageBatchSize();

      maxSendBatchSize = 1;  // Till we add support for send message batch

      try {

          String currentTime = "" + System.currentTimeMillis();

            long startNanoTime = System.nanoTime();

            int totalMessagesSuccessful = 0;

              for (int i = 0; i < maxSendBatchSize; i++) {

                //TBD: Make this a configurable parameter

                String messageBodyRandom = generateRandomMessage(2000);

                long index = rand.nextLong();

                String messageIndex = Thread.currentThread().getId() + "_" + index;

                sendMessageIdSet.add(messageIndex);

            String messageBody = "currentTime=" + currentTime + " messageIndex=" + messageIndex + " messagebody=message_" + currentTime + "_" + messageBodyRandom + "_" + index;

            if (sendMessage(this.queueUrl, messageBody) != null) {

              totalMessagesSuccessful++;

            }

                  logger.debug("Thread=" + threadId + " Action=Sent MessageBody '" + messageBody + "'");

              }

              //addSendMessageCount((int)((System.currentTimeMillis() - startTime)/1000), totalMessagesSuccessful);

              long endNanoTime = System.nanoTime();

              logger.info("Thread=" + threadId + " Action=SendMessageBatch latencyNano=" + (endNanoTime-startNanoTime));

        int count = messageCount.addAndGet(totalMessagesSuccessful);

        if (count % 100 == 0) {

          logger.info("event=publish queueUrl=" + getQueueUrl() + " totalCount=" + count);

        }

      } catch (AmazonServiceException ase) {

        displayServiceException("" + Thread.currentThread().getId(), "SendMessageBatch", ase);

      } finally {

        CMBControllerServlet.valueAccumulator.deleteAllCounters();

      }

    }

    }

  class Receiver extends Thread {

    private String queueUrl;

    private String threadId;

    private long totalMessagesReceived = 0;

    private long totalMessagesDeleted = 0;

    private long totalMessagesRevisibled = 0;

    private long totalOutOfOrderMessages = 0;

    private long lastMessageReceivedTime = 0;

    private boolean continueThread = true;

    private AbstractDurablePersistence persistence;

    private static final int visibilityTimeout = 600;

    private Set<String> messageIds = new HashSet<String>();

    //private List<Integer> deleteLatencyMSList = new ArrayList<Integer>();

    //private Set<Long> flightTimeList = new HashSet<Long>();

    private long emptyResponseCount = 0;

    public Receiver(String queueUrl, int index, AbstractDurablePersistence persistence) {

      setQueueUrl(queueUrl);

      setThreadId(queueUrl, index);

      setPersistence(persistence);

    }

    public String getQueueUrl() {

      return queueUrl;

    }

    public void setQueueUrl(String queueUrl) {

      this.queueUrl = queueUrl;

    }

    public void run() {

      try {

            CMBControllerServlet.valueAccumulator.initializeAllCounters();

      } catch (Exception ex) {

              logger.info("Thread=" + threadId + " Action=setup status=exception ", ex);

      }

      int maxReceiveBatchSize = CQSStressTestProperties.getInstance().getReceiveMessageBatchSize();

      try {

        int emptyCount = 0;

        while (true) {

              long startNanoTime = System.currentTimeMillis();

              List<CQSMessage> messageList = receiveMessage(this.queueUrl, maxReceiveBatchSize, visibilityTimeout);

              if (messageList == null) {

                Thread.sleep(300);

                continue;

              }

              long currentTime = System.currentTimeMillis();

            long messageFlightTime = 0;

            String messageIndex = "";

            if (messageList.size() == 0) {

                        emptyResponseCount++;

              logger.info("Thread=" + threadId + " Action=ReceiveMessage batchSize=" + maxReceiveBatchSize + " status=Empty count=" + emptyCount);

              if (!isContinueThread()) {

                          // sleep an extra second if no messages are there to account for potentially hidden messages

              Thread.sleep(1000);

                          emptyCount++;

                          if (emptyCount > 10) {

                  logger.info("Thread=" + threadId + "Action=ReceiveMessage status=Completed");

                  return;

                }

              }

            } else {

              emptyCount = 0;

              totalMessagesReceived += messageList.size();

              logger.info("Thread=" + threadId + " Action=ReceiveMessage batchSize=" + maxReceiveBatchSize + " receivedCount=" + messageList.size());

              //addReceiveMessageCount((int)((System.currentTimeMillis() - startTime)/1000), messageList.size());

            }

            int delayBetweenReceiveAndDelete = CQSStressTestProperties.getInstance().getDelayBetweenReceiveAndDeleteMS();

            if (delayBetweenReceiveAndDelete > 0) {

              Thread.sleep(delayBetweenReceiveAndDelete + rand.nextInt(40));

            }

            for (CQSMessage message: messageList) {

              String[] bodyParts = message.getBody().split(" ");

              for (String bodyPart : bodyParts) {

                String[] subParts = bodyPart.split("=");

                if (subParts[0].equals("currentTime")) {

                      long messageSendTime = Long.parseLong(subParts[1]);

                  messageFlightTime = currentTime - messageSendTime;

                  if (lastMessageReceivedTime != 0 && lastMessageReceivedTime > messageSendTime) {

                    totalOutOfOrderMessages++;

                    //logger.info("Event=MessageOutOfOrder messageId=" + messageIndex + " messageSendTime=" + messageSendTime + " lastMessageReceivedTime=" + lastMessageReceivedTime + " delta=" + Math.abs(lastMessageReceivedTime-messageSendTime));

                  }

                  lastMessageReceivedTime = messageSendTime;

                    } else if (subParts[0].equals("messageIndex")) {

                      messageIndex = subParts[1];

                    }

                  }

                  if (messageIds.contains(messageIndex)) {

                    logger.error("Action=receiveMessage status=error exception=Duplicate id: " + messageIndex);

                  } else {

                    messageIds.add(messageIndex);

                  }

                  //flightTimeList.add(messageFlightTime);

                  logger.info("Thread=" + threadId + " Action=ReceivedMessage MessageIndex=" + messageIndex + " totalTimeInFlightMillis=" + messageFlightTime);

                  startNanoTime = System.nanoTime();

                  if (revisiblePercentage > 0 && rand.nextInt(100)+1 <= revisiblePercentage) {

                        changeMessageVisibility(this.queueUrl, message.getReceiptHandle());

                        totalMessagesRevisibled += 1;

                        totalMessagesReceived -= 1;

                        messageIds.remove(messageIndex);

                    long endNanoTime = System.nanoTime();

                    logger.info("Thread=" + threadId + " Action=RevisibleMessage MessageIndex=" + messageIndex + " latencyNano=" + (endNanoTime-startNanoTime));

                  } else {

                    sendMessageIdSet.remove(messageIndex);

                    deleteMessage(this.queueUrl, message.getReceiptHandle());

                    totalMessagesDeleted += 1;

                    long endNanoTime = System.nanoTime();

                    logger.info("Thread=" + threadId + " Action=DeleteMessage MessageIndex=" + messageIndex + " latencyNano=" + (endNanoTime-startNanoTime));

                  }

                  //deleteLatencyMSList.add(new Integer((int)(endNanoTime-startNanoTime)/1000000));

                }

            int delayBetweenReceivesMS = CQSStressTestProperties.getInstance().getDelayBetweenReceivesMS();

            if (delayBetweenReceivesMS > 0) {

                  try {

              Thread.sleep(delayBetweenReceivesMS);

            } catch (InterruptedException e) {

              logger.error("Action=ReceiveMessage status=Exception ", e);

            }

                }

        }

      } catch (AmazonServiceException ase) {

        displayServiceException(threadId, "ReceiveMessage/DeleteMessage/RevisibleMessage", ase);

      } catch (InterruptedException e) {

        e.printStackTrace();

      } finally {

        CMBControllerServlet.valueAccumulator.deleteAllCounters();

      }

    }

    public String getThreadId() {

      return threadId;

    }

    public void setThreadId(String queueUrl, int index) {

      this.queueUrl = queueUrl;

      if (queueUrl == null || queueUrl.length() == 0) {

        return;

      }

      String queueName = queueUrl.substring(queueUrl.lastIndexOf('/') + 1);

      this.threadId = "Receiver_" + queueName + "_" + index;

    }

    public long getTotalMessagesReceived() {

      return totalMessagesReceived;

    }

    public void setTotalMessagesReceived(long totalMessagesReceived) {

      this.totalMessagesReceived = totalMessagesReceived;

    }

    public long getTotalMessagesDeleted() {

      return totalMessagesDeleted;

    }

    public void setTotalMessagesDeleted(long totalMessagesDeleted) {

      this.totalMessagesDeleted = totalMessagesDeleted;

    }

    public long getTotalMessagesRevisibled() {

      return totalMessagesRevisibled;

    }

    public void setTotalMessagesRevisibled(long totalMessagesRevisibled) {

      this.totalMessagesRevisibled = totalMessagesRevisibled;

    }

    public synchronized boolean isContinueThread() {

      return continueThread;

    }

    public synchronized void setContinueThread(boolean continueThread) {

      this.continueThread = continueThread;

    }

    public AbstractDurablePersistence getPersistence() {

      return persistence;

    }

    public void setPersistence(AbstractDurablePersistence persistence) {

      this.persistence = persistence;

    }

    public long getTotalOutOfOrderMessages() {

      return this.totalOutOfOrderMessages;

    }

  }

    private void displayServiceException(String threadId, String action, AmazonServiceException ase) {

      logger.error("ThreadId=" + threadId + " Action=" + action);

        logger.error("Caught an AmazonServiceException, which means your request made it to Amazon SQS, but was rejected with an error response for some reason.");

        logger.error("Error Message=" + ase.getMessage());

        logger.error("HTTP Status Code=" + ase.getStatusCode());

        logger.error("AWS Error Code=" + ase.getErrorCode());

        logger.error("Error Type=" + ase.getErrorType());

        logger.error("Request ID=" + ase.getRequestId());

    }

    public void addReceiveMessageCount(int second, int count) {

      AtomicInteger val = null;

    timeReceiveMessageCountMap.putIfAbsent(second, new AtomicInteger(0));

    val = timeReceiveMessageCountMap.get(second);

      val.addAndGet(count);

    }

    public void addSendMessageCount(int second, int count) {

      AtomicInteger val = null;

      timeSendMessageCountMap.putIfAbsent(second, new AtomicInteger(0));

      val = timeSendMessageCountMap.get(second);

      val.addAndGet(count);

    }

    public String createQueue(String queueName) {

      Map<String, String[]> params = new HashMap<String, String[]>();

    CommunicationUtils.addParam(params,"Action", "CreateQueue");

    CommunicationUtils.addParam(params, "QueueName", queueName);

    CommunicationUtils.addParam(params, "AWSAccessKeyId", user.getAccessKey());

    CommunicationUtils.addParam(params, "Version", "2009-02-01");

    try {

      String response = send(params, CMBProperties.getInstance().getCQSServiceUrl());

      return CqsStressTester.deserialize(response, "QueueUrl").trim();

    } catch (Exception e) {

      logger.error("Action=CreateQueue status=error exception=", e);

      return null;

    }

    }

    public String sendMessage(String queueUrl, String messageBody) {

    Map<String, String[]> params = new HashMap<String, String[]>();

    CommunicationUtils.addParam(params,"Action", "SendMessage");

    CommunicationUtils.addParam(params, "MessageBody", messageBody);

    CommunicationUtils.addParam(params, "AWSAccessKeyId", user.getAccessKey());

    CommunicationUtils.addParam(params, "Version", "2009-02-01");

    try {

      String response = send(params, queueUrl);

      String messageId = CqsStressTester.deserialize(response, "MessageId");

      return (messageId!=null)?messageId.trim() : null;

    } catch (Exception e) {

      logger.error("Action=sendMessage status=error exception=", e);

      return null;

    }

    }

    public List<CQSMessage> receiveMessage(String queueUrl, int maxNoOfMessages, int visibilityTimeout) {

      // Max number of messages will be set to 1 for now

      maxNoOfMessages = 1;

    Map<String, String[]> params = new HashMap<String, String[]>();

    CommunicationUtils.addParam(params,"Action", "ReceiveMessage");

    CommunicationUtils.addParam(params, "MaxNumberOfMessages", "" + maxNoOfMessages);

    CommunicationUtils.addParam(params, "VisibilityTimeout", "" + visibilityTimeout);

    CommunicationUtils.addParam(params, "AWSAccessKeyId", user.getAccessKey());

    CommunicationUtils.addParam(params, "Version", "2009-02-01");

    try {

      long ts1 = System.currentTimeMillis();

      String response = send(params, queueUrl);

      long elapsedTime = System.currentTimeMillis() - ts1;

      logger.info("Total time spent on receiveMessage=" + elapsedTime);

      if (response.indexOf("<Body>") > 0) {

        //receiveLacencyMSList.add(elapsedTime);

      }

      return CqsStressTester.deserializeMessage(response);

    } catch (Exception e) {

      logger.error("Action=receiveMessage status=error exception=", e);

      return null;

    }

    }

    public void deleteMessage(String queueUrl, String receiptHandle) {

      Map<String, String[]> params = new HashMap<String, String[]>();

    CommunicationUtils.addParam(params,"Action", "DeleteMessage");

    CommunicationUtils.addParam(params, "ReceiptHandle", receiptHandle);

    CommunicationUtils.addParam(params, "AWSAccessKeyId", user.getAccessKey());

    CommunicationUtils.addParam(params, "Version", "2009-02-01");

    try {

      send(params, queueUrl);

    } catch (Exception e) {

      logger.error("Action=deleteMessage status=error exception=", e);

    }

    }

    public void changeMessageVisibility(String queueUrl, String receiptHandle) {

      Map<String, String[]> params = new HashMap<String, String[]>();

      CommunicationUtils.addParam(params,"Action", "ChangeMessageVisibility");

    CommunicationUtils.addParam(params, "ReceiptHandle", receiptHandle);

    int randomVisibilityTimeoutSecs = rand.nextInt(5);

    CommunicationUtils.addParam(params, "VisibilityTimeout", randomVisibilityTimeoutSecs+"");

    CommunicationUtils.addParam(params, "AWSAccessKeyId", user.getAccessKey());

    CommunicationUtils.addParam(params, "Version", "2009-02-01");

    try {

      send(params, queueUrl);

    } catch (Exception e) {

      logger.error("Action=changeMessageVisibilityTimeout status=error exception=", e);

    }

    }

    public String send(Map<String, String[]> params, String endPoint) throws Exception {

    logger.debug("event=send_cqs_message endpoint=" + endPoint);

    String url = endPoint;

    logger.debug("event=send_cqs_message url=" + url + " endpoint=" + endPoint+ "\"");

    Set<String> parameters = params.keySet();

    boolean first = true;

    for (String param: parameters) {

      if(first) {url += "?"; first=false;}

      else url += "&";

      url += URLEncoder.encode(param,"UTF-8") + "=" + URLEncoder.encode(params.get(param)[0], "UTF-8");

    }

    String resp = "";

    try {

      logger.debug("Sending request to url:" + url);

       //resp = sendHttpMessage(url);

      resp = send(url, "");

    } catch(Exception e) {

      logger.error("event=send_cqs_message endpoint=" + endPoint + " exception=" + e.toString(), e);

      throw new CMBException(CQSErrorCodes.InternalError, "internal service error");

    }

    return resp;

  }

    public String send(String endpoint, String message) throws Exception {

        logger.debug("event=send_http_request endpoint=" + endpoint + "\" message=\"" + message + "\"");

        if ((message == null) || (endpoint == null)) {

            logger.debug("event=send_http_request error_code=MissingParameters endpoint=" + endpoint + "\" message=\"" + message + "\"");

            throw new Exception("Message and Endpoint must both be set");

        }

        HttpPost httpPost = new HttpPost(endpoint);

        StringEntity stringEntity = new StringEntity(message);

        httpPost.setEntity(stringEntity);

        HttpResponse response = httpClient.execute(httpPost);

        response.getStatusLine().getStatusCode();

        HttpEntity entity = response.getEntity();

        if (entity != null) {

            InputStream instream = entity.getContent();

            InputStreamReader responseReader = new InputStreamReader(instream);

            StringBuffer responseB = new StringBuffer();

            char []arr = new char[1024];

            int size = 0;

            while ((size = responseReader.read(arr, 0, arr.length)) != -1) {

                responseB.append(arr, 0, size);

            }

            instream.close();

            return responseB.toString();

        }

        logger.error("Could not get response entity");

        System.exit(1);

        return null;

    }

  public static List<CQSMessage> deserializeMessage(String serializedMessage) {

    javax.xml.parsers.SAXParserFactory fac = new org.apache.xerces.jaxp.SAXParserFactoryImpl();

    javax.xml.parsers.SAXParser saxParser;

    SaxHandler p = new SaxHandler();

    try {

      saxParser = fac.newSAXParser();

      saxParser.parse(new ByteArrayInputStream(serializedMessage.getBytes()), p);

    } catch (ParserConfigurationException e) {

      e.printStackTrace();

    } catch (Exception e) {

        e.printStackTrace();

    }

    CQSMessage cqsMessage = null;

    try {

      if (p.getValueByKey("Body") == null) {

        return new ArrayList<CQSMessage>();

      }

      cqsMessage = new CQSMessage(p.getValueByKey("Body"), p.getAttributes());

        cqsMessage.setMessageId(p.getValueByKey("MessageId"));

      cqsMessage.setReceiptHandle(p.getValueByKey("ReceiptHandle"));

        cqsMessage.setMD5OfBody(p.getValueByKey("MD5OfBody"));

    } catch (NoSuchAlgorithmException e) {

        e.printStackTrace();

    } catch (UnsupportedEncodingException e) {

      e.printStackTrace();

    }

    return java.util.Arrays.asList(cqsMessage) ;

  }

  public static String deserialize(String serialized, String key) {

    javax.xml.parsers.SAXParserFactory fac = new org.apache.xerces.jaxp.SAXParserFactoryImpl();

    javax.xml.parsers.SAXParser saxParser;

    SaxHandler p = new SaxHandler();

    try {

      saxParser = fac.newSAXParser();

      saxParser.parse(new ByteArrayInputStream(serialized.getBytes()), p);

    } catch (ParserConfigurationException e) {

      e.printStackTrace();

    } catch (Exception e) {

        e.printStackTrace();

    }

    return p.getValueByKey(key);

  }

        private CharArrayWriter contents = new CharArrayWriter();

        private HashMap<String, String> messageMap = new HashMap<String, String>();

        private String _name;

        private String _value;

        private HashMap<String, String> messageAttributes = new HashMap<String, String>();

        @Override

        public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {

          // clear the contents since we are in a new tag

          if (qName.equals("Message")) {

            messageMap.clear();

            messageAttributes.clear();

          }

          contents.reset();

        }

        @Override

        public void endElement(String uri, String localName, String qName) {

          if (qName.equals("Name")) {

            _name = contents.toString();

          } else if (qName.equals("Value")) {

            _value = contents.toString();

          } else if (qName.equals("Attribute")) {

            messageAttributes.put(_name, _value);

          } else if (!qName.equals("Message") && !qName.contains("Response") && !qName.contains("Result") && !qName.equals("RequestId")){

            messageMap.put(qName, contents.toString());

          }

        }

        @Override

      public void characters(char[] ch, int start, int length) {

        contents.write(ch, start, length);

        }

        String getValueByKey(String k) {

          return messageMap.get(k);

        }

        HashMap<String, String> getAttributes() {

          return messageAttributes;

        }

    }

}