Examples of com.comcast.cqs.model.CQSMessage

com.comcast.cqs.model.CQSMessage
Model for a cqs message @author aseem, baosen, vvenkatraman

          
          ReceiveMessageResult receiveMessageResult = sqs.receiveMessage(receiveMessageRequest);
          List<Message> msgs = receiveMessageResult.getMessages();
          
          for (Message m : msgs) {
            messages.add(new CQSMessage(m));
          }
        }


        long ts2 = System.currentTimeMillis();
        CMBControllerServlet.valueAccumulator.addToCounter(AccumulatorName.CNSCQSTime, ts2 - ts1);

View Full Code Here

            }
          
          String publishJobQName = CNS_PRODUCER_QUEUE_NAME_PREFIX + partition;
          String queueUrl = CQSHandler.getRelativeCnsInternalQueueUrl(publishJobQName);
          
          CQSMessage msg = null;
          int waitTimeSecs = 0;
          
          if (CMBProperties.getInstance().isCQSLongPollEnabled()) {
            waitTimeSecs = CMBProperties.getInstance().getCMBRequestTimeoutSec();
          }


          List<CQSMessage> l = CQSHandler.receiveMessage(queueUrl, waitTimeSecs, 1); 


          if (l.size() > 0) {
            msg = l.get(0);
          }


          CNSWorkerMonitor.getInstance().registerCQSServiceAvailable(true);


          if (msg != null) {
            
            // if long polling disabled and message found reset exponential backoff
            
            if (!CMBProperties.getInstance().isCQSLongPollEnabled()) {
              processingDelayMillis = 10;
            }
            
              CNSMessage publishMessage = CNSMessage.parseInstance(msg.getBody());
              
              int messageExpirationSeconds = CMBProperties.getInstance().getCNSMessageExpirationSeconds();
              
              if (messageExpirationSeconds != 0 && System.currentTimeMillis() - publishMessage.getTimestamp().getTime() > messageExpirationSeconds*1000) {
                  logger.error("event=deleting_publish_job reason=message_too_old topic_arn=" + publishMessage.getTopicArn());
                  CQSHandler.deleteMessage(queueUrl, msg.getReceiptHandle());
                  CMBControllerServlet.valueAccumulator.deleteAllCounters();
                  return true; // return true to avoid backoff
              }
              
              List<CNSEndpointPublishJob.CNSEndpointSubscriptionInfo> subscriptions = null;
              long t1=System.currentTimeMillis();
              try {
                  subscriptions = getSubscriptionsForTopic(publishMessage.getTopicArn());
              } catch (TopicNotFoundException e) {


                //delete this message/job since the topic was deleted.
                  
                logger.error("event=deleting_publish_job reason=topic_not_found topic_arn=" + publishMessage.getTopicArn());
                  CQSHandler.deleteMessage(queueUrl, msg.getReceiptHandle());
                  CMBControllerServlet.valueAccumulator.deleteAllCounters();
                  return true; // return true to avoid backoff
              
              } catch (Exception ex) {
                  logger.error("event=skipping_publish_job reason=error_fetching_subscriptions", ex);
                  CMBControllerServlet.valueAccumulator.deleteAllCounters();
                  return true; // return true to avoid backoff
              }
              logger.debug("event=get_subscription_list ms="+(System.currentTimeMillis()-t1));
              if (subscriptions != null && subscriptions.size() > 0) {
                
                  messageFound = true;
                  List<CNSEndpointPublishJob> epPublishJobs = createEndpointPublishJobs(publishMessage, subscriptions);
                  
                  for (CNSEndpointPublishJob epPublishJob: epPublishJobs) {
                    
                    String epQueueName =  CMBProperties.getInstance().getCNSEndpointPublishQueueNamePrefix() + ((new Random()).nextInt(CMBProperties.getInstance().getCNSNumEndpointPublishJobQueues()));
                      String epQueueUrl = CQSHandler.getRelativeCnsInternalQueueUrl(epQueueName);
                      CQSHandler.sendMessage(epQueueUrl, epPublishJob.serialize());
                  }
              }
              
              CQSHandler.deleteMessage(queueUrl, msg.getReceiptHandle());
              
              long ts2 = System.currentTimeMillis();
              logger.debug("event=processed_producer_job cns_cqs_ms=" + CMBControllerServlet.valueAccumulator.getCounter(AccumulatorName.CNSCQSTime) + " resp_ms=" + (ts2 - ts1));


          } else {

View Full Code Here

                return messageFound;
            }
          
          String queueName = CNS_CONSUMER_QUEUE_NAME_PREFIX + partition;
          String queueUrl = CQSHandler.getRelativeCnsInternalQueueUrl(queueName);
          CQSMessage msg = null;
          int waitTimeSecs = 0;
          
          if (CMBProperties.getInstance().isCQSLongPollEnabled()) {
            waitTimeSecs = CMBProperties.getInstance().getCMBRequestTimeoutSec();
          }


          List<CQSMessage> l = CQSHandler.receiveMessage(queueUrl, waitTimeSecs, 1); 


          if (l.size() > 0) {
            msg = l.get(0);
          }
          
        CNSWorkerMonitor.getInstance().registerCQSServiceAvailable(true);


            if (msg != null) {   
              
            // if long polling disabled and message found reset exponential backoff
            
            if (!CMBProperties.getInstance().isCQSLongPollEnabled()) {
              processingDelayMillis = 10;
            }
              
                messageFound = true;
                
                try {
                  
                    CNSEndpointPublishJob endpointPublishJob = (CMBProperties.getInstance().isCNSUseSubInfoCache()) ? CNSCachedEndpointPublishJob.parseInstance(msg.getBody()) : CNSEndpointPublishJob.parseInstance(msg.getBody());
                    logger.debug("endpoint_publish_job=" + endpointPublishJob.toString());
                    User pubUser =(new UserAuthModule()).getUserByUserId(endpointPublishJob.getMessage().getUserId());
                    List<? extends CNSEndpointSubscriptionInfo> subs = endpointPublishJob.getSubInfos();
                    
                    CNSWorkerMonitor.getInstance().registerSendsRemaining(endpointPublishJob.getMessage().getMessageId(), subs.size());
                    
                    AtomicInteger endpointPublishJobCount = new AtomicInteger(subs.size());                
                    
                    for (CNSEndpointSubscriptionInfo sub : subs) {             
                        
                      Runnable publishJob = null;
                      CNSMessage message = endpointPublishJob.getMessage();
                      message.setSubscriptionArn(sub.subArn);
                        
                        if (CMBProperties.getInstance().getCNSIOMode() == IO_MODE.SYNC) {
                          publishJob = new CNSPublishJob(message, pubUser, sub.protocol, sub.endpoint, sub.subArn, sub.rawDelivery, queueUrl, msg.getReceiptHandle(), endpointPublishJobCount);
                        } else {
                          publishJob = new CNSAsyncPublishJob(message, pubUser, sub.protocol, sub.endpoint, sub.subArn, sub.rawDelivery, queueUrl, msg.getReceiptHandle(), endpointPublishJobCount);
                        }


                        deliveryHandlers.submit(publishJob);
                    }
                    
                } catch (TopicNotFoundException e) {
                    logger.error("event=topic_not_found action=skip_job");
                    CQSHandler.deleteMessage(queueUrl, msg.getReceiptHandle());
                } catch (Exception e) {
                    logger.error("event=job_consumer_exception action=wait_for_revisibility", e);
                }
                
                long tsFinal = System.currentTimeMillis();

View Full Code Here


               if (memId == null) {
                 throw new IllegalStateException("Underlying storage layer returned a message that was not requested");
               }


               CQSMessage message = messageIdToMessage.getValue();


               if (message == null) {
                 logger.warn("event=message_is_null msg_id=" + messageIdToMessage.getKey());
                 //underlying layer does not have this message, remove it from Redis layer
                 jedis.zrem(key,memId);
                 continue;
               }


               message.setMessageId(memId);
               message.setReceiptHandle(memId);


               //get message-attributes and populate in message
               Map<String, String> msgAttrs = (message.getAttributes() != null) ?  message.getAttributes() : new HashMap<String, String>();
               List<String> attrs = jedis.hmget(queue.getRelativeUrl() + "-" + shard + "-A-" + memId, CQSConstants.REDIS_APPROXIMATE_FIRST_RECEIVE_TIMESTAMP, CQSConstants.REDIS_APPROXIMATE_RECEIVE_COUNT);


               if (attrs.get(0) == null) {
                 String firstRecvTS = Long.toString(System.currentTimeMillis());
                 jedis.hset(queue.getRelativeUrl() + "-" + shard + "-A-" + memId, CQSConstants.REDIS_APPROXIMATE_FIRST_RECEIVE_TIMESTAMP, firstRecvTS);
                 msgAttrs.put(CQSConstants.APPROXIMATE_FIRST_RECEIVE_TIMESTAMP, firstRecvTS);
               } else {                            
                 msgAttrs.put(CQSConstants.APPROXIMATE_FIRST_RECEIVE_TIMESTAMP, attrs.get(0));
               }


               int recvCount = 1;


               if (attrs.get(1) != null) {
                 recvCount = Integer.parseInt(attrs.get(1)) + 1;
               }


               jedis.hset(queue.getRelativeUrl() + "-" + shard + "-A-" + memId, CQSConstants.REDIS_APPROXIMATE_RECEIVE_COUNT, Integer.toString(recvCount));
               jedis.expire(queue.getRelativeUrl() + "-" + shard +  "-A-" + memId, 3600 * 24 * 14); //14 days expiration if not deleted
               msgAttrs.put(CQSConstants.APPROXIMATE_RECEIVE_COUNT, Integer.toString(recvCount));
               message.setAttributes(msgAttrs);
               ret.add(message);
             }


             if (ret.size() > 0) { //There may be cases where the underlying persistent message has two memIds while
               //cache filling. In such cases trying to retrieve message for the second memId may result in no message

View Full Code Here


         jedis = getResource();


         for (String memId : memIdsRet) {


           CQSMessage message = ret.get(memId);


           if (message != null) {


             // get message-attributes and populate in message


             Map<String, String> msgAttrs = (message.getAttributes() != null) ?  message.getAttributes() : new HashMap<String, String>();
             List<String> attrs = jedis.hmget(queueUrl + "-" + shard + "-A-" + memId, CQSConstants.REDIS_APPROXIMATE_FIRST_RECEIVE_TIMESTAMP, CQSConstants.REDIS_APPROXIMATE_RECEIVE_COUNT);


             if (attrs.get(0) != null) {
               msgAttrs.put(CQSConstants.APPROXIMATE_FIRST_RECEIVE_TIMESTAMP, attrs.get(0));
             }


             if (attrs.get(1) != null) {
               msgAttrs.put(CQSConstants.APPROXIMATE_RECEIVE_COUNT, Integer.toString(Integer.parseInt(attrs.get(1))));
             }


             message.setAttributes(msgAttrs);
             messages.add(message);
           }
         }
       } catch (JedisConnectionException e) {
         logger.warn("event=peek_message error_code=redis_unavailable num_connections=" + numRedisConnections.get());
         brokenJedis = true;
         cacheAvailable = false;
       } finally {
         if (jedis != null) {
           returnResource(jedis, brokenJedis);
         }
       }


       logger.debug("event=peek_queue cache=available queue_url=" + queueUrl + " shard=" + shard + " num_messages=" + messages.size());
       return messages;


     } else { //get from underlying storage
       String relativeUrlHash = Util.getQueueUrlHashFromCache(queueUrl);
       if (previousReceiptHandle != null) { //strip out this layer's id
         previousReceiptHandle = getMemQueueMessageMessageId(relativeUrlHash, previousReceiptHandle);
       }


       if (nextReceiptHandle != null) {
         nextReceiptHandle = getMemQueueMessageMessageId(relativeUrlHash, nextReceiptHandle);
       }        


       List<CQSMessage> messages = persistenceStorage.peekQueue(queueUrl, shard, previousReceiptHandle, nextReceiptHandle, length);


       for (CQSMessage message : messages) {
         String memId = getMemQueueMessage(message.getMessageId()); //TODO: initialDelay is 0
         message.setMessageId(memId);
         message.setReceiptHandle(memId);
       }


       logger.debug("event=peek_queue cache=unavailable queue_url=" + queueUrl + " shard=" + shard + " num_messages=" + messages.size());
       return messages;
     }

View Full Code Here


     Map<String, CQSMessage> persisMap = persistenceStorage.getMessages(queueUrl, messageIds);
     Map<String, CQSMessage> ret = new HashMap<String, CQSMessage>();


     for (Entry<String, CQSMessage> idToMessage : persisMap.entrySet()) {
       CQSMessage msg = idToMessage.getValue();
       if (msg == null) {
         continue;
       }
       String memId = messageIdToMemId.get(idToMessage.getKey());
       msg.setMessageId(memId);
       msg.setReceiptHandle(memId);
       ret.put(memId, msg);
     }


     return ret;
   }

View Full Code Here

        attributes.put(CQSConstants.SENDER_ID, user.getUserId());
        attributes.put(CQSConstants.SENT_TIMESTAMP, "" + System.currentTimeMillis());
        attributes.put(CQSConstants.APPROXIMATE_RECEIVE_COUNT, "0");
        attributes.put(CQSConstants.APPROXIMATE_FIRST_RECEIVE_TIMESTAMP, "");
        
        CQSMessage message = new CQSMessage(messageBody, attributes);
      CQSQueue queue = CQSCache.getCachedQueue(user, request);
      
      int shard = 0;
      
    String shardNumber = request.getParameter("Shard");


    if (shardNumber != null) {
      shard = Integer.parseInt(shardNumber);
    } else if (queue.getNumberOfShards() > 1) {
      shard = rand.nextInt(queue.getNumberOfShards());
    }
  
    if (shard < 0 || shard >= queue.getNumberOfShards()) {
          throw new CMBException(CMBErrorCodes.InvalidParameterValue, "Shard number " + shard +" exceeds number of available shards in queue (" + queue.getNumberOfShards() + ").");
    }


        String receiptHandle = PersistenceFactory.getCQSMessagePersistence().sendMessage(queue, shard, message);


        if (receiptHandle == null || receiptHandle.isEmpty()) {
            throw new CMBException(CMBErrorCodes.InternalError, "Failed to add message to queue");
        }
        
        request.setReceiptHandles(Arrays.asList(new String[] {receiptHandle}));
        message.setReceiptHandle(receiptHandle);
        message.setMessageId(receiptHandle);
        String out = CQSMessagePopulator.getSendMessageResponse(message);
        writeResponse(out, response);
        CQSMonitor.getInstance().addNumberOfMessagesReceived(queue.getRelativeUrl(), 1);
        
        try {

View Full Code Here

          continue;
        } else if (column.getValue() == null || column.getValue().length() == 0) {
          continue;
        }
        
        CQSMessage message = extractMessageFromJSON(queueUrl, column);
        messageList.add(message);
      }
      
      if (noMatch && messageList.size() > length) {
        messageList.remove(messageList.size() - 1);

View Full Code Here

  }
  
  private CQSMessage extractMessageFromJSON(String queueUrl, CmbColumn column) throws JSONException, PersistenceException, IOException {
    
    CQSQueue queue = null;
    CQSMessage m = new CQSMessage();
    
    try {
      queue = CQSCache.getCachedQueue(queueUrl);
    } catch (Exception ex) {
      throw new PersistenceException(ex);
    }
    
    if (queue == null) {
      throw new PersistenceException(CMBErrorCodes.InternalError, "Unknown queue " + queueUrl);
    }
    
    JSONObject json = new JSONObject((String)column.getValue());


    m.setMessageId(json.getString("MessageId"));
    m.setReceiptHandle(json.getString("MessageId"));
    m.setMD5OfBody(json.getString("MD5OfBody"));
    m.setBody(json.getString("Body"));
    
    if (m.getAttributes() == null) {
      m.setAttributes(new HashMap<String, String>());
    }
    
    if (json.has(CQSConstants.SENT_TIMESTAMP)) {
      m.getAttributes().put(CQSConstants.SENT_TIMESTAMP, json.getString(CQSConstants.SENT_TIMESTAMP));
    }
    
    if (json.has(CQSConstants.APPROXIMATE_RECEIVE_COUNT)) {
      m.getAttributes().put(CQSConstants.APPROXIMATE_RECEIVE_COUNT, json.getString(CQSConstants.APPROXIMATE_RECEIVE_COUNT));
    }


    if (json.has(CQSConstants.SENDER_ID)) {
      m.getAttributes().put(CQSConstants.SENDER_ID, json.getString(CQSConstants.SENDER_ID));
    }
    
    m.setTimebasedId(column.getName());
    
    if (queue.isCompressed()) {
      m.setBody(Util.decompress(m.getBody()));
    }
    
      return m;
  }

View Full Code Here

    }


  @Test
  public void testSendMessage() throws NoSuchAlgorithmException, PersistenceException, IOException, InterruptedException, JSONException {
    
        CQSMessage message = new CQSMessage("This is a test message " + (new Random()).nextInt(), attributes);
        attributes.put("SentTimestamp", "" + Calendar.getInstance().getTimeInMillis());
    String receiptHandle = persistence.sendMessage(queue, 0, message);
    assertNotNull(receiptHandle);
  }

View Full Code Here

0 1 2

TOP

Related Classes of com.comcast.cqs.model.CQSMessage

com.comcast.cns.tools.CNSEndpointPublisherJobConsumer

com.comcast.cns.tools.CNSEndpointPublisherJobProducer

com.comcast.cns.tools.CQSHandler

com.comcast.cqs.api.CQSAPI

com.comcast.cqs.controller.CQSQueueMessagesPageServlet

com.comcast.cqs.controller.CQSSendMessageAction

com.comcast.cqs.controller.CQSSendMessageBatchAction

com.comcast.cqs.persistence.CQSMessagePartitionedCassandraPersistence

com.comcast.cqs.persistence.RedisCachedCassandraPersistence

com.comcast.cqs.persistence.RedisSortedSetPersistence

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.