//String userId = request.getParameter("userId");
//connect(userId);
IUserPersistence userHandler = PersistenceFactory.getUserPersistence();
User cnsAdminUser;
try {
cnsAdminUser = userHandler.getUserByName(CMBProperties.getInstance().getCNSUserName());
} catch (PersistenceException ex) {
throw new ServletException(ex);
}
String currentDataCenter = null;
if (parameters.containsKey("ClearWorkerQueues")) {
try {
String host = request.getParameter("Host");
httpGet(cnsServiceBaseUrl + "?Action=ManageService&Host="+host+"&Task=ClearWorkerQueues&AWSAccessKeyId=" + cnsAdminUser.getAccessKey());
} catch (Exception ex) {
logger.error("event=failed_to_clear_queues", ex);
throw new ServletException(ex);
}
} else if (parameters.containsKey("RemoveWorkerRecord")) {
try {
String host = request.getParameter("Host");
httpGet(cnsServiceBaseUrl + "?Action=ManageService&Host="+host+"&Task=RemoveWorkerRecord&AWSAccessKeyId=" + cnsAdminUser.getAccessKey());
} catch (Exception ex) {
logger.error("event=failed_to_clear_queues", ex);
throw new ServletException(ex);
}
} else if (parameters.containsKey("ClearAPIStats")) {
try {
String url = request.getParameter("Url");
httpGet(url + "?Action=ManageService&Task=ClearAPIStats&AWSAccessKeyId=" + cnsAdminUser.getAccessKey());
} catch (Exception ex) {
logger.error("event=failed_to_clear_queues", ex);
throw new ServletException(ex);
}
} else if (parameters.containsKey("RemoveRecord")) {
try {
String host = request.getParameter("Host");
httpGet(cnsServiceBaseUrl + "?Action=ManageService&Host="+host+"&Task=RemoveRecord&AWSAccessKeyId=" + cnsAdminUser.getAccessKey());
} catch (Exception ex) {
logger.error("event=failed_to_clear_queues", ex);
throw new ServletException(ex);
}
} else if (parameters.containsKey("StartWorker") ||parameters.containsKey("StopWorker")) {
try {
String dataCenter = request.getParameter("DataCenter");
String actionString;
if(parameters.containsKey("StartWorker")){
actionString = "StartWorker";
} else {
actionString = "StopWorker";
}
httpGet(cnsServiceBaseUrl + "?Action=ManageService&DataCenter="+dataCenter+"&Task="+actionString+"&AWSAccessKeyId=" + cnsAdminUser.getAccessKey());
} catch (Exception ex) {
logger.error("event=failed_to_start_or_stop_worker ", ex);
throw new ServletException(ex);
}
} else if(parameters.containsKey("currentDataCenter")){
currentDataCenter = request.getParameter("currentDataCenter");
}
if(currentDataCenter == null){
currentDataCenter = CMBProperties.getInstance().getCMBDataCenter();
}
out.println("<html>");
this.header(request, out, "CNS Worker State");
out.println("<body>");
//show drop down box for data center at the top
String url = null;
try{
url = cnsServiceBaseUrl + "?Action=GetAPIStats&SubTask=GetDataCenter&AWSAccessKeyId=" + cnsAdminUser.getAccessKey();
String workerStateXml = httpGet(url);
Element root = XmlUtil.buildDoc(workerStateXml);
List<Element> DataCenterList = XmlUtil.getCurrentLevelChildNodes(XmlUtil.getCurrentLevelChildNodes(root, "GetAPIStatsResult").get(0), "DataCenter");
out.println("<table cellspacing='0' cellpadding='0'><tr>");
out.println("<td valign='top'>Data Center: </td>");
out.println("<td><div>");
out.print("<form action=\"\" method=\"POST\">");
out.println("<select class=\"service-area\" name='currentDataCenter' onchange='this.form.submit()'>");
String dataCenter = null;
for(Element dataCenterElement:DataCenterList){
dataCenter=dataCenterElement.getFirstChild().getNodeValue();
if(dataCenter.equals(currentDataCenter)){
out.println("<option selected value=\""+dataCenter+"\">"+dataCenter+"</option>");
}else {
out.println("<option value=\""+dataCenter+"\">"+dataCenter+"</option>");
}
}
out.println("</select>");
out.println("<noscript><input type=\"submit\" value=\"Submit\"></noscript>");
out.println("</form>");
out.println("</div></td></tr></table>");
} catch (Exception ex) {
out.println("<p>Unable to reach " + url + ": "+ex.getMessage()+"</p>");
logger.error("", ex);
}
//show detailed stats
url = null;
try {
url = cnsServiceBaseUrl + "?Action=GetWorkerStats&DataCenter="+currentDataCenter+"&AWSAccessKeyId=" + cnsAdminUser.getAccessKey();
String workerStateXml = httpGet(url);
Element root = XmlUtil.buildDoc(workerStateXml);
List<Element> statsList = XmlUtil.getCurrentLevelChildNodes(XmlUtil.getCurrentLevelChildNodes(root, "GetWorkerStatsResult").get(0), "Stats");
out.println("<h2 align='left'>CNS Worker Stats</h2>");
out.println("<span class='simple'><table border='1'>");
out.println("<tr><th>Host</th><th>Jmx Port</th><th>Mode</th><th>Data Center</th><th>Msg Published</th>");
out.println("<th>Producer Heartbeat</th><th>Active</th><th>Consumer Heartbeat</th><th>Active</th>");
out.println("<th>Delivery Queue Size</th><th>Redelivery Queue Size</th><th>Consumer Overloaded</th><th>Cqs Service Available</th><th>Http Pool Size</th><th></th><th></th></tr>");
String alarmColor = " bgcolor='#C00000'";
String okColor = " bgcolor='#00C000'";
int deliveryQueueMaxSize = CMBProperties.getInstance().getCNSDeliveryHandlerJobQueueLimit();
int redeliveryQueueMaxSize = CMBProperties.getInstance().getCNSReDeliveryHandlerJobQueueLimit();
Map<String, Integer> endpointErrorCounts = new HashMap<String, Integer>();
for (Element stats : statsList) {
out.println("<tr>");
String host = XmlUtil.getCurrentLevelTextValue(stats, "IpAddress");
out.println("<td>"+host+"</td>");
out.println("<td>"+XmlUtil.getCurrentLevelTextValue(stats, "JmxPort")+"</td>");
String mode = XmlUtil.getCurrentLevelTextValue(stats, "Mode");
out.println("<td>"+mode+"</td>");
String dataCenter = XmlUtil.getCurrentLevelTextValue(stats, "DataCenter");
out.println("<td>"+dataCenter+"</td>");
out.println("<td>"+XmlUtil.getCurrentLevelTextValue(stats, "NumPublishedMessages")+"</td>");
out.println("<td>"+new Date(Long.parseLong(XmlUtil.getCurrentLevelTextValue(stats, "ProducerTimestamp")))+"</td>");
boolean activeProducer = Boolean.parseBoolean(XmlUtil.getCurrentLevelTextValue(stats, "ActiveProducer"));
out.println("<td"+(!activeProducer && mode.contains("Producer") ? alarmColor : okColor)+">"+activeProducer+"</td>");
out.println("<td>"+new Date(Long.parseLong(XmlUtil.getCurrentLevelTextValue(stats, "ConsumerTimestamp")))+"</td>");
boolean activeConsumer = Boolean.parseBoolean(XmlUtil.getCurrentLevelTextValue(stats, "ActiveConsumer"));
out.println("<td"+(!activeConsumer && mode.contains("Consumer") ? alarmColor : okColor)+">"+activeConsumer+"</td>");
int deliveryQueueSize = Integer.parseInt(XmlUtil.getCurrentLevelTextValue(stats, "DeliveryQueueSize"));
out.println("<td"+(1.0*deliveryQueueSize/deliveryQueueMaxSize >= 0.75 ? alarmColor : okColor)+">"+deliveryQueueSize+"</td>");
int redeliveryQueueSize = Integer.parseInt(XmlUtil.getCurrentLevelTextValue(stats, "RedeliveryQueueSize"));
out.println("<td"+(1.0*redeliveryQueueSize/redeliveryQueueMaxSize >= 0.75 ? alarmColor : okColor)+">"+redeliveryQueueSize+"</td>");
boolean consumerOverloaded = Boolean.parseBoolean(XmlUtil.getCurrentLevelTextValue(stats, "ConsumerOverloaded"));
out.println("<td"+(consumerOverloaded ? alarmColor : okColor)+">"+consumerOverloaded+"</td>");
boolean cqsServiceAvailable = Boolean.parseBoolean(XmlUtil.getCurrentLevelTextValue(stats, "CqsServiceAvailable"));
out.println("<td"+((!cqsServiceAvailable && (activeProducer || activeConsumer))? alarmColor : okColor)+">"+cqsServiceAvailable+"</td>");
out.println("<td>"+XmlUtil.getCurrentLevelTextValue(stats, "NumPooledHttpConnections")+"</td>");
out.println("<td><form action=\"\" method=\"POST\"><input type='hidden' name='Host' value='"+host+"'><input type='submit' value='Clear Queues' name='ClearWorkerQueues'/></form></td>");
out.println("<td><form action=\"\" method=\"POST\"><input type='hidden' name='Host' value='"+host+"'><input type='submit' value='Remove Record' name='RemoveWorkerRecord'/></form></td>");
out.println("</tr>");
List<Element> errorCounts = XmlUtil.getChildNodes(stats, "ErrorCountForEndpoints");
if (errorCounts.size() == 1) {
for (Element errorCount : XmlUtil.getChildNodes(errorCounts.get(0), "Error")) {
String endpoint = XmlUtil.getCurrentLevelAttributeTextValue(errorCount, "endpoint");
Integer count = XmlUtil.getCurrentLevelAttributeIntValue(errorCount, "count");
if (!endpointErrorCounts.containsKey(endpoint)) {
endpointErrorCounts.put(endpoint, count);
} else {
endpointErrorCounts.put(endpoint, endpointErrorCounts.get(endpoint) + count);
}
}
}
}
out.println("</table></span>");
out.println("<table><tr>");
out.println("<td><form action=\"\" method=\"POST\"><input type='hidden' name='DataCenter' value='"+currentDataCenter+"'><input type='submit' value='Start Workers' name='StartWorker'/></form></td>");
out.println("<td><form action=\"\" method=\"POST\"><input type='hidden' name='DataCenter' value='"+currentDataCenter+"'><input type='submit' value='Stop Workers' name='StopWorker'/></form></td>");
out.println("</tr></table>");
if (endpointErrorCounts.size() > 0) {
out.println("<h2 align='left'>Failed or timed out Responses during past 60 Seconds by Endpoint</h2>");
out.println("<span class='simple'><table border='1'>");
out.println("<tr><th>Endpoint</th><th>Error Count</th><th><th>");
for (String endpoint : endpointErrorCounts.keySet()) {
out.println("<tr>");
out.println("<td>"+endpoint+"</td>");
out.println("<td>"+endpointErrorCounts.get(endpoint)+"</td>");
int failureSuspensionThreshold = CMBProperties.getInstance().getEndpointFailureCountToSuspensionThreshold();
if (failureSuspensionThreshold != 0 && endpointErrorCounts.get(endpoint) > failureSuspensionThreshold) {
out.println("<td bgcolor='#C00000'><i>suspended</i></td>");
} else {
out.println("<td> </td>");
}
out.println("</tr>");
}
out.println("</table></span>");
}
// api call stats
url = cnsServiceBaseUrl + "?Action=GetAPIStats&DataCenter="+currentDataCenter+"&AWSAccessKeyId=" + cnsAdminUser.getAccessKey();
String apiStateXml = httpGet(url);
root = XmlUtil.buildDoc(apiStateXml);
statsList = XmlUtil.getCurrentLevelChildNodes(XmlUtil.getCurrentLevelChildNodes(root, "GetAPIStatsResult").get(0), "Stats");