/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.oodt.pcs.tools;
//JDK imports
import java.util.Calendar;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Vector;
//APACHE imports
import org.apache.xmlrpc.XmlRpcClient;
//OODT imports
import org.apache.oodt.commons.date.DateUtils;
import org.apache.oodt.cas.metadata.Metadata;
import org.apache.oodt.cas.filemgr.metadata.CoreMetKeys;
import org.apache.oodt.cas.filemgr.structs.Product;
import org.apache.oodt.cas.resource.structs.ResourceNode;
import org.apache.oodt.cas.crawl.daemon.CrawlDaemonController;
import org.apache.oodt.pcs.util.FileManagerUtils;
import org.apache.oodt.pcs.util.ResourceManagerUtils;
import org.apache.oodt.pcs.util.WorkflowManagerUtils;
import org.apache.oodt.pcs.health.CrawlInfo;
import org.apache.oodt.pcs.health.CrawlPropertiesFile;
import org.apache.oodt.pcs.health.CrawlerHealth;
import org.apache.oodt.pcs.health.CrawlerStatus;
import org.apache.oodt.pcs.health.JobHealthStatus;
import org.apache.oodt.pcs.health.PCSDaemonStatus;
import org.apache.oodt.pcs.health.PCSHealthMonitorMetKeys;
import org.apache.oodt.pcs.health.PCSHealthMonitorReport;
import org.apache.oodt.pcs.health.WorkflowStatesFile;
/**
*
* A tool to monitor the health of the PCS.
*
* @author mattmann
* @version $Revision$
*/
public final class PCSHealthMonitor implements CoreMetKeys,
PCSHealthMonitorMetKeys {
private FileManagerUtils fm;
private WorkflowManagerUtils wm;
private ResourceManagerUtils rm;
private CrawlPropertiesFile crawlProps;
private WorkflowStatesFile statesFile;
public PCSHealthMonitor(String fmUrlStr, String wmUrlStr, String rmUrlStr,
String crawlPropFilePath, String statesFilePath)
throws InstantiationException {
this.fm = new FileManagerUtils(fmUrlStr);
this.wm = new WorkflowManagerUtils(wmUrlStr);
this.rm = new ResourceManagerUtils(rmUrlStr);
this.crawlProps = new CrawlPropertiesFile(crawlPropFilePath);
this.statesFile = new WorkflowStatesFile(statesFilePath);
}
public PCSHealthMonitorReport getReport() {
PCSHealthMonitorReport report = new PCSHealthMonitorReport();
report.setGenerationDate(new Date());
report.setFmStatus(getFileManagerStatus());
report.setWmStatus(getWorkflowManagerStatus());
report.setRmStatus(getResourceManagerStatus());
report.setBatchStubStatus(getBatchStubStatus());
report.setCrawlerStatus(getCrawlerStatus());
report.setLatestProductsIngested(getProductHealth());
report.setJobHealthStatus(getJobStatusHealth());
report.setCrawlerHealthStatus(getIngestHealth());
return report;
}
public void quickPrintMonitorToConsole() {
System.out.println(HEADER_AND_FOOTER);
System.out.println(REPORT_BANNER);
System.out.println("Generated on: "
+ DateUtils.toString(Calendar.getInstance()));
System.out.println("");
System.out.println("Service Status:");
System.out.println("");
System.out.println(FILE_MANAGER_DAEMON_NAME
+ getStrPadding(FILE_MANAGER_DAEMON_NAME, WORKFLOW_MANAGER_DAEMON_NAME)
+ ":\t[" + this.fm.getFmUrl() + "]: " + printUp(getFmUp()));
System.out.println(WORKFLOW_MANAGER_DAEMON_NAME + ":\t["
+ this.wm.getWmUrl() + "]: " + printUp(getWmUp()));
System.out.println(RESOURCE_MANAGER_DAEMON_NAME + ":\t["
+ this.rm.getResmgrUrl() + "]: " + printUp(getRmUp()));
quickPrintBatchStubs();
System.out.println("");
System.out.println("Crawlers:");
quickPrintCrawlers();
System.out.println("");
System.out.println("PCS Health: ");
System.out.println("");
System.out.println("Files:");
System.out.println(SECTION_SEPARATOR);
quickPrintProductHealth();
System.out.println("");
System.out.println("Jobs:");
System.out.println(SECTION_SEPARATOR);
quickPrintJobStatusHealth();
System.out.println("");
System.out.println("Ingest:");
System.out.println(SECTION_SEPARATOR);
quickPrintIngestStatusHealth();
System.out.println(HEADER_AND_FOOTER);
}
public void printMonitorToConsole(PCSHealthMonitorReport report) {
System.out.println(HEADER_AND_FOOTER);
System.out.println(REPORT_BANNER);
System.out.println("Generated on: " + report.getCreateDateIsoFormat());
System.out.println("");
System.out.println("Service Status:");
System.out.println("");
System.out.println(report.getFmStatus().getDaemonName()
+ getStrPadding(report.getFmStatus().getDaemonName(), report
.getWmStatus().getDaemonName()) + ":\t["
+ report.getFmStatus().getUrlStr() + "]: "
+ report.getFmStatus().getStatus());
System.out.println(report.getWmStatus().getDaemonName() + ":\t["
+ report.getWmStatus().getUrlStr() + "]: "
+ report.getWmStatus().getStatus());
System.out.println(report.getRmStatus().getDaemonName() + ":\t["
+ report.getRmStatus().getUrlStr() + "]: "
+ report.getRmStatus().getStatus());
printBatchStubs(report);
System.out.println("");
System.out.println("Crawlers:");
printCrawlers(report);
System.out.println("");
System.out.println("PCS Health: ");
System.out.println("");
System.out.println("Files:");
System.out.println(SECTION_SEPARATOR);
printProductHealth(report);
System.out.println("");
System.out.println("Jobs:");
System.out.println(SECTION_SEPARATOR);
printJobStatusHealth(report);
System.out.println("");
System.out.println("Ingest:");
System.out.println(SECTION_SEPARATOR);
printIngestStatusHealth(report);
System.out.println(HEADER_AND_FOOTER);
}
public static void main(String[] args) throws Exception {
String usage = "PCSHealthMonitor <fm url> <wm url> <rm url> <crawler xml file path> <workflow states xml file path>\n";
String fmUrlStr = null, wmUrlStr = null, rmUrlStr = null;
String crawlerXmlFilePath = null, workflowStateXmlPath = null;
if (args.length != 5) {
System.err.println(usage);
System.exit(1);
}
fmUrlStr = args[0];
wmUrlStr = args[1];
rmUrlStr = args[2];
crawlerXmlFilePath = args[3];
workflowStateXmlPath = args[4];
PCSHealthMonitor mon = new PCSHealthMonitor(fmUrlStr, wmUrlStr, rmUrlStr,
crawlerXmlFilePath, workflowStateXmlPath);
try {
mon.quickPrintMonitorToConsole();
} catch (Exception e) {
e.printStackTrace();
}
}
private PCSDaemonStatus getFileManagerStatus() {
PCSDaemonStatus fmStatus = new PCSDaemonStatus();
fmStatus.setDaemonName(FILE_MANAGER_DAEMON_NAME);
fmStatus.setStatus(printUp(getFmUp()));
fmStatus.setUrlStr(this.fm.getFmUrl().toString());
return fmStatus;
}
private PCSDaemonStatus getWorkflowManagerStatus() {
PCSDaemonStatus wmStatus = new PCSDaemonStatus();
wmStatus.setDaemonName(WORKFLOW_MANAGER_DAEMON_NAME);
wmStatus.setStatus(printUp(getWmUp()));
wmStatus.setUrlStr(this.wm.getWmUrl().toString());
return wmStatus;
}
private PCSDaemonStatus getResourceManagerStatus() {
PCSDaemonStatus rmStatus = new PCSDaemonStatus();
rmStatus.setDaemonName(RESOURCE_MANAGER_DAEMON_NAME);
rmStatus.setStatus(printUp(getRmUp()));
rmStatus.setUrlStr(this.rm.getResmgrUrl().toString());
return rmStatus;
}
private List getBatchStubStatus() {
List batchStubStatus = new Vector();
if (getRmUp()) {
// only print if the resource manager is up
List resNodes = rm.safeGetResourceNodes();
if (resNodes != null && resNodes.size() > 0) {
for (Iterator i = resNodes.iterator(); i.hasNext();) {
ResourceNode node = (ResourceNode) i.next();
PCSDaemonStatus batchStatus = new PCSDaemonStatus();
batchStatus.setDaemonName(BATCH_STUB_DAEMON_NAME);
batchStatus.setUrlStr(node.getIpAddr().toString());
batchStatus.setStatus(printUp(getBatchStubUp(node)));
batchStubStatus.add(batchStatus);
}
}
}
return batchStubStatus;
}
private List getCrawlerStatus() {
List crawlers = this.crawlProps.getCrawlers();
String crawlHost = this.crawlProps.getCrawlHost();
List statuses = new Vector();
if (crawlers != null && crawlers.size() > 0) {
Collections.sort(crawlers, new Comparator() {
public int compare(Object o1, Object o2) {
CrawlInfo c1 = (CrawlInfo) o1;
CrawlInfo c2 = (CrawlInfo) o2;
return c1.getCrawlerName().compareTo(c2.getCrawlerName());
}
});
for (Iterator i = crawlers.iterator(); i.hasNext();) {
CrawlInfo info = (CrawlInfo) i.next();
String crawlerUrlStr = "http://" + crawlHost + ":"
+ info.getCrawlerPort();
CrawlerStatus status = new CrawlerStatus();
status.setInfo(info);
status.setStatus(printUp(getCrawlerUp(crawlerUrlStr)));
status.setCrawlHost(crawlHost);
statuses.add(status);
}
}
return statuses;
}
private List getProductHealth() {
if (getFmUp()) {
return this.fm.safeGetTopNProducts(TOP_N_PRODUCTS);
} else
return new Vector();
}
private List getJobStatusHealth() {
if (!getWmUp()) {
return new Vector();
}
List statuses = new Vector();
List states = this.statesFile.getStates();
if (states != null && states.size() > 0) {
for (Iterator i = states.iterator(); i.hasNext();) {
String state = (String) i.next();
int numPipelines = this.wm.safeGetNumWorkflowInstancesByStatus(state);
if (numPipelines == -1) {
numPipelines = 0;
}
JobHealthStatus jobStatus = new JobHealthStatus();
jobStatus.setStatus(state);
jobStatus.setNumPipelines(numPipelines);
statuses.add(jobStatus);
}
}
return statuses;
}
private List getIngestHealth() {
if (this.crawlProps.getCrawlers() == null) {
return new Vector();
}
List statuses = new Vector();
for (Iterator i = this.crawlProps.getCrawlers().iterator(); i.hasNext();) {
CrawlInfo info = (CrawlInfo) i.next();
String crawlUrlStr = "http://" + this.crawlProps.getCrawlHost() + ":"
+ info.getCrawlerPort();
try {
CrawlDaemonController controller = new CrawlDaemonController(
crawlUrlStr);
CrawlerHealth health = new CrawlerHealth();
health.setCrawlerName(info.getCrawlerName());
health.setNumCrawls(controller.getNumCrawls());
health
.setAvgCrawlTime((double) (controller.getAverageCrawlTime() / 1000.0));
statuses.add(health);
} catch (Exception e) {
CrawlerHealth health = new CrawlerHealth();
health.setCrawlerName(info.getCrawlerName());
health.setNumCrawls(CRAWLER_DOWN_INT);
health.setAvgCrawlTime(CRAWLER_DOWN_DOUBLE);
statuses.add(health);
}
}
return statuses;
}
private void printIngestStatusHealth(PCSHealthMonitorReport report) {
if (report.getCrawlerHealthStatus() != null
&& report.getCrawlerHealthStatus().size() > 0) {
for (Iterator i = report.getCrawlerHealthStatus().iterator(); i.hasNext();) {
CrawlerHealth health = (CrawlerHealth) i.next();
System.out.print(health.getCrawlerName() + ":");
if (health.getNumCrawls() == CRAWLER_DOWN_INT) {
System.out.println(" DOWN");
} else {
System.out.println("");
System.out.println("Number of Crawls: " + health.getNumCrawls());
System.out.println("Average Crawl Time (seconds): "
+ health.getAvgCrawlTime());
System.out.println("");
}
}
}
}
private void printJobStatusHealth(PCSHealthMonitorReport report) {
if (report.getJobHealthStatus() != null
&& report.getJobHealthStatus().size() > 0) {
for (Iterator i = report.getJobHealthStatus().iterator(); i.hasNext();) {
JobHealthStatus status = (JobHealthStatus) i.next();
System.out.println(status.getNumPipelines() + " pipelines "
+ status.getStatus());
}
}
}
private void printProductHealth(PCSHealthMonitorReport report) {
if (report.getLatestProductsIngested() != null
&& report.getLatestProductsIngested().size() > 0) {
System.out.println("Latest " + TOP_N_PRODUCTS + " products ingested:");
for (Iterator i = report.getLatestProductsIngested().iterator(); i
.hasNext();) {
Product p = (Product) i.next();
p.setProductType(fm.safeGetProductTypeById(p.getProductType()
.getProductTypeId()));
p.setProductReferences(fm.safeGetProductReferences(p));
Metadata prodMet = fm.safeGetMetadata(p);
System.out.println(fm.getFilePath(p) + " at: "
+ prodMet.getMetadata("CAS." + PRODUCT_RECEVIED_TIME));
}
}
}
private void printBatchStubs(PCSHealthMonitorReport report) {
if (report.getBatchStubStatus() != null
&& report.getBatchStubStatus().size() > 0) {
for (Iterator i = report.getBatchStubStatus().iterator(); i.hasNext();) {
PCSDaemonStatus batchStatus = (PCSDaemonStatus) i.next();
System.out.println("> " + batchStatus.getDaemonName() + ": ["
+ batchStatus.getUrlStr() + "]: " + batchStatus.getStatus());
}
}
}
private void printCrawlers(PCSHealthMonitorReport report) {
if (report.getCrawlerStatus() != null
&& report.getCrawlerStatus().size() > 0) {
List crawlers = this.crawlProps.getCrawlers();
String biggestString = getBiggestString(crawlers);
for (Iterator i = report.getCrawlerStatus().iterator(); i.hasNext();) {
CrawlerStatus status = (CrawlerStatus) i.next();
String crawlerUrlStr = "http://" + status.getCrawlHost() + ":"
+ status.getInfo().getCrawlerPort();
System.out.println(getStrPadding(status.getInfo().getCrawlerName(),
biggestString)
+ status.getInfo().getCrawlerName()
+ ": ["
+ crawlerUrlStr
+ "]: "
+ status.getStatus());
}
}
}
private void quickPrintCrawlers() {
List crawlers = this.crawlProps.getCrawlers();
String crawlHost = this.crawlProps.getCrawlHost();
if (crawlers != null && crawlers.size() > 0) {
Collections.sort(crawlers, new Comparator() {
public int compare(Object o1, Object o2) {
CrawlInfo c1 = (CrawlInfo) o1;
CrawlInfo c2 = (CrawlInfo) o2;
return c1.getCrawlerName().compareTo(c2.getCrawlerName());
}
});
String biggestString = getBiggestString(crawlers);
for (Iterator i = crawlers.iterator(); i.hasNext();) {
CrawlInfo info = (CrawlInfo) i.next();
String crawlerUrlStr = "http://" + crawlHost + ":"
+ info.getCrawlerPort();
System.out.println(getStrPadding(info.getCrawlerName(), biggestString)
+ info.getCrawlerName() + ": [" + crawlerUrlStr + "]: "
+ printUp(getCrawlerUp(crawlerUrlStr)));
}
}
}
private void quickPrintBatchStubs() {
List resNodes = null;
if (getRmUp()) {
// only print if the resource manager is up
resNodes = rm.safeGetResourceNodes();
if (resNodes != null && resNodes.size() > 0) {
for (Iterator i = resNodes.iterator(); i.hasNext();) {
ResourceNode node = (ResourceNode) i.next();
System.out.println("> " + BATCH_STUB_DAEMON_NAME + ": ["
+ node.getIpAddr() + "]: " + printUp(getBatchStubUp(node)));
}
}
}
}
private void quickPrintJobStatusHealth() {
if (!getWmUp()) {
return;
}
List states = this.statesFile.getStates();
if (states != null && states.size() > 0) {
for (Iterator i = states.iterator(); i.hasNext();) {
String state = (String) i.next();
int numPipelines = this.wm.safeGetNumWorkflowInstancesByStatus(state);
if (numPipelines == -1) {
numPipelines = 0;
}
System.out.println(numPipelines + " pipelines " + state);
}
}
}
private void quickPrintProductHealth() {
if (getFmUp()) {
System.out.println("Latest " + TOP_N_PRODUCTS + " products ingested:");
List prods = this.fm.safeGetTopNProducts(TOP_N_PRODUCTS);
if (prods != null && prods.size() > 0) {
for (Iterator i = prods.iterator(); i.hasNext();) {
Product p = (Product) i.next();
p.setProductType(fm.safeGetProductTypeById(p.getProductType()
.getProductTypeId()));
p.setProductReferences(fm.safeGetProductReferences(p));
Metadata prodMet = fm.safeGetMetadata(p);
System.out.println(fm.getFilePath(p) + " at: "
+ prodMet.getMetadata("CAS." + PRODUCT_RECEVIED_TIME));
}
}
}
}
private void quickPrintIngestStatusHealth() {
if (this.crawlProps.getCrawlers() == null) {
return;
}
for (Iterator i = this.crawlProps.getCrawlers().iterator(); i.hasNext();) {
CrawlInfo info = (CrawlInfo) i.next();
String crawlUrlStr = "http://" + this.crawlProps.getCrawlHost() + ":"
+ info.getCrawlerPort();
try {
CrawlDaemonController controller = new CrawlDaemonController(
crawlUrlStr);
System.out.println(info.getCrawlerName() + ":");
System.out.println("Number of Crawls: " + controller.getNumCrawls());
System.out.println("Average Crawl Time (seconds): "
+ (double) (controller.getAverageCrawlTime() / 1000.0));
System.out.println("");
} catch (Exception e) {
System.out.println(info.getCrawlerName() + ": DOWN");
}
}
}
private String getBiggestString(List crawlInfos) {
int biggestStrSz = Integer.MIN_VALUE;
String biggestStr = null;
for (Iterator i = crawlInfos.iterator(); i.hasNext();) {
CrawlInfo info = (CrawlInfo) i.next();
String crawlInfoName = info.getCrawlerName();
if (crawlInfoName.length() > biggestStrSz) {
biggestStr = crawlInfoName;
biggestStrSz = biggestStr.length();
}
}
return biggestStr;
}
private String getStrPadding(String initString, String compareString) {
int sizeInitStr = initString.length();
int sizeCompareStr = compareString.length();
int diff = Math.abs(sizeInitStr - sizeCompareStr);
StringBuffer buf = new StringBuffer();
for (int i = 0; i < diff; i++) {
buf.append(" ");
}
return buf.toString();
}
private boolean getBatchStubUp(ResourceNode node) {
XmlRpcClient client = new XmlRpcClient(node.getIpAddr());
Vector argList = new Vector();
try {
return ((Boolean) client.execute("batchstub.isAlive", argList))
.booleanValue();
} catch (Exception e) {
return false;
}
}
private boolean getCrawlerUp(String crawlUrlStr) {
CrawlDaemonController controller = null;
try {
controller = new CrawlDaemonController(crawlUrlStr);
return controller.isRunning();
} catch (Exception e) {
return false;
}
}
private boolean getFmUp() {
return fm.getFmgrClient() != null ? fm.getFmgrClient().isAlive() : false;
}
private boolean getWmUp() {
try {
wm.getClient().getRegisteredEvents();
return true;
} catch (Exception e) {
return false;
}
}
private boolean getRmUp() {
try {
rm.getClient().getNodes();
return true;
} catch (Exception e) {
return false;
}
}
private String printUp(boolean upFlag) {
return upFlag ? STATUS_UP : STATUS_DOWN;
}
}