/* Copyright 1999-2008 Acelet.org. All rights reserved. GPL v2 license */
/** @author Wei Jiang */
package com.acelet.s.watchdog;
import java.io.*;
import java.net.*;
import java.text.*;
import java.util.*;
import com.acelet.lib.Externals;
import com.acelet.lib.JavaVersionException;
import com.acelet.lib.Kit;
import com.acelet.lib.LoggingConstants;
import com.acelet.lib.LoggingTimeoutException;
import com.acelet.lib.Phrase;
import com.acelet.s.MailServerData;
import com.acelet.s.PrintInfo;
import com.acelet.s.chore.CandidateChore;
import com.acelet.s.chore.Chore;
import com.acelet.s.chore.ChoreText;
import com.acelet.s.chore.WorkingChore;
import com.acelet.s.job.MemberTaskInUseException;
import com.acelet.s.job.RetryObject;
public class Watchdogging {
public static String version = "3.10";
public static String releaseDate = "2009-01-16";
public static String productName = "SuperWatchdog";
public static String licenseFileName = "watchdogLicense";
public static String NO_PERIODIC_PROMPT = "NoPeriodicPrompt";
public static final int WORKING_TASK_DISPLAY_MIN = 20;
public static int interval = 2000;
public static int extraIntervalForGateway = 20000;
public static boolean logIncludesResult = false;
public static int workingTaskDisplayMax = 20;
public static int queryTimeout = 15;
public static int logAgentTimeout = 15000;
public static long minimumDuration = 3000;
public static int maxHeartbeatMiss = 3;
public static int heartbeatMiss = -1;
public static long choreRefreshTime = 0;
public static long workingChoreRefreshTime = 0;
public static long preferencesRefreshTime = 0;
public static long candidateRefreshTime = 0;
public static long oldChoreRefreshTime = -1;
public static long oldWorkingChoreRefreshTime = -1;
public static long oldPreferencesRefreshTime = -1;
public static long oldCandidateRefreshTime = -1;
public static long refreshTime = 0;
public static int workingChoreDisplayMax = WORKING_TASK_DISPLAY_MIN;
public static long workingChoreRetireTime = System.currentTimeMillis() - 48*60*60*1000;
public static long displayStartTimeForWorkingChore = -1;
public static int detectMissedChoreOnOtherHostDelay = 10 * 1000;
public static Timer detectMissedChoreOnOtherHostTimer;
public static long superWatchdogStartTime;
public static boolean isDoer;
public static boolean isStartUp = true;
public static boolean isFirstTime = true;
public static boolean isLicenseValid = false;
public static String licenseType;
public static String licenseString;
public static boolean busy = false;
public static String doertalkerName;
public static boolean noPeriodicPrompt = false;
protected static boolean initialized = false;
protected Watchdogging() {
}
public static void cancelAllTimerTasks() {
Enumeration enumeration = Registry.watchTimerTaskHashtable.elements();
while (enumeration.hasMoreElements()) {
TimerTask timerTask = (TimerTask) enumeration.nextElement();
timerTask.cancel();
}
enumeration = Registry.timerTaskHashtable.elements();
while (enumeration.hasMoreElements()) {
TimerTask timerTask = (TimerTask) enumeration.nextElement();
timerTask.cancel();
}
}
static void checkDurations() throws Exception {
if (CommonTimerTask.isShutdown)
return;
long now = System.currentTimeMillis();
Enumeration enumeration = Registry.choreHashtable.elements();
while (enumeration.hasMoreElements()) {
Chore chore = (Chore) enumeration.nextElement();
long duration = chore.duration;
WorkingChore workingChore = null;
if (chore.duration > 0) {
for (int i = 0; i < Registry.workingChoreVector.size(); i++) {
workingChore = (WorkingChore) Registry.workingChoreVector.elementAt(i);
if (chore.getId() == workingChore.getId0()) {
if (workingChore.status == WorkingChore.STATUS_STARTED) {
if (workingChore.modifiedAt + duration < now) {
ForemanTimerTask.reportExceedingDuration(chore, workingChore);
}
}
}
}
}
}
}
public static void executeNow(String name) throws Exception {
Chore chore = Delegate.selectChore(name);
Chore tmpChore = chore.createTemporaryCopyForNow();
WatchTimerTask.createExecutorTimerTask(tmpChore, null);
}
static void iAmAlive() {
DateFormat dateFormat = DateFormat.getDateInstance();
System.out.println("\nWatchdogging: " + Phrase.get("TX_WORKING") + ": " +
new Date(System.currentTimeMillis()));
}
public static void init(boolean isDoer, int runningMode) throws Exception {
superWatchdogStartTime = System.currentTimeMillis();
String info = "SuperWatchdog: version=" + version + " isDoer=" + isDoer +
" runningMode=" + runningMode + " starting at: " + new Date().toString();
System.out.println(info);
new WatchdogProperties().printInfo();
Watchdogging.isDoer = isDoer;
Externals.runningMode = runningMode;
CommonTimerTask.isShutdown = false;
oldChoreRefreshTime = -1;
oldWorkingChoreRefreshTime = -1;
oldPreferencesRefreshTime = -1;
oldCandidateRefreshTime = -1;
Registry.choreHashtable = new Hashtable();
readFromDatabase();
doertalkerName = InetAddress.getLocalHost().getHostName() + " " +
Phrase.get("TX_SUPER_WATCHDOG") + " " +
(isDoer? Phrase.get("TX_DOER"): Phrase.get("TX_TALKER"));
initialized = true;
}
public static boolean isInitialized() {
return initialized;
}
private static void processChore(Chore comingChore) throws Exception {
if (isDoer == false)
return;
Long comingId = new Long(comingChore.getId());
Chore existingChore = (Chore) Registry.choreHashtable.get(comingId);
if (existingChore == null) {
WatchTimerTask.createWatchTimerTask(comingChore);
} else {
if (!existingChore.equals(comingChore)) {
updateTimerTask(comingChore);
}
}
}
static void readFromDatabase() throws Exception {
if (CommonTimerTask.isShutdown)
return;
if (busy) {
if (isFirstTime == false) {
String error = Phrase.get("ER_SUPER_WATCHDOG_INTERVAL_IS_TOO_SMALL") + ". " +
Phrase.get("TX_SEE") + " Errors.html#ER_SUPER_WATCHDOG_INTERVAL_IS_TOO_SMALL";
System.out.println(error);
}
return;
}
try {
busy = true;
Hashtable tmpChoreHashtable = new Hashtable();
Hashtable tmpNameToChoreHashtable = new Hashtable();
long now = System.currentTimeMillis();
try {
readPreference();
if (isDoer) {
runRequestedChores();
}
} catch (Exception e) {
e.printStackTrace();
}
if (choreRefreshTime == oldChoreRefreshTime)
return;
Vector allComingChores = Delegate.selectAllChores();
int choreSize = allComingChores.size();
for (int i = 0; i < choreSize; i++) {
Chore comingChore = (Chore) allComingChores.elementAt(i);
Long comingChoreId = new Long(comingChore.getId());
try {
processChore(comingChore);
} catch (Exception ex) {
if (comingChore.status != Chore.STATUS_ERROR) {
comingChore.status = Chore.STATUS_ERROR;
Delegate.updateChore(comingChore, now);
}
String error = Chore.TASK_ERROR + ChoreText.getChoreBrief(comingChore) +
LoggingConstants.COMMON_DELIMITER + Kit.getExceptionStackTrace(ex);
try {
Delegate.writeLogMessageToDatabaseForWatchdog(error);
} catch (LoggingTimeoutException lte) {
Delegate.sendAlarmEmailForChoreError(comingChore.alarmEmail, comingChore.name,
Phrase.get("ER_LOGGING_TIMEOUT"));
}
}
tmpChoreHashtable.put(comingChoreId, comingChore);
tmpNameToChoreHashtable.put(comingChore.name, comingChore);
}
isStartUp = false;
Registry.choreHashtable = tmpChoreHashtable;
Registry.nameToChoreHashtable = tmpNameToChoreHashtable;
Registry.cleanupDeleted();
if (WatchdogProperties.getIsDebugging())
System.out.println(new Date().toString() + "> " + Registry.toDebugInfo() + "\n");
} catch (Exception ex) {
try {
String subject = "SuperWatchdog" + " " + Phrase.get("ER_DATABASE");
String msg = subject + "\n" + ex.getMessage();
Delegate.sendAlertEmail(subject, msg);
} catch (Exception exception) {
exception.printStackTrace();
}
throw ex;
} finally {
busy = false;
isFirstTime = false;
}
}
static void readPreference() throws Exception {
if (preferencesRefreshTime == oldPreferencesRefreshTime)
return;
Properties properties = Delegate.selectWatchdogPreference();
interval = Integer.parseInt(properties.getProperty("interval"));
extraIntervalForGateway = Integer.parseInt(properties.getProperty("extraIntervalForGateway"));
logIncludesResult = new Boolean(properties.getProperty("logIncludesResult")).booleanValue();
workingChoreDisplayMax = Integer.parseInt(properties.getProperty("workingChoreDisplayMax"));
workingChoreRetireTime = Long.parseLong(properties.getProperty("workingChoreRetireTime"));
queryTimeout = Integer.parseInt(properties.getProperty("queryTimeout"));
logAgentTimeout = Integer.parseInt(properties.getProperty("logAgentTimeout"));
minimumDuration = Integer.parseInt(properties.getProperty("minimumDuration"));
if (workingChoreDisplayMax < WORKING_TASK_DISPLAY_MIN)
workingChoreDisplayMax = WORKING_TASK_DISPLAY_MIN;
workingChoreRetireTime = System.currentTimeMillis() - workingChoreRetireTime;
}
static boolean readRefreshTime() throws Exception {
if (CommonTimerTask.isShutdown)
return false;
try {
long tmpChoreRefreshTime = choreRefreshTime;
long tmpWorkingChoreRefreshTime = workingChoreRefreshTime;
long tmpPreferencesRefreshTime = preferencesRefreshTime;
long tmpCandidateRefreshTime = candidateRefreshTime;
String refreshTimeString = Delegate.getRefreshTime(doertalkerName);
StringTokenizer st = new StringTokenizer(refreshTimeString, " ");
choreRefreshTime = Long.parseLong(st.nextToken());
workingChoreRefreshTime = Long.parseLong(st.nextToken());
preferencesRefreshTime = Long.parseLong(st.nextToken());
candidateRefreshTime = Long.parseLong(st.nextToken());
oldChoreRefreshTime = tmpChoreRefreshTime;
oldWorkingChoreRefreshTime = tmpWorkingChoreRefreshTime;
oldPreferencesRefreshTime = tmpPreferencesRefreshTime;
oldCandidateRefreshTime = tmpCandidateRefreshTime;
heartbeatMiss = -1;
if (choreRefreshTime > oldChoreRefreshTime ||
workingChoreRefreshTime > oldWorkingChoreRefreshTime)
return true;
else
return false;
} catch (Exception ex) {
heartbeatMiss++;
if (heartbeatMiss == maxHeartbeatMiss) {
try {
String subject = "SuperWatchdog" + " " + Phrase.get("ER_DATABASE");
String msg = subject + "\n" + ex.getMessage();
Delegate.sendAlertEmail(subject, msg);
} catch (Exception exception) {
exception.printStackTrace();
}
throw ex;
} else
return false;
}
}
static void readWorkingChores() throws Exception {
if (CommonTimerTask.isShutdown)
return;
if (workingChoreRefreshTime == oldWorkingChoreRefreshTime)
return;
if (busy)
return;
try {
busy = true;
try {
new DatabaseMaintenance().deleteOldWorking(workingChoreRetireTime);
} catch (Exception e) {
e.printStackTrace();
}
Registry.workingChoreVector =
Delegate.selectAllWorkingChores(displayStartTimeForWorkingChore,
Long.MAX_VALUE, workingChoreDisplayMax + 1);
int workingChoreSize = Registry.workingChoreVector.size();
if (workingChoreSize > workingChoreDisplayMax) {
WorkingChore wt = (WorkingChore)
Registry.workingChoreVector.elementAt(workingChoreDisplayMax - 1);
displayStartTimeForWorkingChore = wt.modifiedAt;
}
} finally {
busy = false;
}
}
static void runRequestedChores() throws Exception {
if (candidateRefreshTime == oldCandidateRefreshTime)
return;
Vector allCandidateChores = Delegate.selectAllCandidateChores();
for (int i = 0; i < allCandidateChores.size(); i++) {
try {
CandidateChore candidateChore =
(CandidateChore) allCandidateChores.elementAt(i);
Chore originalChore = Delegate.selectChore(candidateChore.name);
if (originalChore == null) {
Delegate.deleteCandidateChore(candidateChore.id);
String message = Phrase.get("ER_CANNOT_FIND_TASK") + ": " + candidateChore.name +
". " + Phrase.get("TX_REQUEST_IS_DELETED");
Delegate.sendAlarmEmailForChoreError(originalChore.alarmEmail,
candidateChore.name, message);
continue;
}
if (WatchTimerTask.isDesired(originalChore)) {
if (Delegate.deleteCandidateChore(candidateChore.id) == 1) {
executeNow(originalChore.name);
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
public static void stopChore(Chore chore) throws Exception {
Long choreId = new Long(chore.getId());
WatchTimerTask watchTimerTask = (WatchTimerTask) Registry.watchTimerTaskHashtable.get(choreId);
if (watchTimerTask != null) {
watchTimerTask.cancel();
Registry.watchTimerTaskHashtable.remove(choreId);
Timer timer = (Timer) Registry.timerTaskToTimerHashtable.get(watchTimerTask);
if (timer != null) {
timer.cancel();
Registry.timerTaskToTimerHashtable.remove(watchTimerTask);
}
}
}
public static void updateTimerTask(Chore chore) throws Exception {
if (isDoer == false)
return;
stopChore(chore);
WatchTimerTask.createWatchTimerTask(chore);
}
}