boolean delegateDisabled = false;
Delegate delegate = null;
// R E T R Y
// Do retry first if this an Aggregate Controller
if (!isEndpointTheClient && aController instanceof AggregateAnalysisEngineController) {
Endpoint endpoint = null;
if (anErrorContext.get(AsynchAEMessage.Endpoint) != null) {
endpoint = (Endpoint) anErrorContext.get(AsynchAEMessage.Endpoint);
key = ((AggregateAnalysisEngineController) aController).lookUpDelegateKey(endpoint
.getEndpoint());
delegate = ((AggregateAnalysisEngineController) aController).lookupDelegate(key);
}
threshold = super.getThreshold(endpoint, delegateMap, aController);
if (endpoint != null) {
// key =
// ((AggregateAnalysisEngineController)aController).lookUpDelegateKey(endpoint.getEndpoint());
delegateDisabled = ((AggregateAnalysisEngineController) aController)
.isDelegateDisabled(key);
if (threshold != null && threshold.getMaxRetries() > 0 && !delegateDisabled) {
// If max retry count is not reached, send the last command again and return true
if (super.retryLastCommand(AsynchAEMessage.Process, endpoint, aController, key,
threshold, anErrorContext)) {
if (UIMAFramework.getLogger(CLASS_NAME).isLoggable(Level.FINE)) {
UIMAFramework.getLogger(CLASS_NAME).logrb(Level.FINE, getClass().getName(),
"handleError", UIMAEE_Constants.JMS_LOG_RESOURCE_BUNDLE,
"UIMAEE_retry_cas__FINE",
new Object[] { aController.getComponentName(), key, casReferenceId });
}
return true; // Command has been retried. Done here.
}
} else if (threshold == null) {
if (UIMAFramework.getLogger(CLASS_NAME).isLoggable(Level.CONFIG)) {
UIMAFramework.getLogger(CLASS_NAME).logrb(Level.CONFIG, getClass().getName(),
"handleError", UIMAEE_Constants.JMS_LOG_RESOURCE_BUNDLE,
"UIMAEE_no_threshold_for_endpoint__CONFIG",
new Object[] { aController.getComponentName(), "Process", key });
}
}
if (delegate != null) {
// Received reply from the delegate. Remove the CAS from the
// delegate's list of CASes pending reply
// Delegate delegate =
// ((AggregateAnalysisEngineController)aController).lookupDelegate(key);
delegate.removeCasFromOutstandingList(casReferenceId);
}
} else {
if (UIMAFramework.getLogger(CLASS_NAME).isLoggable(Level.INFO)) {
UIMAFramework.getLogger(CLASS_NAME).logrb(Level.INFO, getClass().getName(),
"handleError", UIMAEE_Constants.JMS_LOG_RESOURCE_BUNDLE,
"UIMAEE_no_endpoint_provided__INFO",
new Object[] { aController.getComponentName() });
}
}
} else {
if (delegateMap != null && delegateMap.containsKey(key)) {
threshold = (Threshold) delegateMap.get(key);
}
}
if (key != null && key.trim().length() > 0) {
// Retries either exceeded or not configured for retry
if (UIMAFramework.getLogger(CLASS_NAME).isLoggable(Level.FINE)) {
UIMAFramework.getLogger(CLASS_NAME).logrb(Level.FINE, getClass().getName(), "handleError",
UIMAEE_Constants.JMS_LOG_RESOURCE_BUNDLE, "UIMAEE_cas_retries_exceeded__FINE",
new Object[] { aController.getComponentName(), key, casReferenceId });
}
}
boolean disabledDueToExceededThreshold = false;
// Dont increment errors for destinations that are clients of this service.
if (key != null && !aController.isStopped() && (isRequest || !isEndpointTheClient)) {
synchronized (monitor) {
// Dont increment errors for delegates that have been already disabled
if (!delegateDisabled) {
// Process Error Count is only incremented after retries are done.
super.incrementStatistic(aController.getMonitor(), key, Monitor.ProcessErrorCount);
super.incrementStatistic(aController.getMonitor(), key, Monitor.TotalProcessErrorCount);
aController.getServiceErrors().incrementProcessErrors();
if (aController instanceof AggregateAnalysisEngineController
&& anErrorContext.get(AsynchAEMessage.Endpoint) != null) {
Endpoint endpoint = (Endpoint) anErrorContext.get(AsynchAEMessage.Endpoint);
if (endpoint.isRemote()) {
ServiceErrors serviceErrs = ((AggregateAnalysisEngineController) aController)
.getDelegateServiceErrors(key);
if (serviceErrs != null) {
serviceErrs.incrementProcessErrors();
}
}
}
/***
* if (threshold != null && threshold.getThreshold() > 0 &&
* super.exceedsThresholdWithinWindow(aController.getMonitor(), Monitor.ProcessErrorCount,
* key, threshold) )
*/
long procCount = aController.getMonitor().getLongNumericStatistic(key,
Monitor.ProcessCount).getValue();
if (threshold != null && threshold.exceededWindow(procCount)) {
if (UIMAFramework.getLogger(CLASS_NAME).isLoggable(Level.INFO)) {
UIMAFramework.getLogger(CLASS_NAME).logrb(
Level.INFO,
getClass().getName(),
"handleError",
UIMAEE_Constants.JMS_LOG_RESOURCE_BUNDLE,
"UIMAEE_process_cas_exceeded_threshold__INFO",
new Object[] { aController.getComponentName(), key, casReferenceId,
threshold.getThreshold(), threshold.getAction() });
}
// Add new property to skip handling of CASes in pending lists. Those CASes
// will be handled later in this method, once we complete processing of the CAS
// that caused the exception currently being processed. During handling of the
// CASes in pending state, this error handler is called for each CAS to force
// its timeout.
disabledDueToExceededThreshold = ErrorHandler.DISABLE.equalsIgnoreCase(threshold
.getAction());
if (disabledDueToExceededThreshold) {
delegateKey = key;
anErrorContext.add(AsynchAEMessage.SkipPendingLists, "true");
}
if (ErrorHandler.TERMINATE.equalsIgnoreCase(threshold.getAction())) {
anErrorContext.add(ErrorContext.THROWABLE_ERROR, t);
if (casReferenceId != null) {
try {
CasStateEntry casStateEntry = aController.getLocalCache().lookupEntry(
casReferenceId);
if (casStateEntry != null && casStateEntry.isSubordinate()) {
CasStateEntry parenCasStateEntry = aController.getLocalCache()
.getTopCasAncestor(casReferenceId);
// Replace Cas Id with the parent Cas Id
anErrorContext.remove(AsynchAEMessage.CasReference);
anErrorContext.add(AsynchAEMessage.CasReference, parenCasStateEntry
.getCasReferenceId());
}
} catch (Exception e) {
}
}
}
aController.takeAction(threshold.getAction(), key, anErrorContext);
if (ErrorHandler.CONTINUE.equalsIgnoreCase(threshold.getAction())) {
// The following is only true if the Action=Continue and the FlowController is configured
// to continue on Exception in a delegate with a given key. The exception is provided
// in the anErrorContext. The controller, while handling exception in takeAction(), will set
// ErrorContext.ERROR_HANDLED property iff the FlowController says to continue.
if (anErrorContext.containsKey(ErrorContext.ERROR_HANDLED) && (Boolean)anErrorContext.get(ErrorContext.ERROR_HANDLED) == true )
// The FlowController indicated to continue. The process() method was already called. No
// need to do anything else. At this point the error has been handled.
return true;
}
}
} else {
if (UIMAFramework.getLogger(CLASS_NAME).isLoggable(Level.INFO)) {
UIMAFramework.getLogger(CLASS_NAME).logrb(Level.INFO, getClass().getName(),
"handleError", UIMAEE_Constants.JMS_LOG_RESOURCE_BUNDLE,
"UIMAEE_delegate_already_disabled__INFO",
new Object[] { aController.getComponentName(), key, casReferenceId });
}
}
}
} else {
Endpoint endpt = (Endpoint) anErrorContext.get(AsynchAEMessage.Endpoint);
if (endpt != null) {
if (UIMAFramework.getLogger(CLASS_NAME).isLoggable(Level.INFO)) {
UIMAFramework.getLogger(CLASS_NAME).logrb(
Level.INFO,
getClass().getName(),
"handleError",
UIMAEE_Constants.JMS_LOG_RESOURCE_BUNDLE,
"UIMAEE_process_exception__INFO",
new Object[] { aController.getComponentName(), endpt.getEndpoint(),
casReferenceId });
}
}
}
int totalNumberOfParallelDelegatesProcessingCas = 1; // default
CacheEntry cacheEntry = null;
CasStateEntry casStateEntry = null;
try {
casStateEntry = aController.getLocalCache().lookupEntry(casReferenceId);
cacheEntry = aController.getInProcessCache().getCacheEntryForCAS(casReferenceId);
if (cacheEntry != null) {
totalNumberOfParallelDelegatesProcessingCas = casStateEntry.getNumberOfParallelDelegates();
}
} catch (Exception e) {
if (UIMAFramework.getLogger(CLASS_NAME).isLoggable(Level.INFO)) {
UIMAFramework.getLogger(CLASS_NAME).logrb(Level.INFO, CLASS_NAME.getName(),
"handleError", UIMAEE_Constants.JMS_LOG_RESOURCE_BUNDLE,
"UIMAEE_parent_cas_notin_cache__INFO", new Object[] { aController.getComponentName(), casReferenceId });
}
}
// Determine where to send the message
Endpoint endpoint = getDestination(aController, anErrorContext);
// If the error happened during a parallel step, treat the exception as response from the
// delegate
// When all responses from delegates are accounted for we allow the CAS to move on to the next
// step in the flow. Dont increment parallel delegate response count if a delegate was just
// disabled above. The count has been already incremented in handleAction() method of the
// AnalysisEngineController.
if ( casStateEntry != null
&& totalNumberOfParallelDelegatesProcessingCas > 1
&& (casStateEntry.howManyDelegatesResponded() < totalNumberOfParallelDelegatesProcessingCas)) {
casStateEntry.incrementHowManyDelegatesResponded();
}
if (aController instanceof AggregateAnalysisEngineController && t instanceof Exception) {
boolean flowControllerContinueFlag = false;
// if the deployment descriptor says no retries, dont care what the Flow Controller says
if (threshold != null && threshold.getContinueOnRetryFailure()) {
try {
// Consult Flow Controller to determine if it is ok to continue despite the error
flowControllerContinueFlag = ((AggregateAnalysisEngineController) aController)
.continueOnError(casReferenceId, key, (Exception) t);
} catch (Exception exc) {
if (UIMAFramework.getLogger(CLASS_NAME).isLoggable(Level.WARNING)) {
if ( aController != null ) {
UIMAFramework.getLogger(CLASS_NAME).logrb(Level.WARNING, CLASS_NAME.getName(),
"handleError", UIMAEE_Constants.JMS_LOG_RESOURCE_BUNDLE,
"UIMAEE_service_exception_WARNING", aController.getComponentName());
}
UIMAFramework.getLogger(CLASS_NAME).logrb(Level.WARNING, getClass().getName(),
"handleError", UIMAEE_Constants.JMS_LOG_RESOURCE_BUNDLE,
"UIMAEE_exception__WARNING", exc);
}
}
}
// By default return exception to the client. The exception will not be returned if the CAS is
// a subordinate and the flow controller is *not* configured to continue with bad CAS. In such
// case, the code below will mark the parent CAS as failed. When all child CASes of the parent
// CAS are accounted for, it will be returned to the client with an exception.
boolean doSendReplyToClient = true;
// Check if the caller has already decremented number of subordinates. This property is only
// set in the Aggregate's finalStep() method before the CAS is sent back to the client. If
// there was a problem sending the CAS to the client, we dont want to update the counter
// again. If an exception is reported elsewhere ( not in finalStep()), the default action is
// to decrement the number of subordinates associated with the parent CAS.
if (!flowControllerContinueFlag
&& !anErrorContext.containsKey(AsynchAEMessage.SkipSubordinateCountUpdate)) {
doSendReplyToClient = false;
// Check if the CAS is a subordinate (has parent CAS).
if (casStateEntry != null && casStateEntry.isSubordinate()) {
String parentCasReferenceId = casStateEntry.getInputCasReferenceId();
if (parentCasReferenceId != null) {
try {
CacheEntry parentCasCacheEntry = aController.getInProcessCache().getCacheEntryForCAS(
parentCasReferenceId);
parentCasStateEntry = aController.getLocalCache().lookupEntry(parentCasReferenceId);
String cmEndpointName = cacheEntry.getCasProducerKey();
String cmKey = ((AggregateAnalysisEngineController) aController)
.lookUpDelegateKey(cmEndpointName);
if (cmKey == null) {
cmKey = cacheEntry.getCasProducerKey();
}
Delegate delegateCM = ((AggregateAnalysisEngineController) aController)
.lookupDelegate(cmKey);
// The aggregate will return the input CAS when all child CASes are accounted for
synchronized (parentCasStateEntry) {
if (!parentCasStateEntry.isFailed()) {
CasStateEntry predecessorCas = parentCasStateEntry;
// Processing a failure of the child. Mark the parent CAS
// as failed. All child CASes will be dropped upon return
// from delegates. When all child CASes are dropped the
// aggregate will return an exception to the client containing
// the parent CAS id.
parentCasStateEntry.setFailed();
while (predecessorCas != null && predecessorCas.isSubordinate()) {
predecessorCas = aController.getLocalCache().lookupEntry(
predecessorCas.getInputCasReferenceId());
predecessorCas.setFailed();
}
predecessorCas.addThrowable(t);
// Stop Cas Multiplier
((AggregateAnalysisEngineController) aController).stopCasMultiplier(delegateCM,
parentCasCacheEntry.getCasReferenceId());
}
// Add the exception to the list of exceptions maintained by the parent CAS
parentCasStateEntry.addThrowable(t);
casStateEntry.setReplyReceived();
// Mark this CAS as failed
casStateEntry.setFailed();
if (parentCasStateEntry.getSubordinateCasInPlayCount() == 0
&& parentCasStateEntry.isPendingReply()) {
aController.process(parentCasCacheEntry.getCas(), parentCasCacheEntry
.getCasReferenceId());
} else {
aController.process(null, casStateEntry.getCasReferenceId());
}
}
} catch (Exception ex) {
// Input CAS doesnt exist. Nothing to update, move on
}
}
} else if (casStateEntry != null) { // input CAS
casStateEntry.setFailed();
casStateEntry.addThrowable(t);
aController.process(null, casStateEntry.getCasReferenceId());
}
return true;
}
if (threshold != null && flowControllerContinueFlag) {
// The Exception has been almost fully handled. Check if the delegate was disabled above.
// If it was, we need to force timeout on all CASes in pending state associated with that
// delegate.
if (disabledDueToExceededThreshold && delegateKey != null) {
aController.forceTimeoutOnPendingCases(delegateKey);
}
if (totalNumberOfParallelDelegatesProcessingCas == 1
|| (casStateEntry.howManyDelegatesResponded() == totalNumberOfParallelDelegatesProcessingCas)) {
aController.process(aController.getInProcessCache().getCasByReference(casReferenceId),
casReferenceId);
}
// Dont send request to release the CAS to remote CM. This will happen in the final step. We
// are continuing
// despite the error here.
return true;
} else if (doSendReplyToClient) {
try {
// Send exception to the client if the exception happened while processing input CAS
// Child CASes that cause exceptions will be dropped, their parent CAS will be marked
// as failed and it will be returned back to the client in the final step once all child
// CASes are accounted for and dropped.
if (casStateEntry != null && !casStateEntry.isSubordinate()
&& deliverExceptionToClient(t)) {
sendExceptionToClient(t, casReferenceId, endpoint, aController);
}
} catch (Exception e) {
if (UIMAFramework.getLogger(CLASS_NAME).isLoggable(Level.WARNING)) {
if ( aController != null ) {
UIMAFramework.getLogger(CLASS_NAME).logrb(Level.WARNING, CLASS_NAME.getName(),
"handleError", UIMAEE_Constants.JMS_LOG_RESOURCE_BUNDLE,
"UIMAEE_service_exception_WARNING", aController.getComponentName());
}
UIMAFramework.getLogger(CLASS_NAME).logrb(Level.WARNING, getClass().getName(),
"handleError", UIMAEE_Constants.JMS_LOG_RESOURCE_BUNDLE,
"UIMAEE_exception__WARNING", e);
}
}
}
// Now check if the CAS origin is a remote CAS Multiplier. If so, send a request to release
// the CAS. Remote
// CAS Multipliers must be "gated" to prevent flooding the Aggregate queue with too many
// CASes. There is
// an explicit protocol between an Aggregate AS and its CM. The Aggregate AS sends a request
// to free a CAS
// in the CM whenever the Aggregate has capacity to process more CASes. Here we are recovering
// from an
// an error but we still need to send a request to free the CAS in the remote CM to prevent a
// hang in the CM
try {
// First check if the current controller is the one that first produced the CAS
if (cacheEntry != null
&& aController.getName().equalsIgnoreCase(cacheEntry.getCasProducerAggregateName())) {
// Fetch the key of the Cas Multiplier
String casProducerKey = cacheEntry.getCasProducerKey();
if (casProducerKey != null) {
// Create an endpoint object from the key. This object will be cloned from the Endpoint
// object
// defined in the spring configuration file.
Endpoint cmEndpoint = ((AggregateAnalysisEngineController) aController).lookUpEndpoint(
casProducerKey, true);
// CAS reference id will be different if the CAS originated from a remote Cas
// Multiplier.
// String cmCasReferenceId = cacheEntry.getRemoteCMCasReferenceId();
// If the Cas Multiplier is remote send a request to free a CAS with a given cas id
if (cmEndpoint != null && cmEndpoint.isCasMultiplier() && cmEndpoint.isRemote()) {
cmEndpoint.setReplyEndpoint(true);
cmEndpoint.setIsCasMultiplier(true);
cmEndpoint.setFreeCasEndpoint(true);
aController.getOutputChannel().sendRequest(AsynchAEMessage.ReleaseCAS,
cacheEntry.getCasReferenceId(), cmEndpoint);
}
}
}