long startedTryingPeer = System.currentTimeMillis();
boolean waitedForLoadManagement = false;
boolean retriedForLoadManagement = false;
SlotWaiter waiter = null;
PeerNode lastNext = null;
RequestLikelyAcceptedState lastExpectedAcceptState = null;
RequestLikelyAcceptedState expectedAcceptState = null;
loadWaiterLoop:
while(true) {
boolean canRerouteWhileWaiting = true;
synchronized(this) {
if(rejectedLoops > MAX_REJECTED_LOOPS)
canRerouteWhileWaiting = false;
}
if(logMINOR) Logger.minor(this, "Going around loop");
long now = System.currentTimeMillis();
if(next == null) {
dontDecrementHTLThisTime = true;
routeRequests();
return;
}
expectedAcceptState =
next.outputLoadTracker(realTimeFlag).tryRouteTo(origTag, RequestLikelyAcceptedState.LIKELY, false);
if(expectedAcceptState == RequestLikelyAcceptedState.UNKNOWN) {
// No stats, old style, just go for it.
// This can happen both when talking to an old node and when we've just connected, but should not be the case for long enough to be a problem.
if(logMINOR) Logger.minor(this, "No load stats for "+next);
} else {
if(expectedAcceptState != null) {
if(logMINOR)
Logger.minor(this, "Predicted accept state for "+this+" : "+expectedAcceptState+" realtime="+realTimeFlag);
// FIXME sanity check based on new data. Backoff if not plausible.
// FIXME recalculate with broader check, allow a few percent etc.
if(lastNext == next && lastExpectedAcceptState == RequestLikelyAcceptedState.GUARANTEED &&
(expectedAcceptState == RequestLikelyAcceptedState.GUARANTEED)) {
// We routed it, thinking it was GUARANTEED.
// It was rejected, and as far as we know it's still GUARANTEED. :(
Logger.warning(this, "Rejected overload (last time) yet expected state was "+lastExpectedAcceptState+" is now "+expectedAcceptState+" from "+next.shortToString()+" ("+next.getVersionNumber()+")");
next.rejectedGuaranteed(realTimeFlag);
next.noLongerRoutingTo(origTag, false);
expectedAcceptState = null;
dontDecrementHTLThisTime = true;
routeRequests();
return;
}
}
int canWaitFor = 1;
if(expectedAcceptState == null) {
if(logMINOR)
Logger.minor(this, "Cannot send to "+next+" realtime="+realTimeFlag);
waitedForLoadManagement = true;
if(waiter == null)
waiter = PeerNode.createSlotWaiter(origTag, type, false, realTimeFlag, source);
if(next != null) {
if(!waiter.addWaitingFor(next)) {
dontDecrementHTLThisTime = true;
routeRequests();
return;
// Will be rerouted.
// This is essential to avoid adding the same bogus node again and again.
// This is only an issue with next. Hence the other places we route explicitly so there is no risk as they won't return the same node repeatedly after it is no longer routable.
}
}
if(next.isLowCapacity(realTimeFlag)) {
if(waiter.waitingForCount() == 1 // if not, already accepted
&& canRerouteWhileWaiting) {
canWaitFor++;
// Wait for another one if the first is low capacity.
// Nodes we were waiting for that then became backed off will have been removed from the list.
HashSet<PeerNode> exclude = waiter.waitingForList();
exclude.addAll(nodesRoutedTo);
PeerNode alsoWaitFor = closerPeer(exclude, now, true);
if(alsoWaitFor != null) {
waiter.addWaitingFor(alsoWaitFor);
// We do not need to check the return value here.
// We will not reuse alsoWaitFor if it is disconnected etc.
if(logMINOR) Logger.minor(this, "Waiting for "+next+" and "+alsoWaitFor+" on "+waiter+" because realtime");
PeerNode matched;
try {
matched = waiter.waitForAny(0, false);
} catch (SlotWaiterFailedException e) {
if(logMINOR) Logger.minor(this, "Rerouting as slot waiter failed...");
continue;
}
if(matched != null) {
expectedAcceptState = waiter.getAcceptedState();
next = matched;
if(logMINOR) Logger.minor(this, "Matched "+matched+" with "+expectedAcceptState);
}
}
}
}
}
if(realTimeFlag) canWaitFor++;
// Skip it and go straight to rerouting if no next, as above.
if(expectedAcceptState == null && waiter.waitingForCount() <= canWaitFor
&& canRerouteWhileWaiting) {
// Wait for another one if realtime.
// Nodes we were waiting for that then became backed off will have been removed from the list.
HashSet<PeerNode> exclude = waiter.waitingForList();
exclude.addAll(nodesRoutedTo);
PeerNode alsoWaitFor = closerPeer(exclude, now, true);
if(alsoWaitFor != null) {
waiter.addWaitingFor(alsoWaitFor);
// We do not need to check the return value here.
// We will not reuse alsoWaitFor if it is disconnected etc.
if(logMINOR) Logger.minor(this, "Waiting for "+next+" and "+alsoWaitFor+" on "+waiter+" because realtime");
PeerNode matched;
try {
matched = waiter.waitForAny(0, false);
} catch (SlotWaiterFailedException e) {
if(logMINOR) Logger.minor(this, "Rerouting as slot waiter failed...");
continue;
}
if(matched != null) {
expectedAcceptState = waiter.getAcceptedState();
next = matched;
if(logMINOR) Logger.minor(this, "Matched "+matched+" with "+expectedAcceptState);
}
}
}
if(addedExtraNode) canWaitFor++;
// Skip it and go straight to rerouting if no next, as above.
if(expectedAcceptState == null && waiter.waitingForCount() <= canWaitFor
&& canRerouteWhileWaiting) {
// Wait for another one if realtime.
// Nodes we were waiting for that then became backed off will have been removed from the list.
HashSet<PeerNode> exclude = waiter.waitingForList();
exclude.addAll(nodesRoutedTo);
PeerNode alsoWaitFor = closerPeer(exclude, now, true);
if(alsoWaitFor != null) {
waiter.addWaitingFor(alsoWaitFor);
// We do not need to check the return value here.
// We will not reuse alsoWaitFor if it is disconnected etc.
if(logMINOR) Logger.minor(this, "Waiting for "+next+" and "+alsoWaitFor+" on "+waiter+" because realtime");
PeerNode matched;
try {
matched = waiter.waitForAny(0, false);
} catch (SlotWaiterFailedException e) {
// Reroute.
continue;
}
if(matched != null) {
expectedAcceptState = waiter.getAcceptedState();
next = matched;
}
}
}
if(expectedAcceptState == null) {
long maxWait = getLongSlotWaiterTimeout();
// After waitForAny() it will be null, it is all cleared.
if(!addedExtraNode) {
// Can add another one if it's taking ages.
// However after adding it once, we will wait for as long as it takes.
maxWait = getShortSlotWaiterTimeout();
}
HashSet<PeerNode> waitedFor = waiter.waitingForList();
PeerNode waited;
// FIXME figure out a way to wake-up mid-wait if origTag.hasSourceRestarted().
try {
waited = waiter.waitForAny(maxWait, addedExtraNode);
} catch (SlotWaiterFailedException e) {
// Failed. Reroute.
continue;
}
if(waited == null) {
// Timed out, or not waiting for anything, not failed.
if(logMINOR) Logger.minor(this, "Timed out waiting for a peer to accept "+this+" on "+waiter);
if(addedExtraNode) {
// Backtrack
timedOutWhileWaiting(getLoad(waitedFor));
// Above is responsible for termination or rerouting.
return;
} else {
addedExtraNode = true;
continue;
}
} else {
next = waited;
expectedAcceptState = waiter.getAcceptedState();
long endTime = System.currentTimeMillis();
if(logMINOR) Logger.minor(this, "Sending to "+next+ " after waited for "+TimeUtil.formatTime(endTime-startTime)+" realtime="+realTimeFlag);
expectedAcceptState = waiter.getAcceptedState();
}
}
assert(expectedAcceptState != null);
lastExpectedAcceptState = expectedAcceptState;