hostFailures.put(entry.getKey(), new UpdateFailedException(e));
}
}
if (hostFailures.size() == 0) {
log.debugf("%s servers affected by update %s", servers.size(), i);
result.add(new DomainUpdateApplierResponse(servers));
}
else {
log.debugf("%s server managers failed on update %s", hostFailures.size(), i);
result.add(new DomainUpdateApplierResponse(hostFailures));
ok = false;
// No point processing other updates, as we are going to roll them back.
// Act as if we did the whole thing one update at a time and this
// failure stopped us doing the rest
break;
}
}
if (!ok) {
// Some server manager failed, so we gotta roll 'em all back
log.warn("One or more updates failed on some server managers; rolling back");
// Apply compensating updates to fix our local model
for (int i = 0; i < rollbacks.size(); i++) {
AbstractDomainModelUpdate<?> rollback = rollbacks.get(i);
try {
domainModel.update(rollback);
}
catch (UpdateFailedException e) {
// TODO uh oh. Reload from the file?
}
}
// List of servers we fail to successfully roll back
Set<String> outOfSync = new HashSet<String>();
Map<String, Future<Boolean>> rollbackFutures = new HashMap<String, Future<Boolean>>(futures.size());
for (Map.Entry<String, Future<List<ModelUpdateResponse<List<ServerIdentity>>>>> entry : futures.entrySet()) {
try {
// For this host figure out how many updates need to be rolled back
List<ModelUpdateResponse<List<ServerIdentity>>> rspList = entry.getValue().get();
int idx = rspList.size() - 1;
if (idx >= 0 && !rspList.get(idx).isSuccess()) {
idx--; // !isSuccess one shouldn't have affected model state so no rollback of it
}
if (idx < 0) {
// This host didn't apply anything
continue;
}
// Set up the rollback list
final List<AbstractDomainModelUpdate<?>> serverManagerRollbacks =
(idx == rollbacks.size() -1) ? rollbacks : new ArrayList<AbstractDomainModelUpdate<?>>(idx + 1);
if (serverManagerRollbacks != rollbacks) {
// Rollbacks are in reverse order from updates. We take
// the last X=idx items from the rollback list since
// those correspond to the updates that didn't fail and need rollback
for (int j = rollbacks.size() - 1 - idx; j < rollbacks.size(); j++) {
serverManagerRollbacks.add(rollbacks.get(j));
}
}
// Tell the host to roll back
final ServerManagerClient client = clients.get(entry.getKey());
Callable<Boolean> callable = new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
List<ModelUpdateResponse<List<ServerIdentity>>> rsp = client.updateDomainModel(serverManagerRollbacks);
return Boolean.valueOf(rsp.size() == serverManagerRollbacks.size() && rsp.get(rsp.size() - 1).isSuccess());
}
};
rollbackFutures.put(entry.getKey(), scheduledExecutorService.getValue().submit(callable));
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
outOfSync.add(entry.getKey());
} catch (ExecutionException e) {
outOfSync.add(entry.getKey());
}
}
// Wait until rollbacks complete
for (Map.Entry<String, Future<Boolean>> entry : rollbackFutures.entrySet()) {
try {
if (!entry.getValue().get()) {
outOfSync.add(entry.getKey());
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
outOfSync.add(entry.getKey());
} catch (ExecutionException e) {
outOfSync.add(entry.getKey());
}
}
for (String host : outOfSync) {
// Rollback failed; need to push the whole model
ServerManagerClient client = clients.get(host);
client.updateDomainModel(domainModel);
}
// Update the result list to record the rollbacks
for (int i = 0; i < result.size(); i++) {
DomainUpdateApplierResponse rsp = result.get(i);
if (rsp.getHostFailures().size() < 0) {
result.set(i, new DomainUpdateApplierResponse(false));
}
}
}
return result;