}
public void doTakeover() throws Exception {
try {
ClusterBroadcast mbus = Globals.getClusterBroadcast();
while (mbus == null) {
logger.log(Logger.INFO, BrokerResources.I_CLUSTER_WAIT_PROTOCOLINIT);
try {
Thread.sleep(500);
} catch (InterruptedException e) {};
mbus = Globals.getClusterBroadcast();
}
Iterator itr = downBkrs.iterator();
while (itr.hasNext()) {
downBroker dbroker = (downBroker)itr.next();
HAClusteredBroker cb = dbroker.cb;
logger.log(Logger.INFO, BrokerResources.I_START_TAKEOVER,
cb.getBrokerName() );
boolean takeoverComplete = false;
TakingoverTracker tracker = new TakingoverTracker(
cb.getBrokerName(),
Thread.currentThread());
tracker.setBrokerSessionUID(dbroker.brokerSession);
tracker.setDownStoreSessionUID(dbroker.storeSession);
tracker.setLastHeartbeat(dbroker.lastts);
takingoverTargets.add(tracker);
try {
mbus.preTakeover(cb.getBrokerName(), tracker.getDownStoreSessionUID(),
((BrokerMQAddress)cb.getBrokerURL()).getHost().getHostAddress(),
tracker.getBrokerSessionUID());
TakeoverStoreInfo info = cb.takeover(force, tracker);
tracker.setStage_BEFORE_PROCESSING();
takeoverComplete = true;
logger.log(Logger.INFO, BrokerResources.I_TAKEOVER_OK,
tracker.getBrokerID());
Map msgs = info.getMessageMap();
List txn = info.getTransactionList();
List remoteTxn = info.getRemoteTransactionList();
logger.log(Logger.INFO,
BrokerResources.I_TAKEOVER_TXNS,
tracker.getBrokerID(),
String.valueOf(txn == null ? 0 : txn.size()));
logger.log(Logger.INFO,
BrokerResources.I_TAKEOVER_REMOTE_TXNS,
tracker.getBrokerID(),
String.valueOf(remoteTxn == null ? 0 : remoteTxn.size()));
Destination.remoteCheckTakeoverMsgs(msgs, tracker.getBrokerID());
// handle takeover logic
// process rolling back transactions
List openTxns = new ArrayList();
Iterator titr = txn.iterator();
while (titr.hasNext()) {
boolean rollbackTxn = false;
TransactionUID tid = (TransactionUID) titr.next();
TransactionState ts = Globals.getStore().getTransactionState(tid);
if (ts == null) {
titr.remove();
continue;
}
AutoRollbackType type = ts.getType();
int state = ts.getState();
// OK .. first handle loading messages because
// that means we have all the refs in order
// ok first handle database changes for ANY
// committed or rolledback message
if (state == TransactionState.ROLLEDBACK) {
logger.log(Logger.INFO, "XXX - DEBUG Rolling back "
+ " transaction " + tid);
// ok we may not be done with things yet
// add to opentxn list
openTxns.add(tid);
} else if (state == TransactionState.COMMITTED) {
logger.log(Logger.INFO, "XXX - DEBUG Committing "
+ " transaction " + tid);
// ok we may not be done with things yet
// add to opentxn list
openTxns.add(tid);
} else if (type == AutoRollbackType.ALL) {
// rollback
String args[] = {
tracker.getBrokerID(),
String.valueOf(tid.longValue()),
ts.toString(ts.getState()) };
logger.log(Logger.INFO,
BrokerResources.I_TAKEOVER_TXN_A_ROLLBACK,
args);
ts.setState(TransactionState.ROLLEDBACK);
try {
Globals.getStore().updateTransactionState(tid, ts,
Destination.PERSIST_SYNC);
} catch (IOException e) {
throw new BrokerException(null, e);
}
// ok we may not be done with things yet
// add to opentxn list
openTxns.add(tid);
} else if (ts.getType() == AutoRollbackType.NOT_PREPARED
&& ts.getState() < TransactionState.PREPARED) {
String args[] = {
tracker.getBrokerID(),
String.valueOf(tid.longValue()),
ts.toString(ts.getState()) };
logger.log(Logger.INFO,
BrokerResources.I_TAKEOVER_TXN_P_ROLLBACK,
args);
ts.setState(TransactionState.ROLLEDBACK);
try {
Globals.getStore().updateTransactionState(tid, ts,
Destination.PERSIST_SYNC);
} catch (IOException e) {
throw new BrokerException(null, e);
}
// ok we may not be done with things yet
// add to opentxn list
openTxns.add(tid);
} else {
String args[] = {
tracker.getBrokerID(),
String.valueOf(tid.longValue()),
ts.toString(ts.getState()) };
logger.log(Logger.INFO,
BrokerResources.I_TAKEOVER_TXN, args);
}
}
TransactionUID tid = null;
TransactionState ts = null, myts = null;
TransactionList translist = Globals.getTransactionList();
titr = remoteTxn.iterator();
while (titr.hasNext()) {
tid = (TransactionUID) titr.next();
try {
ts = Globals.getStore().getTransactionState(tid);
} catch (Exception e) {
logger.log(logger.WARNING,
"Unable to get transaction state "+tid+
" for takenover broker "+tracker.getBrokerID());
continue;
}
if (ts == null) continue;
try {
if (ts.getState() < TransactionState.PREPARED &&
translist.retrieveState(tid) != null) { // auto-rollback type
if (!translist.isClusterTransactionBroker(tid,
tracker.getStoreSessionUID())) {
continue;
}
myts = translist.retrieveState(tid);
if (myts == null) continue;
myts = new TransactionState(myts);
translist.updateState(tid,
myts.nextState(PacketType.ROLLBACK_TRANSACTION, null),
myts.getState(), true);
logger.log(logger.INFO,
"Remote transaction "+tid+"("+
TransactionState.toString(ts.getState())+
") from takenover broker "+tracker+" will be rolledback");
}
} catch (Exception e) {
logger.log(logger.WARNING,
"Unable to set ROLLBACK state to transaction "+tid+":"+
e.getMessage()+ " for takenover broker "+tracker.getBrokerID());
continue;
}
}
TakeoverReaper reaper = new TakeoverReaper(
tracker.getBrokerID(), openTxns);
Map m = Globals.getTransactionList().loadTakeoverTxns(txn, remoteTxn, msgs);
logger.log(Logger.INFO,
BrokerResources.I_TAKEOVER_MSGS,
tracker.getBrokerID(),
String.valueOf(msgs == null ? 0 : msgs.size()));
Destination.loadTakeoverMsgs(msgs, txn, m);
tracker.setStage_AFTER_PROCESSING();
takingoverTargets.remove(tracker);
Globals.getTransactionList().unlockTakeoverTxns(txn);
// OK rollback Open txns - we want to do this
boolean done = reaper.processTxns();
MQTimer timer = Globals.getTimer();
try {
if (!done)
timer.schedule(reaper, reaperTimeout, reaperTimeout);
} catch (IllegalStateException ex) {
logger.logStack(Logger.WARNING,
BrokerResources.E_INTERNAL_BROKER_ERROR,
"Unable to start takeover-transaction reaper", ex);
}
logger.log(logger.INFO,
BrokerResources.I_TAKEOVER_DATA_PROCESSED,
tracker.toString());
// we have done processing data, set state to
// complete
cb.setBrokerIsUp(false, tracker.getBrokerSessionUID(),
tracker.getStoreSessionUID());
cb.setStateFailoverProcessed(tracker.getStoreSessionUID());
logger.log(logger.INFO,
BrokerResources.I_TAKEOVER_COMPLETE,
tracker.toString());
itr.remove();
} catch (Exception ex) {
if ( ex instanceof TakeoverLockException ) {
BrokerState state = null;
String takeoverBy = null;
HABrokerInfo bkrInfo = ((TakeoverLockException)ex).getBrokerInfo();
if (bkrInfo == null) {
// This shouldn't happens but just in case
try {
state = cb.getState();
takeoverBy = cb.getTakeoverBroker();
} catch ( BrokerException e ) {}
} else {
state = BrokerState.getState(bkrInfo.getState());
takeoverBy = bkrInfo.getTakeoverBrokerID();
}
if ( state == BrokerState.FAILOVER_STARTED ||
state == BrokerState.FAILOVER_PENDING ||
state == BrokerState.FAILOVER_COMPLETE ) {
if (takeoverBy != null) {
logger.log(Logger.INFO,
BrokerResources.E_UNABLE_TO_TAKEOVER_BKR,
cb.getBrokerName(),
"Broker is being taken over by " + takeoverBy);
} else {
logger.log(Logger.ERROR,
BrokerResources.I_NOT_TAKEOVER_BKR);
}
} else {
logger.logStack(Logger.ERROR,
BrokerResources.E_UNABLE_TO_TAKEOVER_BKR,
cb.getBrokerName(),
"Takeover lock error (state=" + state +
", takeoverBroker=" + takeoverBy + ")", ex);
}
try {
cb.setStateFailoverFailed(dbroker.brokerSession);
} catch (Exception ex2) {
logger.logStack(Logger.ERROR,
BrokerResources.E_INTERNAL_ERROR, ex2);
}
} else if (ex instanceof BrokerException &&
((BrokerException)ex).getStatusCode() == Status.CONFLICT){
logger.log(Logger.INFO, ex.getMessage());
} else {
if (tracker != null &&
tracker.getStage() >= TakingoverTracker.AFTER_DB_SWITCH_OWNER) {
logger.logStack(Logger.ERROR, Globals.getBrokerResources().getKString(
BrokerResources.E_TAKEOVER_DATA_PROCESSING_FAIL, tracker.toString()), ex);
} else {
logger.logStack(Logger.ERROR,
BrokerResources.E_UNABLE_TO_TAKEOVER_BKR,
cb.getBrokerName(), ex.getMessage(), ex);
}
}
cb.setBrokerIsUp(false, dbroker.brokerSession, null);
if (throwex) throw ex;
} finally {
mbus.postTakeover(tracker.getBrokerID(),
(takeoverComplete ? tracker.getStoreSessionUID():
tracker.getDownStoreSessionUID()),
!takeoverComplete);
if (tracker.getStage() < TakingoverTracker.AFTER_DB_SWITCH_OWNER) {
takingoverTargets.remove(tracker);