// We discard the topologies from steps 1 and 2, to test that the topology update in step 3 is enough to start and finish the rebalance.
// Block the GET_STATUS command on node 2
final LocalTopologyManager localTopologyManager2 = TestingUtil.extractGlobalComponent(manager(1),
LocalTopologyManager.class);
final CheckPoint checkpoint = new CheckPoint();
LocalTopologyManager spyLocalTopologyManager2 = spy(localTopologyManager2);
final CacheTopology initialTopology = localTopologyManager2.getCacheTopology(CACHE_NAME);
doAnswer(new Answer<Object>() {
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
int viewId = (Integer) invocation.getArguments()[0];
checkpoint.trigger("GET_STATUS_" + viewId);
log.debugf("Blocking the GET_STATUS command on the new coordinator");
checkpoint.awaitStrict("3 left", 10, TimeUnit.SECONDS);
return invocation.callRealMethod();
}
}).when(spyLocalTopologyManager2).handleStatusRequest(anyInt());
// Delay the first CH_UPDATE after the merge so that it arrives after
doAnswer(new Answer<Object>() {
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
CacheTopology topology = (CacheTopology) invocation.getArguments()[1];
if (topology.getRebalanceId() == initialTopology.getRebalanceId() + 1) {
log.debugf("Discarding CH update command %s", topology);
return null;
}
return invocation.callRealMethod();
}
}).when(spyLocalTopologyManager2).handleTopologyUpdate(eq(CACHE_NAME), any(CacheTopology.class),
any(AvailabilityMode.class), anyInt());
doAnswer(new Answer<Object>() {
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
CacheTopology topology = (CacheTopology) invocation.getArguments()[1];
if (topology.getRebalanceId() == initialTopology.getRebalanceId() + 2) {
log.debugf("Discarding rebalance command %s", topology);
return null;
}
return invocation.callRealMethod();
}
}).when(spyLocalTopologyManager2).handleRebalance(eq(CACHE_NAME), any(CacheTopology.class), anyInt());
TestingUtil.replaceComponent(manager(1), LocalTopologyManager.class, spyLocalTopologyManager2, true);
// Node 1 (the coordinator) dies. Node 2 becomes coordinator and tries to call GET_STATUS
log.debugf("Killing coordinator");
manager(0).stop();
TestingUtil.blockUntilViewsReceived(30000, false, manager(1), manager(2));
// Wait for the GET_STATUS command and stop node 3 abruptly
int viewId = manager(1).getTransport().getViewId();
checkpoint.awaitStrict("GET_STATUS_" + viewId, 10, TimeUnit.SECONDS);
d3.setDiscardAll(true);
manager(2).stop();
TestingUtil.blockUntilViewsReceived(30000, false, manager(1));
checkpoint.triggerForever("3 left");
// Wait for node 2 to install a view with only itself and unblock the GET_STATUS command
TestingUtil.waitForRehashToComplete(c2);
}