public void testBackupOwnerJoiningDuringRemoveWithPreviousValue() throws Exception {
doTest(TestWriteOperation.REMOVE_EXACT);
}
private void doTest(final TestWriteOperation op) throws Exception {
final StateSequencer sequencer = new StateSequencer();
sequencer.logicalThread("st", "st:cache0_before_send_state");
sequencer.logicalThread("write", "write:before_start", "write:start", "write:cache1_before_return", "write:cache2_before_dist", "write:end", "write:after_end");
sequencer.logicalThread("remote_get_cache0", "remote_get_cache0");
sequencer.logicalThread("remote_get_cache1", "remote_get_cache1");
sequencer.order("write:end", "remote_get_cache0").order("write:end", "remote_get_cache1");
sequencer.action("st:cache0_before_send_state", new Callable<Object>() {
@Override
public Object call() throws Exception {
sequencer.advance("write:before_start");
// The whole write logical thread happens here
sequencer.advance("write:after_end");
return null;
}
});
final AdvancedCache<Object, Object> cache0 = advancedCache(0);
final AdvancedCache<Object, Object> cache1 = advancedCache(1);
// We only block the StateResponseCommand on cache0, because that's the node cache2 will ask for the magic key
advanceOnOutboundRpc(sequencer, cache0, matchCommand(StateResponseCommand.class).build()).before("st:cache0_before_send_state");
// Prohibit any remote get from cache2 to either cache0 or cache1
advanceOnInterceptor(sequencer, cache0, StateTransferInterceptor.class, matchCommand(GetKeyValueCommand.class).build()).before("remote_get_cache0");
advanceOnInterceptor(sequencer, cache1, StateTransferInterceptor.class, matchCommand(GetKeyValueCommand.class).build()).before("remote_get_cache1");
// Add a new member, but don't start the cache yet
ConfigurationBuilder c = getConfigurationBuilder();
c.clustering().stateTransfer().awaitInitialTransfer(false);
addClusterEnabledCacheManager(c);
// Start the cache and wait until it's a member in the write CH
log.tracef("Starting the cache on the joiner");
final AdvancedCache<Object,Object> cache2 = advancedCache(2);
// Wait for the write CH to contain the joiner everywhere
eventually(new Condition() {
@Override
public boolean isSatisfied() throws Exception {
return cache0.getRpcManager().getMembers().size() == 3 &&
cache1.getRpcManager().getMembers().size() == 3 &&
cache2.getRpcManager().getMembers().size() == 3;
}
});
CommandMatcher writeCommandMatcher = matchCommand(op.getCommandClass()).build();
// Allow the value to be written on cache1 before "write:cache1_before_return"
advanceOnInterceptor(sequencer, cache1, StateTransferInterceptor.class, writeCommandMatcher).before("write:cache1_before_return");
// The remote get (if any) will happen after "write:cache2_before_dist"
advanceOnInterceptor(sequencer, cache2, StateTransferInterceptor.class, writeCommandMatcher).before("write:cache2_before_dist");
// Wait for cache0 to send the StateResponseCommand to cache2, but keep it blocked
sequencer.advance("write:start");
final MagicKey key = getKeyForCache2();
// Prepare for replace: put a previous value in cache0 and cache1
if (op.getPreviousValue() != null) {
cache0.withFlags(Flag.CACHE_MODE_LOCAL).put(key, op.getPreviousValue());
cache1.withFlags(Flag.CACHE_MODE_LOCAL).put(key, op.getPreviousValue());
}
log.tracef("Initial value set, %s = %s", key, op.getPreviousValue());
// Put from cache0 with cache0 as primary owner, cache2 will become a backup owner for the retry
// The put command will be blocked on cache1 and cache2.
Future<Object> future = fork(new Callable<Object>() {
@Override
public Object call() throws Exception {
return op.perform(cache0, key);
}
});
// Check that the put command didn't fail
Object result = future.get(10, TimeUnit.SECONDS);
assertEquals(op.getReturnValue(), result);
log.tracef("%s operation is done", op);
// Allow the state transfer to finish, and any remote gets
sequencer.advance("write:end");
// Wait for the topology to change everywhere
TestingUtil.waitForRehashToComplete(cache0, cache1, cache2);
// Stop blocking get commands and check the value on all the nodes
sequencer.stop();
assertEquals(op.getValue(), cache0.get(key));
assertEquals(op.getValue(), cache1.get(key));
assertEquals(op.getValue(), cache2.get(key));
}