package org.infinispan.distribution.rehash;
import org.infinispan.AdvancedCache;
import org.infinispan.commands.read.GetKeyValueCommand;
import org.infinispan.commands.write.PutKeyValueCommand;
import org.infinispan.configuration.cache.CacheMode;
import org.infinispan.configuration.cache.ConfigurationBuilder;
import org.infinispan.context.Flag;
import org.infinispan.distribution.MagicKey;
import org.infinispan.statetransfer.StateResponseCommand;
import org.infinispan.statetransfer.StateTransferInterceptor;
import org.infinispan.test.MultipleCacheManagersTest;
import org.infinispan.test.TestingUtil;
import org.infinispan.test.concurrent.CommandMatcher;
import org.infinispan.test.concurrent.StateSequencer;
import org.infinispan.test.fwk.CleanupAfterMethod;
import org.infinispan.transaction.TransactionMode;
import org.testng.annotations.Test;
import java.util.concurrent.Callable;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import static org.infinispan.test.concurrent.StateSequencerUtil.*;
import static org.infinispan.test.concurrent.StateSequencerUtil.matchCommand;
import static org.testng.AssertJUnit.assertEquals;
/**
* Test that a joiner that became a backup owner for a key does not check the previous value when doing a conditional
* write. Also check that if executing a write command during state transfer, it doesn't perform a remote get to obtain
* the previous value from one of the readCH owners.
*
* @author Dan Berindei
*/
@Test(groups = "functional", testName = "distribution.rehash.NonTxJoinerBecomingBackupOwnerTest")
@CleanupAfterMethod
public class NonTxJoinerBecomingBackupOwnerTest extends MultipleCacheManagersTest {
@Override
protected void createCacheManagers() throws Throwable {
ConfigurationBuilder c = getConfigurationBuilder();
addClusterEnabledCacheManager(c);
addClusterEnabledCacheManager(c);
waitForClusterToForm();
}
private ConfigurationBuilder getConfigurationBuilder() {
ConfigurationBuilder c = new ConfigurationBuilder();
c.clustering().cacheMode(CacheMode.DIST_SYNC);
c.transaction().transactionMode(TransactionMode.NON_TRANSACTIONAL);
return c;
}
public void testBackupOwnerJoiningDuringPut() throws Exception {
doTest(TestWriteOperation.PUT_CREATE);
}
public void testBackupOwnerJoiningDuringPutIfAbsent() throws Exception {
doTest(TestWriteOperation.PUT_IF_ABSENT);
}
public void testBackupOwnerJoiningDuringReplace() throws Exception {
doTest(TestWriteOperation.REPLACE);
}
public void testBackupOwnerJoiningDuringReplaceWithPreviousValue() throws Exception {
doTest(TestWriteOperation.REPLACE_EXACT);
}
public void testBackupOwnerJoiningDuringRemove() throws Exception {
doTest(TestWriteOperation.REMOVE);
}
public void testBackupOwnerJoiningDuringRemoveWithPreviousValue() throws Exception {
doTest(TestWriteOperation.REMOVE_EXACT);
}
private void doTest(final TestWriteOperation op) throws Exception {
final StateSequencer sequencer = new StateSequencer();
sequencer.logicalThread("st", "st:cache0_before_send_state");
sequencer.logicalThread("write", "write:before_start", "write:start", "write:cache1_before_return", "write:cache2_before_dist", "write:end", "write:after_end");
sequencer.logicalThread("remote_get_cache0", "remote_get_cache0");
sequencer.logicalThread("remote_get_cache1", "remote_get_cache1");
sequencer.order("write:end", "remote_get_cache0").order("write:end", "remote_get_cache1");
sequencer.action("st:cache0_before_send_state", new Callable<Object>() {
@Override
public Object call() throws Exception {
sequencer.advance("write:before_start");
// The whole write logical thread happens here
sequencer.advance("write:after_end");
return null;
}
});
final AdvancedCache<Object, Object> cache0 = advancedCache(0);
final AdvancedCache<Object, Object> cache1 = advancedCache(1);
// We only block the StateResponseCommand on cache0, because that's the node cache2 will ask for the magic key
advanceOnOutboundRpc(sequencer, cache0, matchCommand(StateResponseCommand.class).build()).before("st:cache0_before_send_state");
// Prohibit any remote get from cache2 to either cache0 or cache1
advanceOnInterceptor(sequencer, cache0, StateTransferInterceptor.class, matchCommand(GetKeyValueCommand.class).build()).before("remote_get_cache0");
advanceOnInterceptor(sequencer, cache1, StateTransferInterceptor.class, matchCommand(GetKeyValueCommand.class).build()).before("remote_get_cache1");
// Add a new member, but don't start the cache yet
ConfigurationBuilder c = getConfigurationBuilder();
c.clustering().stateTransfer().awaitInitialTransfer(false);
addClusterEnabledCacheManager(c);
// Start the cache and wait until it's a member in the write CH
log.tracef("Starting the cache on the joiner");
final AdvancedCache<Object,Object> cache2 = advancedCache(2);
// Wait for the write CH to contain the joiner everywhere
eventually(new Condition() {
@Override
public boolean isSatisfied() throws Exception {
return cache0.getRpcManager().getMembers().size() == 3 &&
cache1.getRpcManager().getMembers().size() == 3 &&
cache2.getRpcManager().getMembers().size() == 3;
}
});
CommandMatcher writeCommandMatcher = matchCommand(op.getCommandClass()).build();
// Allow the value to be written on cache1 before "write:cache1_before_return"
advanceOnInterceptor(sequencer, cache1, StateTransferInterceptor.class, writeCommandMatcher).before("write:cache1_before_return");
// The remote get (if any) will happen after "write:cache2_before_dist"
advanceOnInterceptor(sequencer, cache2, StateTransferInterceptor.class, writeCommandMatcher).before("write:cache2_before_dist");
// Wait for cache0 to send the StateResponseCommand to cache2, but keep it blocked
sequencer.advance("write:start");
final MagicKey key = getKeyForCache2();
// Prepare for replace: put a previous value in cache0 and cache1
if (op.getPreviousValue() != null) {
cache0.withFlags(Flag.CACHE_MODE_LOCAL).put(key, op.getPreviousValue());
cache1.withFlags(Flag.CACHE_MODE_LOCAL).put(key, op.getPreviousValue());
}
log.tracef("Initial value set, %s = %s", key, op.getPreviousValue());
// Put from cache0 with cache0 as primary owner, cache2 will become a backup owner for the retry
// The put command will be blocked on cache1 and cache2.
Future<Object> future = fork(new Callable<Object>() {
@Override
public Object call() throws Exception {
return op.perform(cache0, key);
}
});
// Check that the put command didn't fail
Object result = future.get(10, TimeUnit.SECONDS);
assertEquals(op.getReturnValue(), result);
log.tracef("%s operation is done", op);
// Allow the state transfer to finish, and any remote gets
sequencer.advance("write:end");
// Wait for the topology to change everywhere
TestingUtil.waitForRehashToComplete(cache0, cache1, cache2);
// Stop blocking get commands and check the value on all the nodes
sequencer.stop();
assertEquals(op.getValue(), cache0.get(key));
assertEquals(op.getValue(), cache1.get(key));
assertEquals(op.getValue(), cache2.get(key));
}
private MagicKey getKeyForCache2() {
return new MagicKey(cache(0), cache(1), cache(2));
}
}