/*
*
* JBoss, the OpenSource J2EE webOS
*
* Distributable under LGPL license.
* See terms of license at gnu.org.
*/
package org.jboss.cache.replicated;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.jboss.cache.Cache;
import org.jboss.cache.CacheException;
import org.jboss.cache.CacheSPI;
import org.jboss.cache.DefaultCacheFactory;
import org.jboss.cache.Fqn;
import org.jboss.cache.config.Configuration;
import org.jboss.cache.lock.IsolationLevel;
import org.jboss.cache.lock.TimeoutException;
import org.jboss.cache.misc.TestingUtil;
import org.jboss.cache.notifications.annotation.CacheListener;
import org.jboss.cache.notifications.annotation.NodeModified;
import org.jboss.cache.notifications.event.NodeEvent;
import org.jboss.cache.transaction.TransactionSetup;
import org.jboss.cache.util.CachePrinter;
import static org.testng.AssertJUnit.*;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import javax.transaction.NotSupportedException;
import javax.transaction.RollbackException;
import javax.transaction.Status;
import javax.transaction.Synchronization;
import javax.transaction.SystemException;
import javax.transaction.Transaction;
import javax.transaction.TransactionManager;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Semaphore;
/**
* Replicated unit test for sync transactional CacheImpl
* Note: we use DummyTransactionManager for Tx purpose instead of relying on
* jta.
*
* @version $Revision: 5066 $
*/
@Test(groups = {"functional", "jgroups", "transaction"})
public class SyncReplTxTest
{
private static Log log = LogFactory.getLog(SyncReplTxTest.class);
private CacheSPI<Object, Object> cache1;
private CacheSPI<Object, Object> cache2;
Semaphore lock;
private Throwable t1_ex;
private Throwable t2_ex;
@BeforeMethod(alwaysRun = true)
public void setUp() throws Exception
{
t1_ex = t2_ex = null;
lock = new Semaphore(1);
}
@AfterMethod(alwaysRun = true)
public void tearDown() throws Exception
{
TransactionSetup.cleanup();
destroyCaches();
}
private TransactionManager beginTransaction() throws SystemException, NotSupportedException
{
return beginTransaction(cache1);
}
private TransactionManager beginTransaction(CacheSPI c) throws SystemException, NotSupportedException
{
TransactionManager mgr = c.getConfiguration().getRuntimeConfig().getTransactionManager();
mgr.begin();
return mgr;
}
private void initCaches(Configuration.CacheMode caching_mode) throws Exception
{
cache1 = (CacheSPI<Object, Object>) new DefaultCacheFactory().createCache(false);
cache2 = (CacheSPI<Object, Object>) new DefaultCacheFactory().createCache(false);
cache1.getConfiguration().setCacheMode(caching_mode);
cache2.getConfiguration().setCacheMode(caching_mode);
cache1.getConfiguration().setIsolationLevel(IsolationLevel.SERIALIZABLE);
cache2.getConfiguration().setIsolationLevel(IsolationLevel.SERIALIZABLE);
cache1.getConfiguration().setTransactionManagerLookupClass(TransactionSetup.getManagerLookup());
cache2.getConfiguration().setTransactionManagerLookupClass(TransactionSetup.getManagerLookup());
cache1.getConfiguration().setLockAcquisitionTimeout(5000);
cache2.getConfiguration().setLockAcquisitionTimeout(5000);
configureMultiplexer(cache1);
configureMultiplexer(cache2);
cache1.start();
cache2.start();
validateMultiplexer(cache1);
validateMultiplexer(cache2);
}
/**
* Provides a hook for multiplexer integration. This default implementation
* is a no-op; subclasses that test mux integration would override
* to integrate the given cache with a multiplexer.
* <p/>
* param cache a cache that has been configured but not yet created.
*/
protected void configureMultiplexer(Cache cache) throws Exception
{
// default does nothing
}
/**
* Provides a hook to check that the cache's channel came from the
* multiplexer, or not, as expected. This default impl asserts that
* the channel did not come from the multiplexer.
*
* @param cache a cache that has already been started
*/
protected void validateMultiplexer(Cache cache)
{
assertFalse("Cache is not using multiplexer", cache.getConfiguration().isUsingMultiplexer());
}
private void destroyCaches()
{
if (cache1 != null)
{
cache1.stop();
}
if (cache2 != null)
{
cache2.stop();
}
cache1 = null;
cache2 = null;
}
public void testLockRemoval() throws Exception
{
initCaches(Configuration.CacheMode.REPL_SYNC);
cache1.getConfiguration().setSyncCommitPhase(true);
cache1.getRoot().getLock().releaseAll();
TransactionManager tm = beginTransaction();
cache1.put("/bela/ban", "name", "Bela Ban");
assertEquals(3, cache1.getNumberOfLocksHeld());
assertEquals(0, cache2.getNumberOfLocksHeld());
tm.commit();
assertEquals(0, cache1.getNumberOfLocksHeld());
assertEquals(0, cache2.getNumberOfLocksHeld());
}
public void testSyncRepl() throws Exception
{
Integer age;
Transaction tx;
try
{
initCaches(Configuration.CacheMode.REPL_SYNC);
cache1.getConfiguration().setSyncCommitPhase(true);
cache2.getConfiguration().setSyncCommitPhase(true);
// assertEquals(2, cache1.getMembers().size());
TransactionManager mgr = beginTransaction();
cache1.put("/a/b/c", "age", 38);
tx = mgr.suspend();
assertNull("age on cache2 must be null as the TX has not yet been committed", cache2.get("/a/b/c", "age"));
log.debug("cache1: locks held before commit: " + CachePrinter.printCacheLockingInfo(cache1));
log.debug("cache2: locks held before commit: " + CachePrinter.printCacheLockingInfo(cache2));
mgr.resume(tx);
mgr.commit();
log.debug("cache1: locks held after commit: " + CachePrinter.printCacheLockingInfo(cache1));
log.debug("cache2: locks held after commit: " + CachePrinter.printCacheLockingInfo(cache2));
// value on cache2 must be 38
age = (Integer) cache2.get("/a/b/c", "age");
assertNotNull("\"age\" obtained from cache2 must be non-null ", age);
assertTrue("\"age\" must be 38", age == 38);
}
catch (Exception e)
{
fail(e.toString());
}
}
/**
* @throws Exception
*/
public void testSimplePut() throws Exception
{
initCaches(Configuration.CacheMode.REPL_SYNC);
cache1.put("/JSESSION/localhost/192.168.1.10:32882/Courses/0", "Instructor", "Ben Wang");
cache1.put("/JSESSION/localhost/192.168.1.10:32882/1", "Number", 10);
}
public void testSimpleTxPut() throws Exception
{
TransactionManager tm;
final Fqn NODE1 = Fqn.fromString("/one/two/three");
initCaches(Configuration.CacheMode.REPL_SYNC);
tm = beginTransaction();
cache1.put(NODE1, "age", 38);
System.out.println("TransactionTable for cache1 after cache1.put():\n" + cache1.getTransactionTable().toString(true));
tm.commit();
/*
tx=beginTransaction();
cache1.put(NODE1, "age", new Integer(38));
cache1.put(NODE2, "name", "Ben of The Far East");
cache1.put(NODE3, "key", "UnknowKey");
System.out.println("TransactionTable for cache1 after cache1.put():\n" + cache1.getTransactionTable().toString(true));
tx.commit();
*/
/*
tx=beginTransaction();
cache1.put(NODE1, "age", new Integer(38));
cache1.put(NODE1, "AOPInstance", new AOPInstance());
cache1.put(NODE2, "AOPInstance", new AOPInstance());
cache1.put(NODE1, "AOPInstance", new AOPInstance());
tx.commit();
System.out.println("TransactionTable for cache1 after cache1.put():\n" + cache1.getTransactionTable().toString(true));
*/
}
public void testSyncReplWithModficationsOnBothCaches() throws Exception
{
TransactionManager tm;
final Fqn NODE1 = Fqn.fromString("/one/two/three");
final Fqn NODE2 = Fqn.fromString("/eins/zwei/drei");
initCaches(Configuration.CacheMode.REPL_SYNC);
// create roots first
cache1.put("/one/two", null);
cache2.put("/eins/zwei", null);
cache1.getConfiguration().setSyncCommitPhase(true);
cache2.getConfiguration().setSyncCommitPhase(true);
tm = beginTransaction();
cache1.put(NODE1, "age", 38);
System.out.println("TransactionTable for cache1 after cache1.put():\n" + cache1.getTransactionTable().toString(true));
cache2.put(NODE2, "age", 39);
System.out.println("TransactionTable for cache2 after cache2.put():\n" + cache2.getTransactionTable().toString(true));
System.out.println("cache1 before commit:\n" + CachePrinter.printCacheLockingInfo(cache1));
System.out.println("cache2 before commit:\n" + CachePrinter.printCacheLockingInfo(cache2));
try
{
tm.commit();
fail("Should not succeed with SERIALIZABLE semantics");
}
catch (Exception e)
{
//should be a classic deadlock here.
}
System.out.println("cache1 after commit:\n" + CachePrinter.printCacheLockingInfo(cache1));
System.out.println("cache2 after commit:\n" + CachePrinter.printCacheLockingInfo(cache2));
/*
assertTrue(cache1.exists(NODE1));
assertTrue(cache1.exists(NODE2));
assertTrue(cache1.exists(NODE1));
assertTrue(cache2.exists(NODE2));
age = (Integer) cache1.get(NODE1, "age");
assertNotNull("\"age\" obtained from cache1 for " + NODE1 + " must be non-null ", age);
assertTrue("\"age\" must be 38", age == 38);
age = (Integer) cache2.get(NODE1, "age");
assertNotNull("\"age\" obtained from cache2 for " + NODE1 + " must be non-null ", age);
assertTrue("\"age\" must be 38", age == 38);
age = (Integer) cache1.get(NODE2, "age");
assertNotNull("\"age\" obtained from cache1 for " + NODE2 + " must be non-null ", age);
assertTrue("\"age\" must be 39", age == 39);
age = (Integer) cache2.get(NODE2, "age");
assertNotNull("\"age\" obtained from cache2 for " + NODE2 + " must be non-null ", age);
assertTrue("\"age\" must be 39", age == 39);
*/
assertEquals(0, cache1.getNumberOfLocksHeld());
assertEquals(0, cache2.getNumberOfLocksHeld());
System.out.println("TransactionTable for cache1:\n" + cache1.getTransactionTable().toString(true));
System.out.println("TransactionTable for cache2:\n" + cache2.getTransactionTable().toString(true));
}
public void testSyncReplWithModficationsOnBothCachesSameData() throws Exception
{
TransactionManager tm;
final Fqn NODE = Fqn.fromString("/one/two/three");
initCaches(Configuration.CacheMode.REPL_SYNC);
tm = beginTransaction();
cache1.put(NODE, "age", 38);
System.out.println("TransactionTable for cache1 after cache1.put():\n" + cache1.getTransactionTable().toString(true));
cache2.put(NODE, "age", 39);
System.out.println("TransactionTable for cache2 after cache2.put():\n" + cache2.getTransactionTable().toString(true));
System.out.println("cache1 before commit:\n" + CachePrinter.printCacheLockingInfo(cache1));
System.out.println("cache2 before commit:\n" + CachePrinter.printCacheLockingInfo(cache2));
try
{
tm.commit();
fail("commit should throw a RollbackException, we should not get here");
}
catch (RollbackException rollback)
{
System.out.println("Transaction was rolled back, this is correct");
}
System.out.println("cache1 after commit:\n" + CachePrinter.printCacheLockingInfo(cache1));
System.out.println("cache2 after commit:\n" + CachePrinter.printCacheLockingInfo(cache2));
assertEquals(0, cache1.getNumberOfLocksHeld());
assertEquals(0, cache2.getNumberOfLocksHeld());
assertEquals(0, cache1.getNumberOfNodes());
assertEquals(0, cache2.getNumberOfNodes());
}
public void testSyncReplWithModficationsOnBothCachesWithRollback() throws Exception
{
TransactionManager tm;
final Fqn NODE1 = Fqn.fromString("/one/two/three");
final Fqn NODE2 = Fqn.fromString("/eins/zwei/drei");
initCaches(Configuration.CacheMode.REPL_SYNC);
cache1.getConfiguration().setSyncRollbackPhase(true);
cache2.getConfiguration().setSyncRollbackPhase(true);
tm = beginTransaction();
cache1.put(NODE1, "age", 38);
cache2.put(NODE2, "age", 39);
System.out.println("cache1 (before commit):\n" + CachePrinter.printCacheLockingInfo(cache1));
System.out.println("cache2 (before commit):\n" + CachePrinter.printCacheLockingInfo(cache2));
// this will rollback the transaction
Transaction tx = tm.getTransaction();
tx.registerSynchronization(new TransactionAborter(tx));
try
{
tm.commit();
fail("commit should throw a RollbackException, we should not get here");
}
catch (RollbackException rollback)
{
System.out.println("Transaction was rolled back, this is correct");
}
System.out.println("cache1 (after rollback):\n" + CachePrinter.printCacheLockingInfo(cache1));
System.out.println("cache2 (after rollback):\n" + CachePrinter.printCacheLockingInfo(cache2));
assertEquals(0, cache1.getNumberOfLocksHeld());
assertEquals(0, cache2.getNumberOfLocksHeld());
assertEquals(0, cache1.getNumberOfNodes());
assertEquals(0, cache2.getNumberOfNodes());
}
/**
* Test for JBCACHE-359 -- does a callback into cache from a listener
* interfere with transaction rollback.
*
* @throws Exception
*/
// Is this test still valid after JBCACHE-1022 ?
// public void testSyncReplWithRollbackAndListener() throws Exception
// {
// Transaction tx;
// final Fqn NODE1 = Fqn.fromString("/one/two/three");
//
// initCaches(Configuration.CacheMode.REPL_SYNC);
//
// cache1.getConfiguration().setSyncRollbackPhase(true);
// cache2.getConfiguration().setSyncRollbackPhase(true);
//
// // Test with a rollback on the sending side
//
// CallbackListener cbl1 = new CallbackListener(cache1, "age");
// CallbackListener cbl2 = new CallbackListener(cache2, "age");
//
// tx = beginTransaction();
// cache1.put(NODE1, "age", 38);
//
// System.out.println("cache1 (before commit):\n" + CachePrinter.printCacheLockingInfo(cache1));
// System.out.println("cache2 (before commit):\n" + CachePrinter.printCacheLockingInfo(cache2));
//
// // this will rollback the transaction
// tx.registerSynchronization(new TransactionAborter(tx));
//
// try
// {
// tx.commit();
// fail("commit should throw a RollbackException, we should not get here");
// }
// catch (RollbackException rollback)
// {
// rollback.printStackTrace();
// System.out.println("Transaction was rolled back, this is correct");
// }
//
// // Sleep, as the rollback call to cache2 is async
// TestingUtil.sleepThread(1000);
//
// System.out.println("cache1 (after rollback):\n" + CachePrinter.printCacheLockingInfo(cache1));
// System.out.println("cache2 (after rollback):\n" + CachePrinter.printCacheLockingInfo(cache2));
//
// assertNull(cbl1.getCallbackException());
// assertNull(cbl2.getCallbackException());
//
// assertEquals(0, cache1.getNumberOfLocksHeld());
// assertEquals(0, cache2.getNumberOfLocksHeld());
//
// assertEquals(0, cache1.getNumberOfNodes());
// assertEquals(0, cache2.getNumberOfNodes());
//
// // Test with a rollback on the receiving side
//
// cache2.getNotifier().removeCacheListener(cbl2);
// // listener aborts any active tx
// cbl2 = new TransactionAborterCallbackListener(cache2, "age");
//
// tx = beginTransaction();
// cache1.put(NODE1, "age", 38);
//
// System.out.println("cache1 (before commit):\n" + CachePrinter.printCacheLockingInfo(cache1));
// System.out.println("cache2 (before commit):\n" + CachePrinter.printCacheLockingInfo(cache2));
//
// tx.commit();
//
// // Sleep, as the commit call to cache2 is async
// TestingUtil.sleepThread(1000);
//
// System.out.println("cache1 (after rollback):\n" + CachePrinter.printCacheLockingInfo(cache1));
// System.out.println("cache2 (after rollback):\n" + CachePrinter.printCacheLockingInfo(cache2));
//
// assertNull(cbl1.getCallbackException());
// assertNull(cbl2.getCallbackException());
//
// assertEquals(0, cache1.getNumberOfLocksHeld());
// assertEquals(0, cache2.getNumberOfLocksHeld());
//
// // cache1 didn't fail, so should have 3 nodes
// assertEquals(3, cache1.getNumberOfNodes());
// assertEquals(0, cache2.getNumberOfNodes());
//
// }
/**
* Test for JBCACHE-361 -- does marking a tx on the remote side
* rollback-only cause a rollback on the originating side?
*
* @throws Exception
*/
public void testSyncReplWithRemoteRollback() throws Exception
{
TransactionManager tm;
final Fqn NODE1 = Fqn.fromString("/one/two/three");
initCaches(Configuration.CacheMode.REPL_SYNC);
cache1.getConfiguration().setSyncRollbackPhase(true);
cache2.getConfiguration().setSyncRollbackPhase(true);
// Test with a rollback on the remote side
// listener aborts any active tx
//TransactionAborterListener tal = new TransactionAborterListener(cache2);
tm = beginTransaction();
cache1.put(NODE1, "age", 38);
System.out.println("cache1 (before commit):\n" + CachePrinter.printCacheLockingInfo(cache1));
System.out.println("cache2 (before commit):\n" + CachePrinter.printCacheLockingInfo(cache2));
// instead of a listener lets just get a WL on ROOT on cache2. And hold on to it.
Transaction tx = tm.suspend();
tm.begin();
cache2.getRoot().put("x", "y");
Transaction tx2 = cache2.getTransactionManager().suspend();
System.out.println("cache2 (before commit):\n" + CachePrinter.printCacheLockingInfo(cache2));
tm.resume(tx);
try
{
tm.commit();
fail("commit should throw a RollbackException, we should not get here");
}
catch (RollbackException rollback)
{
System.out.println("Transaction was rolled back, this is correct");
}
finally
{
tm.resume(tx2);
tm.rollback();
}
// Sleep, as the commit call to cache2 is async
TestingUtil.sleepThread(1000);
System.out.println("cache1 (after rollback):\n" + CachePrinter.printCacheLockingInfo(cache1));
System.out.println("cache2 (after rollback):\n" + CachePrinter.printCacheLockingInfo(cache2));
//assertNull(tal.getCallbackException());
assertEquals(0, cache1.getNumberOfLocksHeld());
assertEquals(0, cache2.getNumberOfLocksHeld());
assertEquals(0, cache1.getNumberOfNodes());
assertEquals(0, cache2.getNumberOfNodes());
}
public void testASyncRepl() throws Exception
{
Integer age;
TransactionManager tm;
initCaches(Configuration.CacheMode.REPL_ASYNC);
tm = beginTransaction();
cache1.put("/a/b/c", "age", 38);
Thread.sleep(1000);
assertNull("age on cache2 must be null as the TX has not yet been committed", cache2.get("/a/b/c", "age"));
tm.commit();
Thread.sleep(1000);
// value on cache2 must be 38
age = (Integer) cache2.get("/a/b/c", "age");
assertNotNull("\"age\" obtained from cache2 is null ", age);
assertTrue("\"age\" must be 38", age == 38);
}
/**
* Tests concurrent modifications: thread1 succeeds and thread2 is blocked until thread1 is done, and then succeeds
* too. However, this is flawed with the introduction of interceptors, here's why.<br/>
* <ul>
* <li>Thread1 acquires the lock for /bela/ban on cache1
* <li>Thread2 blocks on Thread1 to release the lock
* <li>Thread1 commits: this means the TransactionInterceptor and the ReplicationInterceptor are called in
* the sequence in which they registered. Unfortunately, the TransactionInterceptor registered first. In the
* PREPARE phase, the ReplicationInterceptor calls prepare() in cache2 synchronously. The TxInterceptor
* does nothing. The the COMMIT phase, the TxInterceptor commits the data by releasing the locks locally and
* then the ReplicationInterceptor sends an asynchronous COMMIT to cache2.
* <li>Because the TxInterceptor for Thread1 releases the locks locally <em>before</em> sending the async COMMIT,
* Thread2 is able to acquire the lock for /bela/ban in cache1 and then starts the PREPARE phase by sending a
* synchronous PREPARE to cache2. If this PREPARE arrives at cache2 <em>before</em> the COMMIT from Thread1,
* the PREPARE will block because it attempts to acquire a lock on /bela/ban on cache2 still held by Thread1
* (which would be released by Thread1's COMMIT). This results in deadlock, which is resolved by Thread2 running
* into a timeout with subsequent rollback and Thread1 succeeding.<br/>
* </ul>
* There are 3 solutions to this:
* <ol>
* <li>Do nothing. This is standard behavior for concurrent access to the same data. Same thing if the 2 threads
* operated on the same data in <em>separate</em> caches, e.g. Thread1 on /bela/ban in cache1 and Thread2 on
* /bela/ban in cache2. The semantics of Tx commit as handled by the interceptors is: after tx1.commit() returns
* the locks held by tx1 are release and a COMMIT message is on the way (if sent asynchronously).
* <li>Force an order over TxInterceptor and ReplicationInterceptor. This would require ReplicationInterceptor
* to always be fired first on TX commit. Downside: the interceptors have an implicit dependency, which is not
* nice.
* <li>Priority-order requests at the receiver; e.g. a COMMIT could release a blocked PREPARE. This is bad because
* it violates JGroups' FIFO ordering guarantees.
* </ol>
* I'm currently investigating solution #2, ie. creating an OrderedSynchronizationHandler, which allows other
* SynchronizationHandlers to register (atHead, atTail), and the OrderedSynchronizationHandler would call the
* SynchronizationHandler in the order in which they are defined.
*
* @throws Exception
*/
public void testConcurrentPuts() throws Exception
{
initCaches(Configuration.CacheMode.REPL_SYNC);
cache1.getConfiguration().setSyncCommitPhase(true);
Thread t1 = new Thread("Thread1")
{
TransactionManager tm;
public void run()
{
try
{
tm = beginTransaction();
cache1.put("/bela/ban", "name", "Bela Ban");
TestingUtil.sleepThread(2000);// Thread2 will be blocked until we commit
tm.commit();
System.out.println("[Thread1] ** LOCK INFO cache1: " + CachePrinter.printCacheLockingInfo(cache1));
System.out.println("[Thread1] ** LOCK INFO cache2: " + CachePrinter.printCacheLockingInfo(cache2));
}
catch (Throwable ex)
{
ex.printStackTrace();
t1_ex = ex;
}
}
};
Thread t2 = new Thread("Thread2")
{
TransactionManager tm;
public void run()
{
try
{
TestingUtil.sleepThread(1000);// give Thread1 time to acquire the lock
tm = beginTransaction();
System.out.println("[Thread2] ** LOCK INFO cache1: " + CachePrinter.printCacheLockingInfo(cache1));
System.out.println("[Thread2] ** LOCK INFO cache2: " + CachePrinter.printCacheLockingInfo(cache2));
cache1.put("/bela/ban", "name", "Michelle Ban");
System.out.println("[Thread2] ** LOCK INFO cache1: " + CachePrinter.printCacheLockingInfo(cache1));
System.out.println("[Thread2] ** LOCK INFO cache2: " + CachePrinter.printCacheLockingInfo(cache2));
tm.commit();
System.out.println("[Thread2] ** LOCK INFO cache1: " + CachePrinter.printCacheLockingInfo(cache1));
System.out.println("[Thread2] ** LOCK INFO cache2: " + CachePrinter.printCacheLockingInfo(cache2));
}
catch (Throwable ex)
{
ex.printStackTrace();
t2_ex = ex;
}
}
};
// Let the game start
t1.start();
t2.start();
// Wait for threads to die
t1.join();
t2.join();
if (t1_ex != null)
{
fail("Thread1 failed: " + t1_ex);
}
if (t2_ex != null)
{
fail("Thread2 failed: " + t2_ex);
}
assertEquals("Michelle Ban", cache1.get("/bela/ban", "name"));
}
/**
* Should reproduce JBCACHE-32 problem (http://jira.jboss.com/jira/browse/JBCACHE-32)
*/
public void testConcurrentCommitsWith1Thread() throws Exception
{
_testConcurrentCommits(1);
}
/**
* Should reproduce JBCACHE-32 problem (http://jira.jboss.com/jira/browse/JBCACHE-32)
*/
public void testConcurrentCommitsWith5Threads() throws Exception
{
_testConcurrentCommits(5);
}
/**
* Should reproduce JBCACHE-32 problem (http://jira.jboss.com/jira/browse/JBCACHE-32)
*/
private void _testConcurrentCommits(int num_threads)
{
Object myMutex = new Object();
final CacheSPI c1 = (CacheSPI) new DefaultCacheFactory().createCache(false);
final CacheSPI c2 = (CacheSPI) new DefaultCacheFactory().createCache(false);
c1.getConfiguration().setClusterName("TempCluster");
c2.getConfiguration().setClusterName("TempCluster");
c1.getConfiguration().setCacheMode(Configuration.CacheMode.REPL_SYNC);
c2.getConfiguration().setCacheMode(Configuration.CacheMode.REPL_SYNC);
c1.getConfiguration().setSyncCommitPhase(true);
c2.getConfiguration().setSyncCommitPhase(true);
c1.getConfiguration().setSyncRollbackPhase(true);
c2.getConfiguration().setSyncRollbackPhase(true);
c1.getConfiguration().setIsolationLevel(IsolationLevel.REPEATABLE_READ);
c2.getConfiguration().setIsolationLevel(IsolationLevel.REPEATABLE_READ);
c1.getConfiguration().setTransactionManagerLookupClass("org.jboss.cache.transaction.DummyTransactionManagerLookup");
c2.getConfiguration().setTransactionManagerLookupClass("org.jboss.cache.transaction.DummyTransactionManagerLookup");
c1.getConfiguration().setLockAcquisitionTimeout(5000);
c2.getConfiguration().setLockAcquisitionTimeout(5000);
c1.start();
c2.start();
final List<Exception> exceptions = new ArrayList<Exception>();
class MyThread extends Thread
{
Object mutex;
public MyThread(String name, Object mutex)
{
super(name);
this.mutex = mutex;
}
public void run()
{
TransactionManager tm = null;
try
{
tm = beginTransaction(c1);
c1.put("/thread/" + getName(), null);
System.out.println("Thread " + getName() + " after put(): " + c1.toString());
System.out.println("Thread " + getName() + " waiting on mutex");
synchronized (mutex)
{
mutex.wait();
}
System.out.println("Thread " + getName() + " committing");
tm.commit();
System.out.println("Thread " + getName() + " committed successfully");
}
catch (Exception e)
{
exceptions.add(e);
}
finally
{
try
{
if (tm != null)
tm.rollback();
}
catch (Exception e)
{
}
}
}
}
MyThread[] threads = new MyThread[num_threads];
for (int i = 0; i < threads.length; i++)
{
threads[i] = new MyThread("#" + i, myMutex);
}
for (int i = 0; i < threads.length; i++)
{
MyThread thread = threads[i];
System.out.println("starting thread #" + i);
thread.start();
}
TestingUtil.sleepThread(6000);
synchronized (myMutex)
{
System.out.println("cache is " + CachePrinter.printCacheLockingInfo(c1));
System.out.println("******************* SIGNALLING THREADS ********************");
myMutex.notifyAll();
}
for (MyThread thread : threads)
{
try
{
thread.join();
System.out.println("Joined thread " + thread.getName());
}
catch (InterruptedException e)
{
e.printStackTrace();
}
}
System.out.println("FINAL c1:\n" + CachePrinter.printCacheDetails(c1) + "\nlocks:\n" + CachePrinter.printCacheLockingInfo(c1));
assertEquals(0, c1.getNumberOfLocksHeld());
assertEquals(0, c2.getNumberOfLocksHeld());
c1.stop();
c2.stop();
// if(ex != null)
// {
// ex.printStackTrace();
// fail("Thread failed: " + ex);
// }
// we can only expect 1 thread to succeed. The others will fail. So, threads.length -1 exceptions.
// this is a timing issue - 2 threads still may succeed on a multi cpu system
// assertEquals(threads.length - 1, exceptions.size());
for (Exception exception : exceptions)
assertEquals(TimeoutException.class, exception.getClass());
}
/**
* Conncurrent put on 2 different instances.
*/
public void testConcurrentPutsOnTwoInstances() throws Exception
{
initCaches(Configuration.CacheMode.REPL_SYNC);
final CacheSPI<Object, Object> c1 = this.cache1;
final CacheSPI<Object, Object> c2 = this.cache2;
Thread t1 = new Thread()
{
TransactionManager tm;
public void run()
{
try
{
tm = beginTransaction();
c1.put("/ben/wang", "name", "Ben Wang");
TestingUtil.sleepThread(8000);
tm.commit();// This should go thru
}
catch (Throwable ex)
{
ex.printStackTrace();
t1_ex = ex;
}
}
};
Thread t2 = new Thread()
{
TransactionManager tm;
public void run()
{
try
{
TestingUtil.sleepThread(1000);// give Thread1 time to acquire the lock
tm = beginTransaction();
c2.put("/ben/wang", "name", "Ben Jr.");
tm.commit();// This will time out and rollback first because Thread1 has a tx going as well.
}
catch (RollbackException rollback_ex)
{
System.out.println("received rollback exception as expected");
}
catch (Throwable ex)
{
ex.printStackTrace();
t2_ex = ex;
}
}
};
// Let the game start
t1.start();
t2.start();
// Wait for thread to die but put an insurance of 5 seconds on it.
t1.join();
t2.join();
if (t1_ex != null)
{
fail("Thread1 failed: " + t1_ex);
}
if (t2_ex != null)
{
fail("Thread2 failed: " + t2_ex);
}
assertEquals("Ben Wang", c1.get("/ben/wang", "name"));
}
public void testPut() throws Exception
{
initCaches(Configuration.CacheMode.REPL_SYNC);
final CacheSPI<Object, Object> c1 = this.cache1;
Thread t1 = new Thread()
{
public void run()
{
try
{
lock.acquire();
System.out.println("-- t1 has lock");
c1.put("/a/b/c", "age", 38);
System.out.println("[Thread1] set value to 38");
System.out.println("-- t1 releases lock");
lock.release();
TestingUtil.sleepThread(300);
Thread.yield();
lock.acquire();
System.out.println("-- t1 has lock");
c1.put("/a/b/c", "age", 39);
System.out.println("[Thread1] set value to 39");
System.out.println("-- t1 releases lock");
lock.release();
assertEquals(39, c1.get("/a/b/c", "age"));
}
catch (Throwable ex)
{
ex.printStackTrace();
t1_ex = ex;
}
finally
{
lock.release();
}
}
};
Thread t2 = new Thread()
{
public void run()
{
try
{
TestingUtil.sleepThread(100);
Thread.yield();
lock.acquire();
System.out.println("-- t2 has lock");
// Should replicate the value right away.
Integer val = (Integer) cache2.get("/a/b/c", "age");
System.out.println("[Thread2] value is " + val);
assertEquals(new Integer(38), val);
System.out.println("-- t2 releases lock");
lock.release();
TestingUtil.sleepThread(300);
Thread.yield();
TestingUtil.sleepThread(500);
lock.acquire();
System.out.println("-- t2 has lock");
val = (Integer) cache2.get("/a/b/c", "age");
System.out.println("-- t2 releases lock");
lock.release();
assertEquals(new Integer(39), val);
}
catch (Throwable ex)
{
ex.printStackTrace();
t2_ex = ex;
}
finally
{
lock.release();
}
}
};
// Let the game start
t1.start();
t2.start();
// Wait for thread to die but put an insurance of 5 seconds on it.
t1.join();
t2.join();
if (t1_ex != null)
{
fail("Thread1 failed: " + t1_ex);
}
if (t2_ex != null)
{
fail("Thread2 failed: " + t2_ex);
}
}
/**
* Test replicated cache with transaction. Idea is to have two threads running
* a local cache each that is replicating. Depending on whether cache1 commit/rollback or not,
* the cache2.get will get different values.
* Note that we have used sleep to interpose thread execution sequence.
* Although it's not fool proof, it is rather simple and intuitive.
*
* @throws Exception
*/
public void testPutTx() throws Exception
{
TransactionManager tm = null;
try
{
initCaches(Configuration.CacheMode.REPL_SYNC);
cache1.getConfiguration().setSyncCommitPhase(true);
cache2.getConfiguration().setSyncCommitPhase(true);
tm = beginTransaction();
cache1.put("/a/b/c", "age", 38);
cache1.put("/a/b/c", "age", 39);
Object val = cache2.get("/a/b/c", "age");// must be null as not yet committed
assertNull(val);
tm.commit();
tm = beginTransaction();
assertEquals(39, cache2.get("/a/b/c", "age"));// must not be null
tm.commit();
}
catch (Throwable t)
{
t.printStackTrace();
t1_ex = t;
}
finally
{
lock.release();
}
}
/**
* Have both cache1 and cache2 do add and commit. cache1 commit should time out
* since it can't obtain the lock when trying to replicate cache2. On the other hand,
* cache2 commit will succeed since now that cache1 is rollbacked and lock is
* released.
*/
public void testPutTx1() throws Exception
{
initCaches(Configuration.CacheMode.REPL_SYNC);
final CacheSPI<Object, Object> c1 = this.cache1;
Thread t1 = new Thread()
{
public void run()
{
TransactionManager tm = null;
try
{
lock.acquire();
tm = beginTransaction();
c1.put("/a/b/c", "age", 38);
c1.put("/a/b/c", "age", 39);
lock.release();
TestingUtil.sleepThread(300);
lock.acquire();
try
{
tm.commit();
}
catch (RollbackException ex)
{
System.out.println("[Thread1] received RollbackException, as expected. Rolling back changes");
}
finally
{
lock.release();
}
}
catch (Throwable ex)
{
ex.printStackTrace();
t1_ex = ex;
}
finally
{
lock.release();
}
}
};
Thread t2 = new Thread()
{
public void run()
{
TransactionManager tm = null;
try
{
sleep(200);
Thread.yield();
lock.acquire();
tm = beginTransaction();
assertNull(cache2.get("/a/b/c", "age"));// must be null as not yet committed
cache2.put("/a/b/c", "age", 40);
lock.release();
TestingUtil.sleepThread(300);
lock.acquire();
assertEquals(40, cache2.get("/a/b/c", "age"));// must not be null
tm.commit();
lock.release();
TestingUtil.sleepThread(1000);
tm = beginTransaction();
assertEquals("After cache2 commit", 40, cache2.get("/a/b/c", "age"));
tm.commit();
}
catch (Throwable ex)
{
ex.printStackTrace();
t2_ex = ex;
}
finally
{
lock.release();
}
}
};
// Let the game start
t1.start();
t2.start();
t1.join();
t2.join();
if (t1_ex != null)
{
fail("Thread1 failed: " + t1_ex);
}
if (t2_ex != null)
{
fail("Thread2 failed: " + t2_ex);
}
}
public void testPutTxWithRollback() throws Exception
{
initCaches(Configuration.CacheMode.REPL_SYNC);
final CacheSPI<Object, Object> c2 = this.cache1;
Thread t1 = new Thread()
{
public void run()
{
TransactionManager tm = null;
try
{
lock.acquire();
tm = beginTransaction();
c2.put("/a/b/c", "age", 38);
c2.put("/a/b/c", "age", 39);
lock.release();
TestingUtil.sleepThread(100);
lock.acquire();
tm.rollback();
lock.release();
}
catch (Throwable ex)
{
ex.printStackTrace();
t1_ex = ex;
}
finally
{
lock.release();
}
}
};
Thread t2 = new Thread()
{
public void run()
{
TransactionManager tm = null;
try
{
sleep(200);
Thread.yield();
lock.acquire();
tm = beginTransaction();
assertNull(cache2.get("/a/b/c", "age"));// must be null as not yet committed
lock.release();
TestingUtil.sleepThread(100);
lock.acquire();
assertNull(cache2.get("/a/b/c", "age"));// must be null as rolledback
tm.commit();
lock.release();
}
catch (Throwable ex)
{
ex.printStackTrace();
t2_ex = ex;
}
finally
{
lock.release();
}
}
};
// Let the game start
t1.start();
t2.start();
// Wait for thread to die but put an insurance of 5 seconds on it.
t1.join();
t2.join();
if (t1_ex != null)
{
fail("Thread1 failed: " + t1_ex);
}
if (t2_ex != null)
{
fail("Thread2 failed: " + t2_ex);
}
}
static class TransactionAborter implements Synchronization
{
Transaction ltx = null;
public TransactionAborter(Transaction ltx)
{
this.ltx = ltx;
}
public void beforeCompletion()
{
try
{
ltx.setRollbackOnly();
}
catch (SystemException e)
{
// who cares
}
}
public void afterCompletion(int status)
{
}
}
@CacheListener
static class CallbackListener
{
CacheSPI<Object, Object> callbackCache;
Object callbackKey;
Exception ex;
Object mutex = new Object();
CallbackListener(CacheSPI<Object, Object> cache, Object callbackKey)
{
this.callbackCache = cache;
this.callbackKey = callbackKey;
cache.getNotifier().addCacheListener(this);
}
@NodeModified
public void nodeModified(NodeEvent e)
{
if (!e.isPre())
{
// Lock on a mutex so test can't check for an exception
// until the get call completes
synchronized (mutex)
{
try
{
callbackCache.get(e.getFqn(), callbackKey);
}
catch (CacheException exc)
{
exc.printStackTrace();
ex = exc;
}
}
}
}
Exception getCallbackException()
{
synchronized (mutex)
{
return ex;
}
}
}
static class TransactionAborterCallbackListener extends CallbackListener
{
TransactionManager callbackTM;
TransactionAborterCallbackListener(CacheSPI<Object, Object> cache, Object callbackKey)
{
super(cache, callbackKey);
callbackTM = callbackCache.getTransactionManager();
}
@NodeModified
public void nodeModified(NodeEvent ne)
{
if (!ne.isPre())
{
try
{
Transaction tx = callbackTM.getTransaction();
if (tx != null && tx.getStatus() == Status.STATUS_ACTIVE)
{
// this will rollback the transaction
tx.registerSynchronization(new TransactionAborter(tx));
}
else
{
super.nodeModified(ne);
}
}
catch (Exception e)
{
e.printStackTrace();
if (ex == null)
{
ex = e;
}
}
}
}
}
}