Description of problem:

org.infinispan.distribution.rehash.ConcurrentNonOverlappingLeaveTest.testTransactional and org.infinispan.distribution.rehash.ConcurrentOverlappingLeaveTest.testTransactional fail randomly with

Error Message

java.lang.RuntimeException: javax.transaction.RollbackException: ARJUNA016053: Could not commit transaction.

Stacktrace

java.util.concurrent.ExecutionException: java.lang.RuntimeException: javax.transaction.RollbackException: ARJUNA016053: Could not commit transaction.
	at java.util.concurrent.FutureTask.report(FutureTask.java:122)
	at java.util.concurrent.FutureTask.get(FutureTask.java:202)
	at org.infinispan.distribution.rehash.RehashTestBase.testTransactional(RehashTestBase.java:132)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:80)
	at org.testng.internal.Invoker.invokeMethod(Invoker.java:714)
	at org.testng.internal.Invoker.invokeTestMethod(Invoker.java:901)
	at org.testng.internal.Invoker.invokeTestMethods(Invoker.java:1231)
	at org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:127)
	at org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:111)
	at org.testng.TestRunner.privateRun(TestRunner.java:767)
	at org.testng.TestRunner.run(TestRunner.java:617)
	at org.testng.SuiteRunner.runTest(SuiteRunner.java:334)
	at org.testng.SuiteRunner.access$000(SuiteRunner.java:37)
	at org.testng.SuiteRunner$SuiteWorker.run(SuiteRunner.java:368)
	at org.testng.internal.thread.ThreadUtil$2.call(ThreadUtil.java:64)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.RuntimeException: javax.transaction.RollbackException: ARJUNA016053: Could not commit transaction.
	at org.infinispan.distribution.rehash.RehashTestBase$1.run(RehashTestBase.java:124)
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
	at org.infinispan.test.AbstractInfinispanTest$LoggingCallable.call(AbstractInfinispanTest.java:200)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	... 1 more
Caused by: javax.transaction.RollbackException: ARJUNA016053: Could not commit transaction.
	at com.arjuna.ats.internal.jta.transaction.arjunacore.TransactionImple.commitAndDisassociate(TransactionImple.java:1201)
	at com.arjuna.ats.internal.jta.transaction.arjunacore.BaseTransaction.commit(BaseTransaction.java:126)
	at org.infinispan.distribution.rehash.RehashTestBase$1.run(RehashTestBase.java:120)
	... 4 more
Caused by: org.infinispan.commons.CacheException: Could not prepare. 
	at org.infinispan.transaction.synchronization.SynchronizationAdapter.beforeCompletion(SynchronizationAdapter.java:46)
	at com.arjuna.ats.internal.jta.resources.arjunacore.SynchronizationImple.beforeCompletion(SynchronizationImple.java:76)
	at com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.beforeCompletion(TwoPhaseCoordinator.java:273)
	at com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.end(TwoPhaseCoordinator.java:93)
	at com.arjuna.ats.arjuna.AtomicAction.commit(AtomicAction.java:162)
	at com.arjuna.ats.internal.jta.transaction.arjunacore.TransactionImple.commitAndDisassociate(TransactionImple.java:1189)
	... 6 more
Caused by: javax.transaction.xa.XAException
	at org.infinispan.transaction.TransactionCoordinator.prepare(TransactionCoordinator.java:142)
	at org.infinispan.transaction.TransactionCoordinator.prepare(TransactionCoordinator.java:104)
	at org.infinispan.transaction.synchronization.SynchronizationAdapter.beforeCompletion(SynchronizationAdapter.java:44)
	... 11 more
Caused by: org.infinispan.remoting.RemoteException: ISPN000217: Received exception from ConcurrentNonOverlappingLeaveTest-NodeH-51029, see cause for remote stack trace
	at org.infinispan.remoting.transport.AbstractTransport.checkResponse(AbstractTransport.java:46)
	at org.infinispan.remoting.transport.AbstractTransport.parseResponseAndAddToResponseList(AbstractTransport.java:71)
	at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:590)
	at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:292)
	at org.infinispan.interceptors.distribution.TxDistributionInterceptor.prepareOnAffectedNodes(TxDistributionInterceptor.java:223)
	at org.infinispan.interceptors.distribution.TxDistributionInterceptor.visitPrepareCommand(TxDistributionInterceptor.java:207)
	at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:125)
	at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98)
	at org.infinispan.interceptors.distribution.L1TxInterceptor.visitPrepareCommand(L1TxInterceptor.java:53)
	at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:125)
	at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98)
	at org.infinispan.interceptors.EntryWrappingInterceptor.visitPrepareCommand(EntryWrappingInterceptor.java:101)
	at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:125)
	at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98)
	at org.infinispan.interceptors.distribution.L1LastChanceInterceptor.visitPrepareCommand(L1LastChanceInterceptor.java:121)
	at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:125)
	at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98)
	at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.invokeNextAndCommitIf1Pc(AbstractTxLockingInterceptor.java:95)
	at org.infinispan.interceptors.locking.OptimisticLockingInterceptor.visitPrepareCommand(OptimisticLockingInterceptor.java:88)
	at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:125)
	at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98)
	at org.infinispan.interceptors.NotificationInterceptor.visitPrepareCommand(NotificationInterceptor.java:36)
	at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:125)
	at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98)
	at org.infinispan.interceptors.TxInterceptor.invokeNextInterceptorAndVerifyTransaction(TxInterceptor.java:134)
	at org.infinispan.interceptors.TxInterceptor.visitPrepareCommand(TxInterceptor.java:121)
	at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:125)
	at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98)
	at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:112)
	at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:117)
	at org.infinispan.statetransfer.TransactionSynchronizerInterceptor.visitPrepareCommand(TransactionSynchronizerInterceptor.java:42)
	at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:125)
	at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98)
	at org.infinispan.statetransfer.StateTransferInterceptor.handleTxCommand(StateTransferInterceptor.java:215)
	at org.infinispan.statetransfer.StateTransferInterceptor.visitPrepareCommand(StateTransferInterceptor.java:103)
	at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:125)
	at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98)
	at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:112)
	at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:117)
	at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:125)
	at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98)
	at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:104)
	at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:73)
	at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:117)
	at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:125)
	at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98)
	at org.infinispan.interceptors.BatchingInterceptor.handleDefault(BatchingInterceptor.java:66)
	at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:117)
	at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:125)
	at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:336)
	at org.infinispan.transaction.TransactionCoordinator.prepare(TransactionCoordinator.java:122)
	... 13 more
Caused by: org.infinispan.IllegalLifecycleStateException: ISPN000323: Cache 'dist' is in 'TERMINATED' state and so it does not accept new invocations. Either restart it or recreate the cache container.
	at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:91)
	at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:73)
	at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:117)
	at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:125)
	at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98)
	at org.infinispan.interceptors.BatchingInterceptor.handleDefault(BatchingInterceptor.java:58)
	at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:117)
	at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:125)
	at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:336)
	at org.infinispan.commands.tx.PrepareCommand.perform(PrepareCommand.java:115)
	at org.infinispan.remoting.InboundInvocationHandlerImpl.handleInternal(InboundInvocationHandlerImpl.java:92)
	at org.infinispan.remoting.InboundInvocationHandlerImpl.access$000(InboundInvocationHandlerImpl.java:47)
	at org.infinispan.remoting.InboundInvocationHandlerImpl$2.run(InboundInvocationHandlerImpl.java:185)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more