In a lot of clustering test jobs, we are seeing Infinispan rethrowing SuspectException: 18:48:05,695 ERROR [org.infinispan.interceptors.InvocationContextInterceptor] (EJB default - 6) ISPN000136: Execution error: org.infinispan.CacheException: java.util.concurrent.ExecutionException: SuspectedException at org.infinispan.util.Util.rewrapAsCacheException(Util.java:541) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommands(CommandAwareRpcDispatcher.java:153) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:518) at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:175) at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:197) at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:254) at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:241) at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:236) at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:231) at org.infinispan.interceptors.distribution.TxDistributionInterceptor.prepareOnAffectedNodes(TxDistributionInterceptor.java:207) at org.infinispan.interceptors.distribution.TxDistributionInterceptor.visitPrepareCommand(TxDistributionInterceptor.java:194) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:132) at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:126) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:132) at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:126) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118) at org.infinispan.interceptors.EntryWrappingInterceptor.visitPrepareCommand(EntryWrappingInterceptor.java:106) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118) at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.invokeNextAndCommitIf1Pc(AbstractTxLockingInterceptor.java:107) at org.infinispan.interceptors.locking.OptimisticLockingInterceptor.visitPrepareCommand(OptimisticLockingInterceptor.java:135) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118) at org.infinispan.interceptors.NotificationInterceptor.visitPrepareCommand(NotificationInterceptor.java:58) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:132) at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:126) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118) at org.infinispan.interceptors.TxInterceptor.invokeNextInterceptorAndVerifyTransaction(TxInterceptor.java:128) at org.infinispan.interceptors.TxInterceptor.visitPrepareCommand(TxInterceptor.java:115) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:132) at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:126) at org.infinispan.statetransfer.TransactionSynchronizerInterceptor.visitPrepareCommand(TransactionSynchronizerInterceptor.java:61) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118) at org.infinispan.statetransfer.StateTransferInterceptor.handleTopologyAffectedCommand(StateTransferInterceptor.java:216) at org.infinispan.statetransfer.StateTransferInterceptor.handleTxCommand(StateTransferInterceptor.java:189) at org.infinispan.statetransfer.StateTransferInterceptor.visitPrepareCommand(StateTransferInterceptor.java:93) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:132) at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:126) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118) at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:128) at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:92) at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:126) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:132) at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:126) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118) at org.infinispan.interceptors.BatchingInterceptor.handleDefault(BatchingInterceptor.java:85) at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:126) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:343) at org.infinispan.transaction.TransactionCoordinator.prepare(TransactionCoordinator.java:141) at org.infinispan.transaction.TransactionCoordinator.prepare(TransactionCoordinator.java:123) at org.infinispan.transaction.synchronization.SynchronizationAdapter.beforeCompletion(SynchronizationAdapter.java:68) at org.infinispan.transaction.tm.DummyTransaction.notifyBeforeCompletion(DummyTransaction.java:230) at org.infinispan.transaction.tm.DummyTransaction.runPrepare(DummyTransaction.java:242) at org.infinispan.transaction.tm.DummyTransaction.commit(DummyTransaction.java:86) at org.infinispan.transaction.tm.DummyBaseTransactionManager.commit(DummyBaseTransactionManager.java:102) at org.infinispan.batch.BatchContainer.resolveTransaction(BatchContainer.java:123) at org.infinispan.batch.BatchContainer.endBatch(BatchContainer.java:105) at org.infinispan.batch.BatchContainer.endBatch(BatchContainer.java:86) at org.infinispan.CacheImpl.endBatch(CacheImpl.java:661) at org.infinispan.AbstractDelegatingCache.endBatch(AbstractDelegatingCache.java:80) at org.jboss.as.clustering.infinispan.invoker.BatchCacheInvoker.invoke(BatchCacheInvoker.java:53) at org.jboss.as.clustering.infinispan.invoker.RetryingCacheInvoker.invoke(RetryingCacheInvoker.java:81) at org.jboss.as.clustering.ejb3.cache.backing.infinispan.InfinispanBackingCacheEntryStore.update(InfinispanBackingCacheEntryStore.java:214) at org.jboss.as.ejb3.cache.impl.backing.SerializationGroupMemberContainer.update(SerializationGroupMemberContainer.java:280) [jboss-as-ejb3-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.jboss.as.ejb3.cache.impl.backing.SerializationGroupMemberContainer.update(SerializationGroupMemberContainer.java:51) [jboss-as-ejb3-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.jboss.as.ejb3.cache.impl.backing.PassivatingBackingCacheImpl.release(PassivatingBackingCacheImpl.java:227) [jboss-as-ejb3-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.jboss.as.ejb3.cache.spi.impl.AbstractCache.release(AbstractCache.java:92) [jboss-as-ejb3-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.jboss.as.ejb3.cache.spi.impl.AbstractCache.release(AbstractCache.java:39) [jboss-as-ejb3-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.jboss.as.ejb3.component.stateful.StatefulSessionSynchronizationInterceptor.releaseInstance(StatefulSessionSynchronizationInterceptor.java:199) [jboss-as-ejb3-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.jboss.as.ejb3.component.stateful.StatefulSessionSynchronizationInterceptor.access$200(StatefulSessionSynchronizationInterceptor.java:50) [jboss-as-ejb3-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.jboss.as.ejb3.component.stateful.StatefulSessionSynchronizationInterceptor$StatefulSessionSynchronization.afterCompletion(StatefulSessionSynchronizationInterceptor.java:273) [jboss-as-ejb3-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at com.arjuna.ats.internal.jta.resources.arjunacore.SynchronizationImple.afterCompletion(SynchronizationImple.java:96) at com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.afterCompletion(TwoPhaseCoordinator.java:402) at com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.end(TwoPhaseCoordinator.java:103) at com.arjuna.ats.arjuna.AtomicAction.commit(AtomicAction.java:162) at com.arjuna.ats.internal.jta.transaction.arjunacore.TransactionImple.commitAndDisassociate(TransactionImple.java:1165) at com.arjuna.ats.internal.jta.transaction.arjunacore.BaseTransaction.commit(BaseTransaction.java:126) at com.arjuna.ats.jbossatx.BaseTransactionManagerDelegate.commit(BaseTransactionManagerDelegate.java:75) at org.jboss.as.ejb3.tx.CMTTxInterceptor.endTransaction(CMTTxInterceptor.java:91) [jboss-as-ejb3-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.jboss.as.ejb3.tx.CMTTxInterceptor.invokeInOurTx(CMTTxInterceptor.java:252) [jboss-as-ejb3-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.jboss.as.ejb3.tx.CMTTxInterceptor.required(CMTTxInterceptor.java:315) [jboss-as-ejb3-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.jboss.as.ejb3.tx.CMTTxInterceptor.processInvocation(CMTTxInterceptor.java:214) [jboss-as-ejb3-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:288) [jboss-invocation-1.1.1.Final-redhat-2.jar:1.1.1.Final-redhat-2] at org.jboss.as.ejb3.remote.EJBRemoteTransactionPropagatingInterceptor.processInvocation(EJBRemoteTransactionPropagatingInterceptor.java:79) [jboss-as-ejb3-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:288) [jboss-invocation-1.1.1.Final-redhat-2.jar:1.1.1.Final-redhat-2] at org.jboss.as.ejb3.component.interceptors.CurrentInvocationContextInterceptor.processInvocation(CurrentInvocationContextInterceptor.java:41) [jboss-as-ejb3-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:288) [jboss-invocation-1.1.1.Final-redhat-2.jar:1.1.1.Final-redhat-2] at org.jboss.as.ejb3.component.interceptors.ShutDownInterceptorFactory$1.processInvocation(ShutDownInterceptorFactory.java:64) [jboss-as-ejb3-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:288) [jboss-invocation-1.1.1.Final-redhat-2.jar:1.1.1.Final-redhat-2] at org.jboss.as.ejb3.component.interceptors.LoggingInterceptor.processInvocation(LoggingInterceptor.java:59) [jboss-as-ejb3-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:288) [jboss-invocation-1.1.1.Final-redhat-2.jar:1.1.1.Final-redhat-2] at org.jboss.as.ee.component.NamespaceContextInterceptor.processInvocation(NamespaceContextInterceptor.java:50) [jboss-as-ee-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:288) [jboss-invocation-1.1.1.Final-redhat-2.jar:1.1.1.Final-redhat-2] at org.jboss.as.ejb3.component.interceptors.AdditionalSetupInterceptor.processInvocation(AdditionalSetupInterceptor.java:65) [jboss-as-ejb3-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:288) [jboss-invocation-1.1.1.Final-redhat-2.jar:1.1.1.Final-redhat-2] at org.jboss.as.ee.component.TCCLInterceptor.processInvocation(TCCLInterceptor.java:45) [jboss-as-ee-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:288) [jboss-invocation-1.1.1.Final-redhat-2.jar:1.1.1.Final-redhat-2] at org.jboss.invocation.ChainedInterceptor.processInvocation(ChainedInterceptor.java:61) [jboss-invocation-1.1.1.Final-redhat-2.jar:1.1.1.Final-redhat-2] at org.jboss.as.ee.component.ViewService$View.invoke(ViewService.java:165) [jboss-as-ee-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.jboss.as.ejb3.remote.protocol.versionone.MethodInvocationMessageHandler.invokeMethod(MethodInvocationMessageHandler.java:321) [jboss-as-ejb3-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.jboss.as.ejb3.remote.protocol.versionone.MethodInvocationMessageHandler.access$100(MethodInvocationMessageHandler.java:69) [jboss-as-ejb3-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.jboss.as.ejb3.remote.protocol.versionone.MethodInvocationMessageHandler$1.run(MethodInvocationMessageHandler.java:202) [jboss-as-ejb3-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:439) [rt.jar:1.6.0_38] at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303) [rt.jar:1.6.0_38] at java.util.concurrent.FutureTask.run(FutureTask.java:138) [rt.jar:1.6.0_38] at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) [rt.jar:1.6.0_38] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) [rt.jar:1.6.0_38] at java.lang.Thread.run(Thread.java:662) [rt.jar:1.6.0_38] at org.jboss.threads.JBossThread.run(JBossThread.java:122) Caused by: java.util.concurrent.ExecutionException: SuspectedException at org.jgroups.blocks.UnicastRequest.getValue(UnicastRequest.java:195) at org.jgroups.blocks.UnicastRequest.get(UnicastRequest.java:230) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.processCalls(CommandAwareRpcDispatcher.java:365) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommands(CommandAwareRpcDispatcher.java:146) ... 125 more Caused by: SuspectedException ... 129 more Seen in https://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/eap-6x-failover-ejb-ejbremote-shutdown-dist-sync-3owners/6/console-perf21/ 04:08:54,801 ERROR [org.infinispan.interceptors.InvocationContextInterceptor] (ajp-/2620:52:0:105f:0:0:ffff:40%2:8009-10) ISPN000136: Execution error: org.infinispan.remoting.transport.jgroups.SuspectException: Node dev86/web was suspected at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:184) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:515) at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:175) at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:197) at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:254) at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:241) at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:236) at org.infinispan.remoting.rpc.RpcManagerImpl.broadcastRpcCommand(RpcManagerImpl.java:215) at org.infinispan.interceptors.ReplicationInterceptor.broadcastPrepare(ReplicationInterceptor.java:131) at org.infinispan.interceptors.ReplicationInterceptor.visitPrepareCommand(ReplicationInterceptor.java:123) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:132) at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:126) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:132) at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:126) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118) at org.infinispan.interceptors.EntryWrappingInterceptor.visitPrepareCommand(EntryWrappingInterceptor.java:106) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118) at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.invokeNextAndCommitIf1Pc(AbstractTxLockingInterceptor.java:107) at org.infinispan.interceptors.locking.OptimisticLockingInterceptor.visitPrepareCommand(OptimisticLockingInterceptor.java:135) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118) at org.infinispan.interceptors.NotificationInterceptor.visitPrepareCommand(NotificationInterceptor.java:58) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118) at org.infinispan.interceptors.TxInterceptor.invokeNextInterceptorAndVerifyTransaction(TxInterceptor.java:128) at org.infinispan.interceptors.TxInterceptor.visitPrepareCommand(TxInterceptor.java:115) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:132) at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:126) at org.infinispan.statetransfer.TransactionSynchronizerInterceptor.visitPrepareCommand(TransactionSynchronizerInterceptor.java:61) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118) at org.infinispan.statetransfer.StateTransferInterceptor.handleTopologyAffectedCommand(StateTransferInterceptor.java:216) at org.infinispan.statetransfer.StateTransferInterceptor.handleTxCommand(StateTransferInterceptor.java:189) at org.infinispan.statetransfer.StateTransferInterceptor.visitPrepareCommand(StateTransferInterceptor.java:93) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:132) at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:126) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118) at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:128) at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:92) at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:126) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118) at org.infinispan.interceptors.BatchingInterceptor.handleDefault(BatchingInterceptor.java:85) at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:126) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:343) at org.infinispan.transaction.TransactionCoordinator.prepare(TransactionCoordinator.java:141) at org.infinispan.transaction.TransactionCoordinator.prepare(TransactionCoordinator.java:123) at org.infinispan.transaction.synchronization.SynchronizationAdapter.beforeCompletion(SynchronizationAdapter.java:68) at org.infinispan.transaction.tm.DummyTransaction.notifyBeforeCompletion(DummyTransaction.java:230) at org.infinispan.transaction.tm.DummyTransaction.runPrepare(DummyTransaction.java:242) at org.infinispan.transaction.tm.DummyTransaction.commit(DummyTransaction.java:86) at org.infinispan.transaction.tm.DummyBaseTransactionManager.commit(DummyBaseTransactionManager.java:102) at org.jboss.as.clustering.web.impl.TransactionBatchingManager.endBatch(TransactionBatchingManager.java:75) at org.jboss.as.web.session.DistributableSessionManager.processSessionRepl(DistributableSessionManager.java:1527) [jboss-as-web-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.jboss.as.web.session.DistributableSessionManager.storeSession(DistributableSessionManager.java:865) [jboss-as-web-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.jboss.as.web.session.InstantSnapshotManager.snapshot(InstantSnapshotManager.java:47) [jboss-as-web-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.jboss.as.web.session.ClusteredSessionValve.handleRequest(ClusteredSessionValve.java:142) [jboss-as-web-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.jboss.as.web.session.ClusteredSessionValve.invoke(ClusteredSessionValve.java:99) [jboss-as-web-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.jboss.as.web.session.JvmRouteValve.invoke(JvmRouteValve.java:92) [jboss-as-web-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.jboss.as.web.session.LockingValve.invoke(LockingValve.java:64) [jboss-as-web-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.jboss.as.web.security.SecurityContextAssociationValve.invoke(SecurityContextAssociationValve.java:169) [jboss-as-web-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:145) [jbossweb-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:97) [jbossweb-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:102) [jbossweb-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:336) [jbossweb-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.apache.coyote.ajp.AjpProcessor.process(AjpProcessor.java:488) [jbossweb-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.apache.coyote.ajp.AjpProtocol$AjpConnectionHandler.process(AjpProtocol.java:420) [jbossweb-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:920) [jbossweb-7.2.0.Final-redhat-1.jar:7.2.0.Final-redhat-1] at java.lang.Thread.run(Thread.java:662) [rt.jar:1.6.0_38] Caused by: SuspectedException at org.jgroups.blocks.MessageDispatcher.sendMessage(MessageDispatcher.java:377) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.processSingleCall(CommandAwareRpcDispatcher.java:301) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:179) ... 80 more Seen in https://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/eap-6x-failover-ejb-ejbservlet-jvmkill-repl-sync-modcluster-dualstack-ipv6/11/console-dev89/ It can also be seen in Infinispan TS (see bug 911120). This happens right after one node in the cluster fails and this node receives new view. It looks like it's caused by a remote call on the failing node that couldn't finish.
I believe the correct component here is clustering.
Can you please elaborate more on this bug? This is a negative test with JVM kill where SuspectedException are expected.
There is a plan to silence most of the SuspectedExceptions in Infinispan 5.3.x, see: https://issues.jboss.org/browse/ISPN-2402 https://issues.jboss.org/browse/ISPN-1896 https://issues.jboss.org/browse/ISPN-2577 etc
I've realized now that there is still 'needsinfo' for me. The thing is that -- and by now, I think that everyone already knows -- Infinispan started to rethrow SuspectExceptions much more often than in the past, which pollutes the logs with unnecessary errors. Since Infinispan team already wants to address this in 5.3.0, I think that we don't have to do anything here, just wait.
Still seeing this with EAP 6.1.0.ER8 (and I know that this is expected). For example: https://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/eap-6x-failover-http-session-undeploy-repl-sync/54/artifact/report/config/jboss-perf21/server.log https://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/eap-6x-failover-http-session-jvmkill-repl-sync/53/artifact/report/config/jboss-perf18/server.log
Still seeing this with EAP 6.1.1.ER7 (and I know it's expected). For example: https://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/eap-6x-failover-http-session-undeploy-repl-sync/60/ https://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/eap-6x-failover-http-session-jvmkill-repl-sync/57/
The RetryCacheInvoker already retries on SuspectException, however, if the SuspectException is wrapped as some other exception it slips through the retry logic. We should be able to handle this in the RetryCacheInvoker.
https://github.com/jbossas/jboss-eap/pull/467
I'm still seeing this with EAP 6.2.0.ER6, Infinispan 5.2.7.Final and all of our node removal scenarios: jvmkill, network failure, shutdown and undeploy. It's mostly sync cache, but one async configuration as well. All the configurations that I've seen this in: http-session-undeploy-repl-sync http-granular-shutdown-repl-sync http-session-jvmkill-dist-sync http-session-jvmkill-repl-sync http-session-netDown-dist-sync http-session-netDown-repl-async http-session-netDown-repl-sync http-session-shutdown-repl-sync ejb-ejbservlet-jvmkill-dist-sync ejb-ejbservlet-shutdown-repl-sync ejb-ejbservlet-undeploy-repl-sync ejb-ejbremote-jvmkill-repl-sync ejb-ejbremote-netDown-dist-sync ejb-ejbremote-undeploy-repl-sync https://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/eap-6x-failover-http-session-undeploy-repl-sync/66/ https://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/eap-6x-failover-http-granular-shutdown-repl-sync/33/ https://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/eap-6x-failover-http-session-jvmkill-dist-sync/55/ https://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/eap-6x-failover-http-session-jvmkill-repl-sync/63/ https://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/eap-6x-failover-http-session-netDown-dist-sync/21/ https://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/eap-6x-failover-http-session-netDown-repl-async/22/ https://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/eap-6x-failover-http-session-netDown-repl-sync/22/ https://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/eap-6x-failover-http-session-shutdown-repl-sync/96/ https://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/eap-6x-failover-ejb-ejbservlet-jvmkill-dist-sync/23/ https://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/eap-6x-failover-ejb-ejbservlet-shutdown-repl-sync/22/ https://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/eap-6x-failover-ejb-ejbservlet-undeploy-repl-sync/43/ https://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/eap-6x-failover-ejb-ejbremote-jvmkill-repl-sync/39/ https://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/eap-6x-failover-ejb-ejbremote-netDown-dist-sync/31/ https://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/eap-6x-failover-ejb-ejbremote-undeploy-repl-sync/37/ Sample stacktrace: [JBossINF] [0m[0m17:28:33,757 INFO [org.jboss.as.clustering] (Incoming-11,shared=udp) JBAS010226: New cluster view for partition web: 4 (org.jboss.as.clustering.impl.CoreGroupCommunicationService$GroupView@4ff2919 delta: -1, merge: false) [JBossINF] [0m[0m17:28:33,758 INFO [org.infinispan.remoting.transport.jgroups.JGroupsTransport] (Incoming-11,shared=udp) ISPN000094: Received new cluster view: [perf19/web|4] [perf19/web, perf21/web, perf20/web] [JBossINF] [0m[31m17:28:33,760 ERROR [org.infinispan.interceptors.InvocationContextInterceptor] (ajp-perf20/10.16.90.58:8009-28) ISPN000136: Execution error: org.infinispan.remoting.transport.jgroups.SuspectException: Suspected member: perf18/web [JBossINF] at org.infinispan.remoting.transport.AbstractTransport.parseResponseAndAddToResponseList(AbstractTransport.java:105) [JBossINF] at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:541) [JBossINF] at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:169) [JBossINF] at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:190) [JBossINF] at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:247) [JBossINF] at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:234) [JBossINF] at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:229) [JBossINF] at org.infinispan.remoting.rpc.RpcManagerImpl.broadcastRpcCommand(RpcManagerImpl.java:208) [JBossINF] at org.infinispan.interceptors.ReplicationInterceptor.broadcastPrepare(ReplicationInterceptor.java:109) [JBossINF] at org.infinispan.interceptors.ReplicationInterceptor.visitPrepareCommand(ReplicationInterceptor.java:101) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:120) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:134) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:126) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:120) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:134) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:126) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:120) [JBossINF] at org.infinispan.interceptors.EntryWrappingInterceptor.visitPrepareCommand(EntryWrappingInterceptor.java:106) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:120) [JBossINF] at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.invokeNextAndCommitIf1Pc(AbstractTxLockingInterceptor.java:108) [JBossINF] at org.infinispan.interceptors.locking.OptimisticLockingInterceptor.visitPrepareCommand(OptimisticLockingInterceptor.java:135) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:120) [JBossINF] at org.infinispan.interceptors.NotificationInterceptor.visitPrepareCommand(NotificationInterceptor.java:58) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:120) [JBossINF] at org.infinispan.interceptors.TxInterceptor.invokeNextInterceptorAndVerifyTransaction(TxInterceptor.java:128) [JBossINF] at org.infinispan.interceptors.TxInterceptor.visitPrepareCommand(TxInterceptor.java:115) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:120) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:134) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:126) [JBossINF] at org.infinispan.statetransfer.TransactionSynchronizerInterceptor.visitPrepareCommand(TransactionSynchronizerInterceptor.java:61) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:120) [JBossINF] at org.infinispan.statetransfer.StateTransferInterceptor.handleTopologyAffectedCommand(StateTransferInterceptor.java:216) [JBossINF] at org.infinispan.statetransfer.StateTransferInterceptor.handleTxCommand(StateTransferInterceptor.java:189) [JBossINF] at org.infinispan.statetransfer.StateTransferInterceptor.visitPrepareCommand(StateTransferInterceptor.java:93) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:120) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:134) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:126) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:120) [JBossINF] at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:128) [JBossINF] at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:92) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:126) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:120) [JBossINF] at org.infinispan.interceptors.BatchingInterceptor.handleDefault(BatchingInterceptor.java:92) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:126) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) [JBossINF] at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:343) [JBossINF] at org.infinispan.transaction.TransactionCoordinator.prepare(TransactionCoordinator.java:141) [JBossINF] at org.infinispan.transaction.TransactionCoordinator.prepare(TransactionCoordinator.java:123) [JBossINF] at org.infinispan.transaction.synchronization.SynchronizationAdapter.beforeCompletion(SynchronizationAdapter.java:68) [JBossINF] at org.infinispan.transaction.tm.DummyTransaction.notifyBeforeCompletion(DummyTransaction.java:230) [JBossINF] at org.infinispan.transaction.tm.DummyTransaction.runPrepare(DummyTransaction.java:242) [JBossINF] at org.infinispan.transaction.tm.DummyTransaction.commit(DummyTransaction.java:86) [JBossINF] at org.infinispan.transaction.tm.DummyBaseTransactionManager.commit(DummyBaseTransactionManager.java:102) [JBossINF] at org.jboss.as.clustering.web.impl.TransactionBatchingManager.endBatch(TransactionBatchingManager.java:75) [JBossINF] at org.jboss.as.web.session.DistributableSessionManager.processSessionRepl(DistributableSessionManager.java:1564) [jboss-as-web-7.3.0.Final-redhat-9.jar:7.3.0.Final-redhat-9] [JBossINF] at org.jboss.as.web.session.DistributableSessionManager.storeSession(DistributableSessionManager.java:865) [jboss-as-web-7.3.0.Final-redhat-9.jar:7.3.0.Final-redhat-9] [JBossINF] at org.jboss.as.web.session.InstantSnapshotManager.snapshot(InstantSnapshotManager.java:47) [jboss-as-web-7.3.0.Final-redhat-9.jar:7.3.0.Final-redhat-9] [JBossINF] at org.jboss.as.web.session.ClusteredSessionValve.handleRequest(ClusteredSessionValve.java:142) [jboss-as-web-7.3.0.Final-redhat-9.jar:7.3.0.Final-redhat-9] [JBossINF] at org.jboss.as.web.session.ClusteredSessionValve.invoke(ClusteredSessionValve.java:99) [jboss-as-web-7.3.0.Final-redhat-9.jar:7.3.0.Final-redhat-9] [JBossINF] at org.jboss.as.web.session.JvmRouteValve.invoke(JvmRouteValve.java:92) [jboss-as-web-7.3.0.Final-redhat-9.jar:7.3.0.Final-redhat-9] [JBossINF] at org.jboss.as.web.session.LockingValve.invoke(LockingValve.java:64) [jboss-as-web-7.3.0.Final-redhat-9.jar:7.3.0.Final-redhat-9] [JBossINF] at org.jboss.as.web.security.SecurityContextAssociationValve.invoke(SecurityContextAssociationValve.java:169) [jboss-as-web-7.3.0.Final-redhat-9.jar:7.3.0.Final-redhat-9] [JBossINF] at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:145) [jbossweb-7.2.2.Final-redhat-1.jar:7.2.2.Final-redhat-1] [JBossINF] at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:97) [jbossweb-7.2.2.Final-redhat-1.jar:7.2.2.Final-redhat-1] [JBossINF] at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:102) [jbossweb-7.2.2.Final-redhat-1.jar:7.2.2.Final-redhat-1] [JBossINF] at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:336) [jbossweb-7.2.2.Final-redhat-1.jar:7.2.2.Final-redhat-1] [JBossINF] at org.apache.coyote.ajp.AjpProcessor.process(AjpProcessor.java:488) [jbossweb-7.2.2.Final-redhat-1.jar:7.2.2.Final-redhat-1] [JBossINF] at org.apache.coyote.ajp.AjpProtocol$AjpConnectionHandler.process(AjpProtocol.java:420) [jbossweb-7.2.2.Final-redhat-1.jar:7.2.2.Final-redhat-1] [JBossINF] at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:920) [jbossweb-7.2.2.Final-redhat-1.jar:7.2.2.Final-redhat-1] [JBossINF] at java.lang.Thread.run(Thread.java:662) [rt.jar:1.6.0_45] https://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/eap-6x-failover-http-session-undeploy-repl-sync/66/console-perf20/ https://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/eap-6x-failover-http-session-undeploy-repl-sync/66/console-perf21/
Still an issue, moving to 6.4.
Nacking this. This needs to be fixed in Infinispan and there are no plans to do this for 5.2.x.
Paul is right, this requires a fix in Infinispan. We have a partial fix in 7.0 that catches SuspectException and retries the operation, but it only covers non-transactional caches.
Dan Berindei <dberinde> updated the status of jira ISPN-2577 to Resolved
The needinfo request[s] on this closed bug have been removed as they have been unresolved for 1000 days