Bug 1255652 - DistributedIntermediateSharedCacheFourNodesMapReduceTest and DistributedSharedCacheTwoNodesMapReduceTest fails with timeout exception
DistributedIntermediateSharedCacheFourNodesMapReduceTest and DistributedShare...
Status: NEW
Product: JBoss Data Grid 6
Classification: JBoss
Component: TestSuite (Show other bugs)
6.5.1
Unspecified Unspecified
unspecified Severity unspecified
: ---
: ---
Assigned To: Martin Gencur
Martin Gencur
:
Depends On:
Blocks:
  Show dependency treegraph
 
Reported: 2015-08-21 05:19 EDT by Vojtech Juranek
Modified: 2015-08-21 05:49 EDT (History)
2 users (show)

See Also:
Fixed In Version:
Doc Type: Bug Fix
Doc Text:
Story Points: ---
Clone Of:
Environment:
Last Closed:
Type: Bug
Regression: ---
Mount Type: ---
Documentation: ---
CRM:
Verified Versions:
Category: ---
oVirt Team: ---
RHEL 7.3 requirements from Atomic Host:
Cloudforms Team: ---


Attachments (Terms of Use)

  None (edit)
Description Vojtech Juranek 2015-08-21 05:19:36 EDT
org.infinispan.commons.CacheException: java.util.concurrent.ExecutionException: org.infinispan.commons.CacheException: org.infinispan.commons.CacheException: Could not move intermediate keys/values for M/R task 65ba6afd-e70f-4b00-a07b-0de489f3c0ec
	at org.infinispan.distexec.mapreduce.MapReduceTask.executeHelper(MapReduceTask.java:482)
	at org.infinispan.distexec.mapreduce.MapReduceTask.execute(MapReduceTask.java:412)
	at org.infinispan.distexec.mapreduce.BaseWordCountMapReduceTest.testInvokeMapReduceOnAllKeysWithCombiner(BaseWordCountMapReduceTest.java:188)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
	at java.lang.reflect.Method.invoke(Method.java:597)
	at org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:80)
	at org.testng.internal.Invoker.invokeMethod(Invoker.java:714)
	at org.testng.internal.Invoker.invokeTestMethod(Invoker.java:901)
	at org.testng.internal.Invoker.invokeTestMethods(Invoker.java:1231)
	at org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:127)
	at org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:111)
	at org.testng.TestRunner.privateRun(TestRunner.java:767)
	at org.testng.TestRunner.run(TestRunner.java:617)
	at org.testng.SuiteRunner.runTest(SuiteRunner.java:334)
	at org.testng.SuiteRunner.access$000(SuiteRunner.java:37)
	at org.testng.SuiteRunner$SuiteWorker.run(SuiteRunner.java:368)
	at org.testng.internal.thread.ThreadUtil$2.call(ThreadUtil.java:64)
	at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
	at java.util.concurrent.FutureTask.run(FutureTask.java:138)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:895)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:918)
	at java.lang.Thread.run(Thread.java:662)
Caused by: java.util.concurrent.ExecutionException: org.infinispan.commons.CacheException: org.infinispan.commons.CacheException: Could not move intermediate keys/values for M/R task 65ba6afd-e70f-4b00-a07b-0de489f3c0ec
	at java.util.concurrent.FutureTask$Sync.innerGet(FutureTask.java:222)
	at java.util.concurrent.FutureTask.get(FutureTask.java:83)
	at org.infinispan.distexec.mapreduce.MapReduceTask$TaskPart.get(MapReduceTask.java:1067)
	at org.infinispan.distexec.mapreduce.MapReduceTask.executeMapPhase(MapReduceTask.java:604)
	at org.infinispan.distexec.mapreduce.MapReduceTask.executeHelper(MapReduceTask.java:476)
	... 23 more
Caused by: org.infinispan.commons.CacheException: org.infinispan.commons.CacheException: Could not move intermediate keys/values for M/R task 65ba6afd-e70f-4b00-a07b-0de489f3c0ec
	at org.infinispan.distexec.mapreduce.MapReduceManagerImpl.mapAndCombineForDistributedReduction(MapReduceManagerImpl.java:116)
	at org.infinispan.distexec.mapreduce.MapReduceTask$MapTaskPart.invokeMapCombineLocally(MapReduceTask.java:1193)
	at org.infinispan.distexec.mapreduce.MapReduceTask$MapTaskPart.access$200(MapReduceTask.java:1120)
	at org.infinispan.distexec.mapreduce.MapReduceTask$MapTaskPart$1.call(MapReduceTask.java:1142)
	at org.infinispan.distexec.mapreduce.MapReduceTask$MapTaskPart$1.call(MapReduceTask.java:1138)
	at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
	at java.util.concurrent.FutureTask.run(FutureTask.java:138)
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:439)
	... 5 more
Caused by: org.infinispan.commons.CacheException: Could not move intermediate keys/values for M/R task 65ba6afd-e70f-4b00-a07b-0de489f3c0ec
	at org.infinispan.distexec.mapreduce.MapReduceManagerImpl.migrateIntermediateKeysAndValues(MapReduceManagerImpl.java:375)
	at org.infinispan.distexec.mapreduce.MapReduceManagerImpl.mapAndCombine(MapReduceManagerImpl.java:282)
	at org.infinispan.distexec.mapreduce.MapReduceManagerImpl.mapAndCombineForDistributedReduction(MapReduceManagerImpl.java:114)
	... 12 more
Caused by: org.infinispan.util.concurrent.TimeoutException: Node DistributedIntermediateSharedCacheFourNodesMapReduceTest-NodeC-20073 timed out
	at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:217)
	at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:564)
	at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:292)
	at org.infinispan.interceptors.distribution.BaseDistributionInterceptor.handleNonTxWriteCommand(BaseDistributionInterceptor.java:316)
	at org.infinispan.interceptors.distribution.NonTxDistributionInterceptor.visitPutKeyValueCommand(NonTxDistributionInterceptor.java:88)
	at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:71)
	at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98)
	at org.infinispan.interceptors.EntryWrappingInterceptor.invokeNextAndApplyChanges(EntryWrappingInterceptor.java:366)
	at org.infinispan.interceptors.EntryWrappingInterceptor.setSkipRemoteGetsAndInvokeNextForDataCommand(EntryWrappingInterceptor.java:454)
	at org.infinispan.interceptors.EntryWrappingInterceptor.visitPutKeyValueCommand(EntryWrappingInterceptor.java:188)
	at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:71)
	at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98)
	at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitPutKeyValueCommand(AbstractLockingInterceptor.java:46)
	at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:71)
	at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98)
	at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:112)
	at org.infinispan.commands.AbstractVisitor.visitPutKeyValueCommand(AbstractVisitor.java:43)
	at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:71)
	at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98)
	at org.infinispan.statetransfer.StateTransferInterceptor.handleNonTxWriteCommand(StateTransferInterceptor.java:336)
	at org.infinispan.statetransfer.StateTransferInterceptor.handleWriteCommand(StateTransferInterceptor.java:268)
	at org.infinispan.statetransfer.StateTransferInterceptor.visitPutKeyValueCommand(StateTransferInterceptor.java:123)
	at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:71)
	at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98)
	at org.infinispan.interceptors.CacheMgmtInterceptor.updateStoreStatistics(CacheMgmtInterceptor.java:182)
	at org.infinispan.interceptors.CacheMgmtInterceptor.visitPutKeyValueCommand(CacheMgmtInterceptor.java:168)
	at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:71)
	at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98)
	at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:104)
	at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:73)
	at org.infinispan.commands.AbstractVisitor.visitPutKeyValueCommand(AbstractVisitor.java:43)
	at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:71)
	at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:336)
	at org.infinispan.CacheImpl.executeCommandAndCommitIfNeeded(CacheImpl.java:1592)
	at org.infinispan.CacheImpl.putInternal(CacheImpl.java:1064)
	at org.infinispan.CacheImpl.put(CacheImpl.java:1055)
	at org.infinispan.DecoratedCache.put(DecoratedCache.java:413)
	at org.infinispan.distexec.mapreduce.MapReduceManagerImpl.migrateIntermediateKeysAndValues(MapReduceManagerImpl.java:367)
	... 14 more
Caused by: org.jgroups.TimeoutException: timeout waiting for response from DistributedIntermediateSharedCacheFourNodesMapReduceTest-NodeC-20073, request: org.jgroups.blocks.UnicastRequest@6758499c, req_id=201318, mode=GET_ALL, target=DistributedIntermediateSharedCacheFourNodesMapReduceTest-NodeC-20073
	at org.jgroups.blocks.MessageDispatcher.sendMessage(MessageDispatcher.java:427)
	at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.processSingleCall(CommandAwareRpcDispatcher.java:401)
	at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:210)
	... 51 more
Comment 2 Vojtech Juranek 2015-08-21 05:32:54 EDT
other tests fails randomly as well, e.g. DistributedIntermediateSharedCacheFourNodesMapReduceTest.testInvokeMapReduceOnEmptyKeys
Comment 3 Vojtech Juranek 2015-08-21 05:49:45 EDT
Similar in DistributedSharedCacheTwoNodesMapReduceTest.testInvokeMapReduceOnAllKeysWithCombiner: 

org.infinispan.commons.CacheException: Could not commit implicit transaction
	at org.infinispan.CacheImpl.executeCommandAndCommitIfNeeded(CacheImpl.java:1605)
	at org.infinispan.CacheImpl.putInternal(CacheImpl.java:1064)
	at org.infinispan.CacheImpl.put(CacheImpl.java:1055)
	at org.infinispan.CacheImpl.put(CacheImpl.java:1637)
	at org.infinispan.CacheImpl.put(CacheImpl.java:251)
	at org.infinispan.distexec.mapreduce.BaseWordCountMapReduceTest.invokeMapReduce(BaseWordCountMapReduceTest.java:115)
	at org.infinispan.distexec.mapreduce.BaseWordCountMapReduceTest.invokeMapReduce(BaseWordCountMapReduceTest.java:156)
	at org.infinispan.distexec.mapreduce.BaseWordCountMapReduceTest.testInvokeMapReduceOnAllKeysWithCombiner(BaseWordCountMapReduceTest.java:187)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:60)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:37)
	at java.lang.reflect.Method.invoke(Method.java:611)
	at org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:80)
	at org.testng.internal.Invoker.invokeMethod(Invoker.java:714)
	at org.testng.internal.Invoker.invokeTestMethod(Invoker.java:901)
	at org.testng.internal.Invoker.invokeTestMethods(Invoker.java:1231)
	at org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:127)
	at org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:111)
	at org.testng.TestRunner.privateRun(TestRunner.java:767)
	at org.testng.TestRunner.run(TestRunner.java:617)
	at org.testng.SuiteRunner.runTest(SuiteRunner.java:334)
	at org.testng.SuiteRunner.access$000(SuiteRunner.java:37)
	at org.testng.SuiteRunner$SuiteWorker.run(SuiteRunner.java:368)
	at org.testng.internal.thread.ThreadUtil$2.call(ThreadUtil.java:64)
	at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:314)
	at java.util.concurrent.FutureTask.run(FutureTask.java:149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:895)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:918)
	at java.lang.Thread.run(Thread.java:767)
Caused by: javax.transaction.RollbackException: ARJUNA016053: Could not commit transaction.
	at com.arjuna.ats.internal.jta.transaction.arjunacore.TransactionImple.commitAndDisassociate(TransactionImple.java:1201)
	at com.arjuna.ats.internal.jta.transaction.arjunacore.BaseTransaction.commit(BaseTransaction.java:126)
	at org.infinispan.CacheImpl.executeCommandAndCommitIfNeeded(CacheImpl.java:1602)
	... 28 more

Note You need to log in before you can comment on or make changes to this bug.