While testing enomaly ecp with 3.1.6qa3 I hit a deadlock in io-cache. I have pasted the back trace below.

#0  0x000000346220d4c4 in __lll_lock_wait () from /lib64/libpthread.so.0
#1  0x0000003462208e1a in _L_lock_1034 () from /lib64/libpthread.so.0
#2  0x0000003462208cdc in pthread_mutex_lock () from /lib64/libpthread.so.0
#3  0x00002aaaabc9e98f in ioc_page_destroy (page=0x2aaab000db60) at page.c:133
#4  0x00002aaaabca1510 in __ioc_page_error (page=0x2aaab000db60, op_ret=-1, op_errno=77) at page.c:923
#5  0x00002aaaabc9f56a in ioc_fault_cbk (frame=0x2acb0c907ae8, cookie=0x2acb0c678574, this=0xe671980, op_ret=-1, op_errno=77, vector=0x2aaab401d600, count=0, stbuf=0x2aaab401c8d0, iobref=0x2aaab4004950) at page.c:424
#6  0x00002aaaaba90ad9 in ra_frame_unwind (frame=0x2acb0c678574) at page.c:403
#7  0x00002aaaaba90bac in ra_frame_return (frame=0x2acb0c678574) at page.c:435
#8  0x00002aaaaba8f771 in ra_waitq_return (waitq=0x2aaab000dca0) at page.c:127
#9  0x00002aaaaba8fa5c in ra_fault_cbk (frame=0x2acb0c905334, cookie=0x2acb0c679dcc, this=0xe670750, op_ret=-1, op_errno=77, vector=0x0, count=0, stbuf=0x0, iobref=0x0) at page.c:196
#10 0x00002aaaab8809db in wb_readv_cbk (frame=0x2acb0c679dcc, cookie=0x2acb0c677050, this=0xe66f510, op_ret=-1, op_errno=77, vector=0x0, count=0, stbuf=0x0, iobref=0x0) at write-behind.c:2102
#11 0x00002aaaab659b76 in dht_readv_cbk (frame=0x2acb0c677050, cookie=0x2acb0c679a98, this=0xe66e1c0, op_ret=-1, op_errno=77, vector=0x0, count=0, stbuf=0x0, iobref=0x0) at dht-common.c:2289
#12 0x00002aaaab3e1f55 in afr_readv_cbk (frame=0x2acb0c679a98, cookie=0x0, this=0xe66c940, op_ret=-1, op_errno=77, vector=0x0, count=0, buf=0x0, iobref=0x0) at afr-inode-read.c:879
#13 0x00002aaaab1b09b1 in client3_1_readv (frame=0x2acb0c6791a0, this=0xe667de0, data=0x7fff081b7990) at client3_1-fops.c:3261
#14 0x00002aaaab19db9e in client_readv (frame=0x2acb0c6791a0, this=0xe667de0, fd=0x2aaaadf961e8, size=131072, offset=18939904000) at client.c:654
#15 0x00002aaaab3e1da6 in afr_readv_cbk (frame=0x2acb0c679a98, cookie=0x0, this=0xe66c940, op_ret=-1, op_errno=107, vector=0x7fff081b7cb0, count=0, buf=0x7fff081b7c40, iobref=0x0) at afr-inode-read.c:866
#16 0x00002aaaab1abfec in client3_1_readv_cbk (req=0x2aaaacd1b020, iov=0x0, count=0, myframe=0x2acb0c674d14) at client3_1-fops.c:2012
#17 0x00002acb0b9de12d in rpc_clnt_submit (rpc=0xe6845a0, prog=0x2aaaab3c6520, procnum=12, cbkfn=0x2aaaab1abccd <client3_1_readv_cbk>, proghdr=0x7fff081b7fa0, proghdrcount=1, progpayload=0x0, progpayloadcount=0, iobref=0x2aaab4001160, 
    frame=0x2acb0c674d14, rsphdr=0x0, rsphdr_count=0, rsp_payload=0x7fff081b8050, rsp_payload_count=1, rsp_iobref=0x2aaab4004c40) at rpc-clnt.c:1527
#18 0x00002aaaab19b4ae in client_submit_request (this=0xe666f10, req=0x7fff081b8060, frame=0x2acb0c674d14, prog=0x2aaaab3c6520, procnum=12, cbk=0x2aaaab1abccd <client3_1_readv_cbk>, iobref=0x0, 
    sfunc=0x2acb0bbf28ac <xdr_from_readv_req>, rsphdr=0x0, rsphdr_count=0, rsp_payload=0x7fff081b8050, rsp_payload_count=1, rsp_iobref=0x2aaab4004c40) at client.c:113
#19 0x00002aaaab1b0835 in client3_1_readv (frame=0x2acb0c674d14, this=0xe666f10, data=0x7fff081b8140) at client3_1-fops.c:3242
#20 0x00002aaaab19db9e in client_readv (frame=0x2acb0c674d14, this=0xe666f10, fd=0x2aaaadf961e8, size=131072, offset=18939904000) at client.c:654
#21 0x00002aaaab3e2625 in afr_readv (frame=0x2acb0c679a98, this=0xe66c940, fd=0x2aaaadf961e8, size=131072, offset=18939904000) at afr-inode-read.c:942
#22 0x00002aaaab65a064 in dht_readv (frame=0x2acb0c677050, this=0xe66e1c0, fd=0x2aaaadf961e8, size=131072, off=18939904000) at dht-common.c:2324
#23 0x00002aaaab880c04 in wb_readv_helper (frame=0x2acb0c679dcc, this=0xe66f510, fd=0x2aaaadf961e8, size=131072, offset=18939904000) at write-behind.c:2112
#24 0x00002acb0b78ae1a in call_resume_wind (stub=0x2acb0cb6f96c) at call-stub.c:2226
#25 0x00002acb0b79164c in call_resume (stub=0x2acb0cb6f96c) at call-stub.c:3870
#26 0x00002aaaab87f617 in wb_resume_other_requests (frame=0x2acb0c906070, file=0x2aaab400f820, other_requests=0x7fff081b8580) at write-behind.c:1730
#27 0x00002aaaab87f6c3 in wb_do_ops (frame=0x2acb0c906070, file=0x2aaab400f820, winds=0x7fff081b85a0, unwinds=0x7fff081b8590, other_requests=0x7fff081b8580) at write-behind.c:1757
#28 0x00002aaaab87fd66 in wb_process_queue (frame=0x2acb0c906070, file=0x2aaab400f820) at write-behind.c:1933
#29 0x00002aaaab879d04 in wb_sync_cbk (frame=0x2acb0c906070, cookie=0x2acb0c67819c, this=0xe66f510, op_ret=-1, op_errno=107, prebuf=0x2aaab401f658, postbuf=0x2aaab401f6c8) at write-behind.c:393
#30 0x00002aaaab65a412 in dht_writev_cbk (frame=0x2acb0c67819c, cookie=0x2acb0c6780f8, this=0xe66e1c0, op_ret=-1, op_errno=107, prebuf=0x2aaab401f658, postbuf=0x2aaab401f6c8) at dht-common.c:2360
#31 0x00002aaaab3e29f0 in afr_writev_unwind (frame=0x2acb0c905ba0, this=0xe66c940) at afr-inode-write.c:72
#32 0x00002aaaab3e31b3 in afr_writev_done (frame=0x2acb0c905ba0, this=0xe66c940) at afr-inode-write.c:185
#33 0x00002aaaab3f0f78 in afr_post_blocking_inodelk_cbk (frame=0x2acb0c905ba0, this=0xe66c940) at afr-transaction.c:949
#34 0x00002aaaab40cb3e in afr_unlock_inodelk (frame=0x2acb0c905ba0, this=0xe66c940) at afr-lk-common.c:605
#35 0x00002aaaab41154d in afr_unlock (frame=0x2acb0c905ba0, this=0xe66c940) at afr-lk-common.c:1680
#36 0x00002aaaab40e16c in afr_lock_blocking (frame=0x2acb0c905ba0, this=0xe66c940, child_index=2) at afr-lk-common.c:973
#37 0x00002aaaab40d755 in afr_lock_cbk (frame=0x2acb0c905ba0, cookie=0x0, this=0xe66c940, op_ret=-1, op_errno=107) at afr-lk-common.c:760
#38 0x00002aaaab40d7bc in afr_blocking_inodelk_cbk (frame=0x2acb0c905ba0, cookie=0x0, this=0xe66c940, op_ret=-1, op_errno=107) at afr-lk-common.c:774
#39 0x00002aaaab1a8933 in client3_1_finodelk_cbk (req=0x2aaaacd1a288, iov=0x0, count=0, myframe=0x2acb0c6750ec) at client3_1-fops.c:1101
#40 0x00002acb0b9de12d in rpc_clnt_submit (rpc=0xe6845a0, prog=0x2aaaab3c6520, procnum=30, cbkfn=0x2aaaab1a8754 <client3_1_finodelk_cbk>, proghdr=0x7fff081b8c80, proghdrcount=1, progpayload=0x0, progpayloadcount=0, 
    iobref=0x2aaab4004740, frame=0x2acb0c6750ec, rsphdr=0x0, rsphdr_count=0, rsp_payload=0x0, rsp_payload_count=0, rsp_iobref=0x0) at rpc-clnt.c:1527
#41 0x00002aaaab19b4ae in client_submit_request (this=0xe666f10, req=0x7fff081b8d30, frame=0x2acb0c6750ec, prog=0x2aaaab3c6520, procnum=30, cbk=0x2aaaab1a8754 <client3_1_finodelk_cbk>, iobref=0x0, 
    sfunc=0x2acb0bbf25e1 <xdr_from_finodelk_req>, rsphdr=0x0, rsphdr_count=0, rsp_payload=0x0, rsp_payload_count=0, rsp_iobref=0x0) at client.c:113
#42 0x00002aaaab1b5806 in client3_1_finodelk (frame=0x2acb0c6750ec, this=0xe666f10, data=0x7fff081b8e00) at client3_1-fops.c:4608
#43 0x00002aaaab1a072a in client_finodelk (frame=0x2acb0c6750ec, this=0xe666f10, volume=0xe66c780 "hosdu-replicate-2", fd=0x2aaaadf961e8, cmd=7, lock=0x7fff081b8f70) at client.c:1290
#44 0x00002aaaab40e5c1 in afr_lock_blocking (frame=0x2acb0c905ba0, this=0xe66c940, child_index=0) at afr-lk-common.c:1005
#45 0x00002aaaab40f3bd in afr_blocking_lock (frame=0x2acb0c905ba0, this=0xe66c940) at afr-lk-common.c:1119
#46 0x00002aaaab3f108f in afr_post_nonblocking_inodelk_cbk (frame=0x2acb0c905ba0, this=0xe66c940) at afr-transaction.c:975
#47 0x00002aaaab40cb3e in afr_unlock_inodelk (frame=0x2acb0c905ba0, this=0xe66c940) at afr-lk-common.c:605
#48 0x00002aaaab41154d in afr_unlock (frame=0x2acb0c905ba0, this=0xe66c940) at afr-lk-common.c:1680
#49 0x00002aaaab4102c0 in afr_nonblocking_inodelk_cbk (frame=0x2acb0c905ba0, cookie=0x0, this=0xe66c940, op_ret=-1, op_errno=107) at afr-lk-common.c:1351
#50 0x00002aaaab1a8933 in client3_1_finodelk_cbk (req=0x2aaaacd1b4a8, iov=0x0, count=0, myframe=0x2acb0c6772e0) at client3_1-fops.c:1101
#51 0x00002acb0b9de12d in rpc_clnt_submit (rpc=0xe6845a0, prog=0x2aaaab3c6520, procnum=30, cbkfn=0x2aaaab1a8754 <client3_1_finodelk_cbk>, proghdr=0x7fff081b9430, proghdrcount=1, progpayload=0x0, progpayloadcount=0, 
    iobref=0x2aaab40035e0, frame=0x2acb0c6772e0, rsphdr=0x0, rsphdr_count=0, rsp_payload=0x0, rsp_payload_count=0, rsp_iobref=0x0) at rpc-clnt.c:1527
---Type <return> to continue, or q <return> to quit---
#52 0x00002aaaab19b4ae in client_submit_request (this=0xe666f10, req=0x7fff081b94e0, frame=0x2acb0c6772e0, prog=0x2aaaab3c6520, procnum=30, cbk=0x2aaaab1a8754 <client3_1_finodelk_cbk>, iobref=0x0, 
    sfunc=0x2acb0bbf25e1 <xdr_from_finodelk_req>, rsphdr=0x0, rsphdr_count=0, rsp_payload=0x0, rsp_payload_count=0, rsp_iobref=0x0) at client.c:113
#53 0x00002aaaab1b5806 in client3_1_finodelk (frame=0x2acb0c6772e0, this=0xe666f10, data=0x7fff081b95b0) at client3_1-fops.c:4608
#54 0x00002aaaab1a072a in client_finodelk (frame=0x2acb0c6772e0, this=0xe666f10, volume=0xe66c780 "hosdu-replicate-2", fd=0x2aaaadf961e8, cmd=6, lock=0x7fff081b96e0) at client.c:1290
#55 0x00002aaaab410802 in afr_nonblocking_inodelk (frame=0x2acb0c905ba0, this=0xe66c940) at afr-lk-common.c:1418
#56 0x00002aaaab3f1594 in afr_lock_rec (frame=0x2acb0c905ba0, this=0xe66c940) at afr-transaction.c:1110
#57 0x00002aaaab3f1653 in afr_lock (frame=0x2acb0c905ba0, this=0xe66c940) at afr-transaction.c:1146
#58 0x00002aaaab3f18be in afr_transaction (frame=0x2acb0c905ba0, this=0xe66c940, type=AFR_DATA_TRANSACTION) at afr-transaction.c:1237
#59 0x00002aaaab3e3586 in afr_do_writev (frame=0x2acb0c6780f8, this=0xe66c940) at afr-inode-write.c:233
#60 0x00002aaaab3e3d98 in afr_writev (frame=0x2acb0c6780f8, this=0xe66c940, fd=0x2aaaadf961e8, vector=0x2aaab401dd00, count=1, offset=107125248, iobref=0x2aaab401e020) at afr-inode-write.c:298
#61 0x00002aaaab65a92a in dht_writev (frame=0x2acb0c67819c, this=0xe66e1c0, fd=0x2aaaadf961e8, vector=0x2aaab401dd00, count=1, off=107125248, iobref=0x2aaab401e020) at dht-common.c:2397
#62 0x00002aaaab87a6cc in wb_sync (frame=0x2acb0c907148, file=0x2aaab400f820, winds=0x7fff081b9b80) at write-behind.c:535
#63 0x00002aaaab87f6a9 in wb_do_ops (frame=0x2acb0c907148, file=0x2aaab400f820, winds=0x7fff081b9b80, unwinds=0x7fff081b9b70, other_requests=0x7fff081b9b60) at write-behind.c:1752
#64 0x00002aaaab87fd66 in wb_process_queue (frame=0x2acb0c907148, file=0x2aaab400f820) at write-behind.c:1933
#65 0x00002aaaab879d04 in wb_sync_cbk (frame=0x2acb0c907148, cookie=0x2acb0c675940, this=0xe66f510, op_ret=-1, op_errno=107, prebuf=0xe69e798, postbuf=0xe69e808) at write-behind.c:393
#66 0x00002aaaab65a412 in dht_writev_cbk (frame=0x2acb0c675940, cookie=0x2acb0c6799f4, this=0xe66e1c0, op_ret=-1, op_errno=107, prebuf=0xe69e798, postbuf=0xe69e808) at dht-common.c:2360
#67 0x00002aaaab3e29f0 in afr_writev_unwind (frame=0x2acb0c905cd4, this=0xe66c940) at afr-inode-write.c:72
#68 0x00002aaaab3e31b3 in afr_writev_done (frame=0x2acb0c905cd4, this=0xe66c940) at afr-inode-write.c:185
#69 0x00002aaaab40c91e in afr_unlock_common_cbk (frame=0x2acb0c905cd4, cookie=0x0, this=0xe66c940, op_ret=-1, op_errno=107) at afr-lk-common.c:549
#70 0x00002aaaab40ca14 in afr_unlock_inodelk_cbk (frame=0x2acb0c905cd4, cookie=0x0, this=0xe66c940, op_ret=-1, op_errno=107) at afr-lk-common.c:572
#71 0x00002aaaab1a8933 in client3_1_finodelk_cbk (req=0x2aaaacd1c240, iov=0x0, count=0, myframe=0x2acb0c674898) at client3_1-fops.c:1101
#72 0x00002acb0b9de12d in rpc_clnt_submit (rpc=0xe6845a0, prog=0x2aaaab3c6520, procnum=30, cbkfn=0x2aaaab1a8754 <client3_1_finodelk_cbk>, proghdr=0x7fff081ba000, proghdrcount=1, progpayload=0x0, progpayloadcount=0, 
    iobref=0x2aaab401dab0, frame=0x2acb0c674898, rsphdr=0x0, rsphdr_count=0, rsp_payload=0x0, rsp_payload_count=0, rsp_iobref=0x0) at rpc-clnt.c:1527
#73 0x00002aaaab19b4ae in client_submit_request (this=0xe666f10, req=0x7fff081ba0b0, frame=0x2acb0c674898, prog=0x2aaaab3c6520, procnum=30, cbk=0x2aaaab1a8754 <client3_1_finodelk_cbk>, iobref=0x0, 
    sfunc=0x2acb0bbf25e1 <xdr_from_finodelk_req>, rsphdr=0x0, rsphdr_count=0, rsp_payload=0x0, rsp_payload_count=0, rsp_iobref=0x0) at client.c:113
#74 0x00002aaaab1b5806 in client3_1_finodelk (frame=0x2acb0c674898, this=0xe666f10, data=0x7fff081ba180) at client3_1-fops.c:4608
#75 0x00002aaaab1a072a in client_finodelk (frame=0x2acb0c674898, this=0xe666f10, volume=0xe66c780 "hosdu-replicate-2", fd=0x2aaaadf961e8, cmd=6, lock=0x7fff081ba2a0) at client.c:1290
#76 0x00002aaaab40ce33 in afr_unlock_inodelk (frame=0x2acb0c905cd4, this=0xe66c940) at afr-lk-common.c:615
#77 0x00002aaaab41154d in afr_unlock (frame=0x2acb0c905cd4, this=0xe66c940) at afr-lk-common.c:1680
#78 0x00002aaaab3ed8b2 in afr_changelog_post_op (frame=0x2acb0c905cd4, this=0xe66c940) at afr-transaction.c:502
#79 0x00002aaaab3f1764 in afr_transaction_resume (frame=0x2acb0c905cd4, this=0xe66c940) at afr-transaction.c:1188
#80 0x00002aaaab3e2e47 in afr_writev_wind (frame=0x2acb0c905cd4, this=0xe66c940) at afr-inode-write.c:148
#81 0x00002aaaab3ef3b5 in afr_changelog_pre_op_cbk (frame=0x2acb0c905cd4, cookie=0x0, this=0xe66c940, op_ret=-1, op_errno=107, xattr=0x0) at afr-transaction.c:725
#82 0x00002aaaab1a95ae in client3_1_fxattrop_cbk (req=0x2aaaacd1a044, iov=0x7fff081ba6c0, count=1, myframe=0x2acb0c679c84) at client3_1-fops.c:1305
#83 0x00002acb0b9db77a in saved_frames_unwind (saved_frames=0xe689800) at rpc-clnt.c:344
#84 0x00002acb0b9db813 in saved_frames_destroy (frames=0xe689800) at rpc-clnt.c:363
#85 0x00002acb0b9dbcdd in rpc_clnt_connection_cleanup (conn=0xe6845d0) at rpc-clnt.c:511
#86 0x00002acb0b9dc70b in rpc_clnt_notify (trans=0xe684750, mydata=0xe6845d0, event=RPC_TRANSPORT_DISCONNECT, data=0xe684750) at rpc-clnt.c:831
#87 0x00002acb0b9d9fc9 in rpc_transport_notify (this=0xe684750, event=RPC_TRANSPORT_DISCONNECT, data=0xe684750) at rpc-transport.c:1043
#88 0x00002aaaaad69356 in socket_event_poll_err (this=0xe684750) at socket.c:662
#89 0x00002aaaaad6c895 in socket_event_handler (fd=7, idx=1, data=0xe684750, poll_in=1, poll_out=0, poll_err=0) at socket.c:1742
#90 0x00002acb0b795730 in event_dispatch_epoll_handler (event_pool=0xe658230, events=0xe65cd30, i=0) at event.c:812
#91 0x00002acb0b79591f in event_dispatch_epoll (event_pool=0xe658230) at event.c:876
#92 0x00002acb0b795c7b in event_dispatch (event_pool=0xe658230) at event.c:984
#93 0x0000000000405fc7 in main (argc=5, argv=0x7fff081bacd8) at glusterfsd.c:1455

Unable to reproduce this.