Hide Forgot
configuration: server-side afr with two children tests being run: bonnie++ or iozone (not sure which one of these). (gdb) bt #0 0x0000003496e0ce74 in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x0000003496e08874 in _L_lock_106 () from /lib64/libpthread.so.0 #2 0x0000003496e082e0 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00002b04171e912d in pl_inode_setlk (this=0x1d08a830, pl_inode=0x1d099420, lock=0x2aaab00016e0, can_block=0, dom=0x1d099080) at inodelk.c:454 #4 0x00002b04171e991f in pl_common_inodelk (frame=0x2aaab0000d10, this=0x1d08a830, volume=0x1d08c420 "gfs-ds-afr", inode=0x2aaab0000d80, cmd=6, flock=0x7fffc0e79db0, loc=0x0, fd=0x2aaab0000f80) at inodelk.c:603 #5 0x00002b04171e9bef in pl_finodelk (frame=0x2aaab0000d10, this=0x1d08a830, volume=0x1d08c420 "gfs-ds-afr", fd=0x2aaab0000f80, cmd=6, flock=0x7fffc0e79db0) at inodelk.c:653 #6 0x00002b0417855f10 in afr_unlock (frame=0x1d09de38, this=0x1d08c440) at afr-transaction.c:569 #7 0x00002b041785a85b in afr_transaction_resume (frame=0x1d09de38, this=0x1d08c440) at afr-transaction.c:1400 #8 0x00002b041784d612 in afr_writev_wind_cbk (frame=0x1d09de38, cookie=0x1, this=0x1d08c440, op_ret=8192, op_errno=0, prebuf=0x7fffc0e79f70, postbuf=0x7fffc0e79f70) at afr-inode-write.c:140 #9 0x00002b0417623071 in wb_stack_unwind (unwinds=0x7fffc0e7a0e0) at write-behind.c:1585 #10 0x00002b041762320d in wb_do_ops (frame=0x2aaab0005938, file=0x2aaab0001110, winds=0x7fffc0e7a0f0, unwinds=0x7fffc0e7a0e0, other_requests=0x7fffc0e7a0d0) at write-behind.c:1643 #11 0x00002b04176236ff in wb_process_queue (frame=0x2aaab0005938, file=0x2aaab0001110, flush_all=0 '\0') at write-behind.c:1771 #12 0x00002b0417623e62 in wb_writev (frame=0x2aaab0003330, this=0x1d08bb10, fd=0x2aaab0000f80, vector=0x1d0c5e40, count=1, offset=301162496, iobref=0x2aaab00010b0) at write-behind.c:1890 #13 0x00002b041784d8e2 in afr_writev_wind (frame=0x1d09de38, this=0x1d08c440) at afr-inode-write.c:170 #14 0x00002b0417859a37 in afr_lock_rec (frame=0x1d09de38, this=0x1d08c440, child_index=1) at afr-transaction.c:1287 #15 0x00002b041785924f in afr_lock_cbk (frame=0x1d09de38, cookie=0x0, this=0x1d08c440, op_ret=0, op_errno=0) at afr-transaction.c:1117 #16 0x00002b04171e8ac6 in __grant_blocked_inode_locks (this=0x1d08a830, pl_inode=0x1d099420, dom=0x1d099080) at inodelk.c:337 #17 0x00002b04171e8c6d in grant_blocked_inode_locks (this=0x1d08a830, pl_inode=0x1d099420, lock=0x1d09d670, dom=0x1d099080) at inodelk.c:354 #18 0x00002b04171e938a in pl_inode_setlk (this=0x1d08a830, pl_inode=0x1d099420, lock=0x2aaab0003270, can_block=0, dom=0x1d099080) at inodelk.c:490 #19 0x00002b04171e991f in pl_common_inodelk (frame=0x2aaab0003210, this=0x1d08a830, volume=0x1d08c420 "gfs-ds-afr", inode=0x2aaab0000d80, cmd=6, flock=0x7fffc0e7a770, loc=0x0, fd=0x2aaab0001190) at inodelk.c:603 #20 0x00002b04171e9bef in pl_finodelk (frame=0x2aaab0003210, this=0x1d08a830, volume=0x1d08c420 "gfs-ds-afr", fd=0x2aaab0001190, cmd=6, flock=0x7fffc0e7a770) at inodelk.c:653 #21 0x00002b0417855f10 in afr_unlock (frame=0x1d099eb8, this=0x1d08c440) at afr-transaction.c:569 #22 0x00002b0417856cf6 in afr_changelog_post_op_cbk (frame=0x1d099eb8, cookie=0x2aaab0002bd0, this=0x1d08c440, op_ret=0, op_errno=11, xattr=0x2aaab0002af0) at afr-transaction.c:702 #23 0x00002b041650e704 in default_fxattrop_cbk (frame=Could not find the frame base for "default_fxattrop_cbk". ) at defaults.c:998 #24 0x00002b041740ad8b in client_fxattrop_cbk (frame=0x2aaab0002c30, hdr=0x2aaab00055d0, hdrlen=213, iobuf=0x0) at client-protocol.c:3836 #25 0x00002b041741300d in protocol_client_interpret (this=0x1d08b050, trans=0x1d095560, hdr_p=0x2aaab00055d0 "", hdrlen=213, iobuf=0x0) at client-protocol.c:6507 #26 0x00002b0417413cd3 in protocol_client_pollin (this=0x1d08b050, trans=0x1d095560) at client-protocol.c:6805 #27 0x00002b041741434a in notify (this=0x1d08b050, event=2, data=0x1d095560) at client-protocol.c:6924 #28 0x00002b041650759b in xlator_notify (xl=Could not find the frame base for "xlator_notify". ) at xlator.c:923 #29 0x00002aaaaaaaed9a in socket_event_poll_in (this=0x1d095560) at socket.c:729 #30 0x00002aaaaaaaf08f in socket_event_handler (fd=8, idx=1, data=0x1d095560, poll_in=1, poll_out=0, poll_err=0) at socket.c:829 #31 0x00002b041652c185 in event_dispatch_epoll_handler (event_pool=Could not find the frame base for "event_dispatch_epoll_handler". ) at event.c:804 #32 0x00002b041652c35a in event_dispatch_epoll (event_pool=Could not find the frame base for "event_dispatch_epoll". ) at event.c:867 #33 0x00002b041652c670 in event_dispatch (event_pool=Could not find the frame base for "event_dispatch". ) at event.c:975 #34 0x0000000000405e10 in main (argc=3, argv=0x7fffc0e7b588) at glusterfsd.c:1378 The reason for this is grant_blocked_inode_locks is being called with pl_inode->mutex held.
Yes. We should not UNWIND with a mutex held. Patch coming.
PATCH: http://patches.gluster.com/patch/2651 in master (features/locks: Do not call grant blocked locks in pl_forget)
PATCH: http://patches.gluster.com/patch/2652 in master (features/locks: Fix broken interface grant_blocked_inode_locks)