Hide Forgot
Crash occurs in afr-lk-common.c at 1354: fd_ctx = (afr_fd_ctx_t *)(long) ctx; call_count = internal_lock_count (frame, this, fd_ctx); local->lk_call_count = call_count; /* Send non-blocking inodelk calls only on up children and where the fd has been opened */ for (i = 0; i < priv->child_count; i++) { #### HERE ##### if (local->child_up[i] && fd_ctx->opened_on[i]) { Because fd_ctx got above is corrupted or NULL.
Reported by Harsha over email. Seen at customer site. rgp 1 backtrace 1 dlfcn 1 fdatasync 1 libpthread 1 llistxattr 1 setfsid 1 spinlock 1 epoll.h 1 xattr.h 1 st_atim.tv_nsec 1 package-string: glusterfs nfs_beta_rc11 /lib64/libc.so.6[0x3e970332f0] /usr/lib64/glusterfs/nfs_beta_rc11/xlator/cluster/replicate.so(afr_nonblocking_inodelk+0xd8)[0x7fd915571218] /usr/lib64/glusterfs/nfs_beta_rc11/xlator/cluster/replicate.so(afr_lock_rec+0xa8)[0x7fd91555c468] /usr/lib64/glusterfs/nfs_beta_rc11/xlator/cluster/replicate.so(afr_transaction+0x15b)[0x7fd91555cfab] /usr/lib64/glusterfs/nfs_beta_rc11/xlator/cluster/replicate.so(afr_do_writev+0x1fa)[0x7fd9155585da] /usr/lib64/glusterfs/nfs_beta_rc11/xlator/cluster/replicate.so(afr_writev+0x443)[0x7fd915558b43] /usr/lib64/glusterfs/nfs_beta_rc11/xlator/cluster/distribute.so(dht_writev+0x14e)[0x7fd9153241ae] /usr/lib64/glusterfs/nfs_beta_rc11/xlator/nfs/server.so(nfs_fop_write+0x209)[0x7fd9150e47d9] /usr/lib64/glusterfs/nfs_beta_rc11/xlator/nfs/server.so(__nfs3_write_resume+0x8e)[0x7fd9150f0bde] /usr/lib64/glusterfs/nfs_beta_rc11/xlator/nfs/server.so(nfs3_write_resume+0x2f)[0x7fd9150f45cf] /usr/lib64/glusterfs/nfs_beta_rc11/xlator/nfs/server.so(nfs3_file_open_and_resume+0x58)[0x7fd9150ff5e8] /usr/lib64/glusterfs/nfs_beta_rc11/xlator/nfs/server.so(nfs3_write_open_resume+0x38)[0x7fd9150f4508] /usr/lib64/glusterfs/nfs_beta_rc11/xlator/nfs/server.so(nfs3_fh_resolve_inode_done+0x4d)[0x7fd9150fdd1d] /usr/lib64/glusterfs/nfs_beta_rc11/xlator/nfs/server.so(nfs3_fh_resolve_inode+0x5e)[0x7fd9150fddce] /usr/lib64/glusterfs/nfs_beta_rc11/xlator/nfs/server.so(nfs3_write+0x37b)[0x7fd9150fa27b] /usr/lib64/glusterfs/nfs_beta_rc11/xlator/nfs/server.so(nfs3svc_write_vec+0x89)[0x7fd9150fa509] /usr/lib64/libglrpcsvc.so.0(rpcsvc_record_vectored_call_actor+0x52)[0x7fd914ecc402] /usr/lib64/libglrpcsvc.so.0(rpcsvc_update_vectored_state+0xb8)[0x7fd914eccc28] /usr/lib64/libglrpcsvc.so.0(rpcsvc_record_update_state+0x11d)[0x7fd914eccead] /usr/lib64/libglrpcsvc.so.0(rpcsvc_conn_data_handler+0x68)[0x7fd914ecd1f8] /usr/lib64/libglusterfs.so.0[0x31bca2fe4d] /usr/lib64/libglrpcsvc.so.0(rpcsvc_stage_proc+0x12)[0x7fd914ece5e2] /lib64/libpthread.so.0[0x3e97c0685a] /lib64/libc.so.6(clone+0x6d)[0x3e970de22d] #0 0x00007fd915571218 in afr_nonblocking_inodelk (frame=<value optimized out>, this=<value optimized out>) at afr-lk-common.c:1354 #1 0x00007fd91555c468 in afr_lock_rec (frame=0x7fd8d5dd04c8, this=<value optimized out>) at afr-transaction.c:1018 #2 0x00007fd91555cfab in afr_transaction (frame=0x7fd8d5dd04c8, this=0x6e5100, type=<value optimized out>) at afr-transaction.c:1159 #3 0x00007fd9155585da in afr_do_writev (frame=0x7fd8d5df76f0, this=0x6e5100) at afr-inode-write.c:247 #4 0x00007fd915558b43 in afr_writev (frame=0x7fd8d5de0370, this=0x6e5100, fd=0x7c79a0, vector=0x7fd913d98330, count=1, offset=<value optimized out>, iobref=0x7fd8d5de0dd0) at afr-inode-write.c:317 #5 0x00007fd9153241ae in dht_writev (frame=<value optimized out>, this=<value optimized out>, fd=0x7c79a0, vector=<value optimized out>, count=1, off=15067783168, iobref=0x7fd8d5de0dd0) at dht-common.c:1865 #6 0x00007fd9150e47d9 in nfs_fop_write (nfsx=<value optimized out>, xl=0x6e5bd0, nfu=<value optimized out>, fd=0x7c79a0, srciob=<value optimized out>, vector=0x7fd913d98330, count=1, offset=15067783168, cbk=0x7fd9150f4660 <nfs3svc_write_cbk>, local=0x7fd913d97fa0) at nfs-fops.c:1147 #7 0x00007fd9150f0bde in __nfs3_write_resume (cs=0x7fd913d97fa0) at nfs3.c:1753 #8 0x00007fd9150f45cf in nfs3_write_resume (carg=0x0) at nfs3.c:1773 #9 0x00007fd9150ff5e8 in nfs3_file_open_and_resume (cs=0x7fd913d97fa0, resume=<value optimized out>) at nfs3-helpers.c:2194 #10 0x00007fd9150f4508 in nfs3_write_open_resume (carg=0x0) at nfs3.c:1800 #11 0x00007fd9150fdd1d in nfs3_fh_resolve_inode_done (cs=0x7fd913d97fa0, inode=<value optimized out>) at nfs3-helpers.c:2479 #12 0x00007fd9150fddce in nfs3_fh_resolve_inode (cs=0x7fd913d97fa0) at nfs3-helpers.c:2976 #13 0x00007fd9150fa27b in nfs3_write (req=0x7fd8dc75c750, fh=0x7fd8d5de33e0, offset=<value optimized out>, count=<value optimized out>, stable=FILE_SYNC, payload={iov_base = 0x7fd903e40000, iov_len = 131072}, iob=<value optimized out>) at nfs3.c:1846 #14 0x00007fd9150fa509 in nfs3svc_write_vec (req=0x7fd8dc75c750, iob=0x78ad30) at nfs3.c:1933 #15 0x00007fd914ecc402 in rpcsvc_record_vectored_call_actor ( conn=0x7fd8dc739d60) at rpcsvc.c:2179 #16 0x00007fd914eccc28 in rpcsvc_update_vectored_state (conn=0x7fd8dc739d60) at rpcsvc.c:2236 #17 0x00007fd914eccead in rpcsvc_record_update_state (conn=0x7fd8dc739d60, dataread=65536) at rpcsvc.c:2331 #18 0x00007fd914ecd1f8 in rpcsvc_conn_data_handler (fd=<value optimized out>, idx=0, data=0x7fd8dc739d60, poll_in=-1746832768, poll_out=10001312, poll_err=1) at rpcsvc.c:2528 #19 0x00000031bca2fe4d in event_dispatch_epoll_handler ( ---Type <return> to continue, or q <return> to quit--- i=<value optimized out>, events=<value optimized out>, event_pool=<value optimized out>) at event.c:804 #20 event_dispatch_epoll (i=<value optimized out>, events=<value optimized out>, event_pool=<value optimized out>) at event.c:867 #21 0x00007fd914ece5e2 in rpcsvc_stage_proc (arg=<value optimized out>) at rpcsvc.c:64 #22 0x0000003e97c0685a in start_thread () from /lib64/libpthread.so.0 #23 0x0000003e970de22d in clone () from /lib64/libc.so.6 #24 0x0000000000000000 in ?? ()
PATCH: http://patches.gluster.com/patch/4940 in master (Check for possible fd/ctx NULL in afr)