Description of problem:

While running upstream regression, observed a coredump generated by fuse client process.

Thread 1 (Thread 0x7f34ef722700 (LWP 9310)):
16:58:12 #0  0x00007f34ee70d0b9 in dht_common_mark_mdsxattr (frame=0x7f34e80a44d8, errst=0x0, mark_during_fresh_lookup=1) at /home/jenkins/root/workspace/regression-on-demand-full-run/xlators/cluster/dht/src/dht-common.c:855
16:58:12         local = 0x7f34e80b37f8
16:58:12         this = 0x7f34e80252f0
16:58:12         hashed_subvol = 0x0
16:58:12         ret = 0
16:58:12         i = 2
16:58:12         xattrs = 0x0
16:58:12         gfid_local = "a8c9cc71-1430-4217-86e6-f32eeb69d4ce"
16:58:12         zero = {0}
16:58:12         conf = 0x7f34e8054600
16:58:12         layout = 0x0
16:58:12         copy_local = 0x0
16:58:12         xattr_frame = 0x0
16:58:12         vol_down = false
16:58:12         __FUNCTION__ = "dht_common_mark_mdsxattr"
16:58:12 #1  0x00007f34ee7120f7 in dht_revalidate_cbk (frame=0x7f34e80a44d8, cookie=0x7f34e8020b50, this=0x7f34e80252f0, op_ret=0, op_errno=0, inode=0x7f34dc02bd98, stbuf=0x7f34e80a15e8, xattr=0x7f34e80c5b78, postparent=0x7f34e80a1680) at /home/jenkins/root/workspace/regression-on-demand-full-run/xlators/cluster/dht/src/dht-common.c:1780
16:58:12         local = 0x7f34e80b37f8
16:58:12         this_call_cnt = 0
16:58:12         prev = 0x7f34e8020b50
16:58:12         layout = 0x7f34dc011250
16:58:12         conf = 0x7f34e8054600
16:58:12         ret = 0
16:58:12         is_dir = 1
16:58:12         is_linkfile = 0
16:58:12         follow_link = 0
16:58:12         copy = 0x0
16:58:12         copy_local = 0x0
16:58:12         gfid = "a8c9cc71-1430-4217-86e6-f32eeb69d4ce"
16:58:12         vol_commit_hash = 0
16:58:12         subvol = 0x0
16:58:12         check_mds = 0
16:58:12         errst = 0
16:58:12         mds_xattr_val = {0}
16:58:12         __FUNCTION__ = "dht_revalidate_cbk"
16:58:12 #2  0x00007f34eea387a1 in afr_lookup_done (frame=0x7f34e80970b8, this=0x7f34e8020b50) at /home/jenkins/root/workspace/regression-on-demand-full-run/xlators/cluster/afr/src/afr-common.c:2499
16:58:12         fn = 0x7f34ee710dde <dht_revalidate_cbk>
16:58:12         _parent = 0x7f34e80a44d8
16:58:12         old_THIS = 0x7f34e8020b50
16:58:12         __local = 0x7f34e8099198
16:58:12         __this = 0x7f34e8020b50
16:58:12         __op_ret = 0
16:58:12         __op_errno = 0
16:58:12         priv = 0x7f34e805efe0
16:58:12         local = 0x7f34e8099198
16:58:12         i = 3
16:58:12         op_errno = 0
16:58:12         read_subvol = 1
16:58:12         par_read_subvol = 1
16:58:12         ret = -2
16:58:12         readable = 0x7f34ef721330 ""
16:58:12         success_replies = 0x7f34ef721310 "\001\001\001\357\064\177"
16:58:12         event = 3
16:58:12         replies = 0x7f34e80a11f0
16:58:12         read_gfid = "\250\311\314q\024\060B\027\206\346\363.\353i\324", <incomplete sequence \316>
16:58:12         locked_entry = false
16:58:12         can_interpret = true
16:58:12         parent = 0x7f34e8001de8
16:58:12         ia_type = IA_IFDIR
16:58:12         args = {ia_type = IA_IFDIR, gfid = "\250\311\314q\024\060B\027\206\346\363.\353i\324", <incomplete sequence \316>}
16:58:12         gfid_heal_msg = 0x0
16:58:12         __FUNCTION__ = "afr_lookup_done"
16:58:12 #3  0x00007f34eea39c93 in afr_lookup_metadata_heal_check (frame=0x7f34e80970b8, this=0x7f34e8020b50) at /home/jenkins/root/workspace/regression-on-demand-full-run/xlators/cluster/afr/src/afr-common.c:2807
16:58:12         heal = 0x0
16:58:12         local = 0x7f34e8099198
16:58:12         ret = 0
16:58:12 #4  0x00007f34eea3a87b in afr_lookup_entry_heal (frame=0x7f34e80970b8, this=0x7f34e8020b50) at /home/jenkins/root/workspace/regression-on-demand-full-run/xlators/cluster/afr/src/afr-common.c:2955
16:58:12         local = 0x7f34e8099198
16:58:12         priv = 0x7f34e805efe0
16:58:12         heal = 0x0
16:58:12         i = 3
16:58:12         first = 0
16:58:12         name_state_mismatch = false
16:58:12         replies = 0x7f34e80a11f0
16:58:12         ret = 0
16:58:12         par_readables = 0x7f34ef721500 ""
16:58:12         success = 0x7f34ef7214e0 "\001\001\001"
16:58:12         op_errno = 0
16:58:12         gfid = "\250\311\314q\024\060B\027\206\346\363.\353i\324", <incomplete sequence \316>
16:58:12 #5  0x00007f34eea3aab5 in afr_lookup_cbk (frame=0x7f34e80970b8, cookie=0x1, this=0x7f34e8020b50, op_ret=0, op_errno=0, inode=0x7f34dc02bd98, buf=0x7f34ef721750, xdata=0x7f34e80c5b78, postparent=0x7f34ef7216b0) at /home/jenkins/root/workspace/regression-on-demand-full-run/xlators/cluster/afr/src/afr-common.c:3002
16:58:12         local = 0x7f34e8099198
16:58:12         call_count = 0
16:58:12         child_index = 1
16:58:12         ret = 0
16:58:12         need_heal = 1 '\001'
16:58:12 #6  0x00007f34eecf5a51 in client4_0_lookup_cbk (req=0x7f34e807a038, iov=0x7f34e807a070, count=1, myframe=0x7f34e80a4238) at /home/jenkins/root/workspace/regression-on-demand-full-run/xlators/protocol/client/src/client-rpc-fops_v2.c:2641
16:58:12         fn = 0x7f34eea3a887 <afr_lookup_cbk>
16:58:12         _parent = 0x7f34e80970b8
16:58:12         old_THIS = 0x7f34e800d050
16:58:12         __local = 0x7f34e80a5248
16:58:12         rsp = {op_ret = 0, op_errno = 0, xdata = {xdr_size = 436, count = 9, pairs = {pairs_len = 9, pairs_val = 0x7f34e80c65b0}}, prestat = {ia_gfid = "\250\311\314q\024\060B\027\206\346\363.\353i\324", <incomplete sequence \316>, ia_flags = 6143, ia_ino = 9720724228569421006, ia_dev = 1792, ia_rdev = 0, ia_size = 6, ia_blocks = 0, ia_attributes = 0, ia_attributes_mask = 0, ia_atime = 1552377223, ia_mtime = 1552377223, ia_ctime = 1552377261, ia_btime = 0, ia_atime_nsec = 225620000, ia_mtime_nsec = 225620000, ia_ctime_nsec = 95181074, ia_btime_nsec = 0, ia_nlink = 2, ia_uid = 0, ia_gid = 0, ia_blksize = 4096, mode = 16877}, poststat = {ia_gfid = '\000' <repeats 15 times>, "\001", ia_flags = 6143, ia_ino = 1, ia_dev = 1792, ia_rdev = 0, ia_size = 4096, ia_blocks = 8, ia_attributes = 0, ia_attributes_mask = 0, ia_atime = 1552377217, ia_mtime = 1552377342, ia_ctime = 1552377342, ia_btime = 0, ia_atime_nsec = 889585720, ia_mtime_nsec = 159385135, ia_ctime_nsec = 159385135, ia_btime_nsec = 0, ia_nlink = 17, ia_uid = 0, ia_gid = 0, ia_blksize = 4096, mode = 16877}}
16:58:12         local = 0x7f34e80a5248
16:58:12         frame = 0x7f34e80a4238
16:58:12         ret = 0
16:58:12         stbuf = {ia_flags = 6143, ia_ino = 9720724228569421006, ia_dev = 1792, ia_rdev = 0, ia_size = 6, ia_nlink = 2, ia_uid = 0, ia_gid = 0, ia_blksize = 4096, ia_blocks = 0, ia_atime = 1552377223, ia_mtime = 1552377223, ia_ctime = 1552377261, ia_btime = 0, ia_atime_nsec = 225620000, ia_mtime_nsec = 225620000, ia_ctime_nsec = 95181074, ia_btime_nsec = 0, ia_attributes = 0, ia_attributes_mask = 0, ia_gfid = "\250\311\314q\024\060B\027\206\346\363.\353i\324", <incomplete sequence \316>, ia_type = IA_IFDIR, ia_prot = {suid = 0 '\000', sgid = 0 '\000', sticky = 0 '\000', owner = {read = 1 '\001', write = 1 '\001', exec = 1 '\001'}, group = {read = 1 '\001', write = 0 '\000', exec = 1 '\001'}, other = {read = 1 '\001', write = 0 '\000', exec = 1 '\001'}}}
16:58:12         postparent = {ia_flags = 6143, ia_ino = 1, ia_dev = 1792, ia_rdev = 0, ia_size = 4096, ia_nlink = 17, ia_uid = 0, ia_gid = 0, ia_blksize = 4096, ia_blocks = 8, ia_atime = 1552377217, ia_mtime = 1552377342, ia_ctime = 1552377342, ia_btime = 0, ia_atime_nsec = 889585720, ia_mtime_nsec = 159385135, ia_ctime_nsec = 159385135, ia_btime_nsec = 0, ia_attributes = 0, ia_attributes_mask = 0, ia_gfid = '\000' <repeats 15 times>, "\001", ia_type = IA_IFDIR, ia_prot = {suid = 0 '\000', sgid = 0 '\000', sticky = 0 '\000', owner = {read = 1 '\001', write = 1 '\001', exec = 1 '\001'}, group = {read = 1 '\001', write = 0 '\000', exec = 1 '\001'}, other = {read = 1 '\001', write = 0 '\000', exec = 1 '\001'}}}
16:58:12         op_errno = 0
16:58:12         xdata = 0x7f34e80c5b78
16:58:12         inode = 0x7f34dc02bd98
16:58:12         this = 0x7f34e800d050
16:58:12         __FUNCTION__ = "client4_0_lookup_cbk"
16:58:12 #7  0x00007f34fd4105d0 in rpc_clnt_handle_reply (clnt=0x7f34e8073ce0, pollin=0x7f34e80a4730) at /home/jenkins/root/workspace/regression-on-demand-full-run/rpc/rpc-lib/src/rpc-clnt.c:764
16:58:12         conn = 0x7f34e8073d10
16:58:12         saved_frame = 0x7f34e80a2db8
16:58:12         ret = 0
16:58:12         req = 0x7f34e807a038
16:58:12         xid = 59
16:58:12         __FUNCTION__ = "rpc_clnt_handle_reply"
16:58:12 #8  0x00007f34fd410af9 in rpc_clnt_notify (trans=0x7f34e8073ff0, mydata=0x7f34e8073d10, event=RPC_TRANSPORT_MSG_RECEIVED, data=0x7f34e80a4730) at /home/jenkins/root/workspace/regression-on-demand-full-run/rpc/rpc-lib/src/rpc-clnt.c:931
16:58:12         conn = 0x7f34e8073d10
16:58:12         clnt = 0x7f34e8073ce0
16:58:12         ret = -1
16:58:12         req_info = 0x0
16:58:12         pollin = 0x7f34e80a4730
16:58:12         clnt_mydata = 0x0
16:58:12         old_THIS = 0x7f34e800d050
16:58:12         __FUNCTION__ = "rpc_clnt_notify"
16:58:12 #9  0x00007f34fd40cade in rpc_transport_notify (this=0x7f34e8073ff0, event=RPC_TRANSPORT_MSG_RECEIVED, data=0x7f34e80a4730) at /home/jenkins/root/workspace/regression-on-demand-full-run/rpc/rpc-lib/src/rpc-transport.c:544
16:58:12         ret = -1
16:58:12         __FUNCTION__ = "rpc_transport_notify"
16:58:12 #10 0x00007f34f17d8a06 in socket_event_poll_in_async (xl=0x7f34e800d050, async=0x7f34e80a4858) at /home/jenkins/root/workspace/regression-on-demand-full-run/rpc/rpc-transport/socket/src/socket.c:2509
16:58:12         pollin = 0x7f34e80a4730
16:58:12         this = 0x7f34e8073ff0
16:58:12         priv = 0x7f34e8074670
16:58:12 #11 0x00007f34f17d02ec in gf_async (async=0x7f34e80a4858, xl=0x7f34e800d050, cbk=0x7f34f17d89af <socket_event_poll_in_async>) at /home/jenkins/root/workspace/regression-on-demand-full-run/libglusterfs/src/glusterfs/async.h:189
16:58:12         __FUNCTION__ = "gf_async"
16:58:12 #12 0x00007f34f17d8b94 in socket_event_poll_in (this=0x7f34e8073ff0, notify_handled=true) at /home/jenkins/root/workspace/regression-on-demand-full-run/rpc/rpc-transport/socket/src/socket.c:2550
16:58:12         ret = 0
16:58:12         pollin = 0x7f34e80a4730
16:58:12         priv = 0x7f34e8074670
16:58:12         ctx = 0xbd2010
16:58:12 #13 0x00007f34f17d9b33 in socket_event_handler (fd=12, idx=3, gen=4, data=0x7f34e8073ff0, poll_in=1, poll_out=0, poll_err=0, event_thread_died=0 '\000') at /home/jenkins/root/workspace/regression-on-demand-full-run/rpc/rpc-transport/socket/src/socket.c:2941
16:58:12         this = 0x7f34e8073ff0
16:58:12         priv = 0x7f34e8074670
16:58:12         ret = 0
16:58:12         ctx = 0xbd2010
16:58:12         socket_closed = false
16:58:12         notify_handled = false
16:58:12         __FUNCTION__ = "socket_event_handler"
16:58:12 #14 0x00007f34fd6ec9f3 in event_dispatch_epoll_handler (event_pool=0xc08e90, event=0x7f34ef721e80) at /home/jenkins/root/workspace/regression-on-demand-full-run/libglusterfs/src/event-epoll.c:648
16:58:12         ev_data = 0x7f34ef721e84
16:58:12         slot = 0xc4ff20
16:58:12         handler = 0x7f34f17d968e <socket_event_handler>
16:58:12         data = 0x7f34e8073ff0
16:58:12         idx = 3
16:58:12         gen = 4
16:58:12         ret = 0
16:58:12         fd = 12
16:58:12         handled_error_previously = false
16:58:12         __FUNCTION__ = "event_dispatch_epoll_handler"
16:58:12 #15 0x00007f34fd6ecf0c in event_dispatch_epoll_worker (data=0xc6c150) at /home/jenkins/root/workspace/regression-on-demand-full-run/libglusterfs/src/event-epoll.c:761
16:58:12         event = {events = 1, data = {ptr = 0x400000003, fd = 3, u32 = 3, u64 = 17179869187}}
16:58:12         ret = 1
16:58:12         ev_data = 0xc6c150
16:58:12         event_pool = 0xc08e90
16:58:12         myindex = 2
16:58:12         timetodie = 0
16:58:12         gen = 0
16:58:12         poller_death_notify = {next = 0x0, prev = 0x0}
16:58:12         slot = 0x0
16:58:12         tmp = 0x0
16:58:12         __FUNCTION__ = "event_dispatch_epoll_worker"
16:58:12 #16 0x00007f34fc498dd5 in start_thread () from /lib64/libpthread.so.0

Version-Release number of selected component (if applicable):


How reproducible:

very rare.

Steps to Reproduce:


1.run test tests/00-geo-rep/georep-basic-dr-rsync-arbiter.t
2.
3.

Actual results:


Expected results:


Additional info:

https://build.gluster.org/job/regression-on-demand-full-run/259/consoleFull