Bug 762600 (GLUSTER-868)

Summary: crash in ra_fstat
Product: [Community] GlusterFS Reporter: Anand Avati <aavati>
Component: read-aheadAssignee: Raghavendra G <raghavendra>
Status: CLOSED CURRENTRELEASE QA Contact:
Severity: low Docs Contact:
Priority: low    
Version: 3.0.3CC: amarts, chrisw, gluster-bugs, joe, pavan
Target Milestone: ---   
Target Release: ---   
Hardware: All   
OS: Linux   
Whiteboard:
Fixed In Version: Doc Type: Bug Fix
Doc Text:
Story Points: ---
Clone Of: Environment:
Last Closed: Type: ---
Regression: RTP Mount Type: ---
Documentation: --- CRM:
Verified Versions: Category: ---
oVirt Team: --- RHEL 7.3 requirements from Atomic Host:
Cloudforms Team: --- Target Upstream Version:

Description Anand Avati 2010-04-28 18:00:57 UTC
Report from Joe Warren-Meeks <joe.uk> on gluster-users (Subject: [Gluster-users] Transport endpoint not connected)

Thread 3 (process 13560):
#0  0x00007ffd0db44402 in ?? () from /lib/libpthread.so.0
No symbol table info available.
#1  0x00007ffd0c7600ff in ioc_open_cbk (frame=0x122c6a0,
cookie=0x122c7d0,
   this=0x6104e0, op_ret=0, op_errno=117, fd=0x12af1f0) at
io-cache.c:474
       tmp_ioc_inode = 0
       local = (ioc_local_t *) 0x122c700
       table = <value optimized out>
       ioc_inode = <value optimized out>
       inode = (inode_t *) 0x61d220
       weight = <value optimized out>
       path = 0x12af130 "/docs/cbolds/PAGES/V1/THEATRENET"
       __FUNCTION__ = "ioc_open_cbk"
#2  0x00007ffd0c96a395 in ra_open_cbk (frame=0x122c7d0,
   cookie=<value optimized out>, this=<value optimized out>, op_ret=0,
   op_errno=117, fd=0x12af1f0) at read-ahead.c:116
       fn = (fop_open_cbk_t) 0x7ffd0c760090 <ioc_open_cbk>
       _parent = (call_frame_t *) 0x122c6a0
       old_THIS = (xlator_t *) 0x60fbe0
       file = <value optimized out>
       __FUNCTION__ = "ra_open_cbk"
#3  0x00007ffd0cb8f4a2 in afr_open_cbk (frame=0x122c830, cookie=0x1,
   this=<value optimized out>, op_ret=<value optimized out>,
   op_errno=<value optimized out>, fd=0x12af1f0) at afr-open.c:140
       fn = (fop_open_cbk_t) 0x7ffd0c96a1f0 <ra_open_cbk>
       _parent = (call_frame_t *) 0x122c7d0
       old_THIS = (xlator_t *) 0x60f390
       __local = (afr_local_t *) 0x122c890
       __this = (xlator_t *) 0x60f390
       local = (afr_local_t *) 0x122c890
       ctx = 140724576331344
       fd_ctx = <value optimized out>
       ret = <value optimized out>
       call_count = <value optimized out>
       __FUNCTION__ = "afr_open_cbk"
#4  0x00007ffd0cdbe27b in client_open_cbk (frame=0x12af600,
   hdr=<value optimized out>, hdrlen=<value optimized out>,
   iobuf=<value optimized out>) at client-protocol.c:4090
       fn = (ret_fn_t) 0x7ffd0cb8f280 <afr_open_cbk>
       _parent = (call_frame_t *) 0x122c830
       old_THIS = (xlator_t *) 0x60e110
       op_ret = 0
       op_errno = 0
       fd = (fd_t *) 0x12af1f0
       local = (client_local_t *) 0x12af840
       fdctx = <value optimized out>
       ino = 3793176
       gen = 5462270833604952119
#5  0x00007ffd0cdb8a2a in protocol_client_pollin (this=0x60e110,
   trans=0x6151d0) at client-protocol.c:6827
       conf = (client_conf_t *) 0x614bd0
       ret = 0
       iobuf = (struct iobuf *) 0x0
       hdr = 0x7ffcfe6249a0 ""
       hdrlen = 116
#6  0x00007ffd0cdbfc7a in notify (this=0x61d228, event=<value optimized
out>,
   data=0x6151d0) at client-protocol.c:6946
       ret = <value optimized out>
       child_down = <value optimized out>
       was_not_down = <value optimized out>
       trans = (transport_t *) 0x122c7d0
       conn = <value optimized out>
       conf = (client_conf_t *) 0x614bd0
       parent = <value optimized out>
       __FUNCTION__ = "notify"
#7  0x00007ffd0df6e033 in xlator_notify (xl=0x60e110, event=2,
data=0x6151d0)
   at xlator.c:924
       old_THIS = (xlator_t *) 0x7ffd0e19ec00
       ret = <value optimized out>
#8  0x00007ffd0bd1345b in socket_event_handler (fd=<value optimized
out>,
   idx=3, data=0x6151d0, poll_in=1, poll_out=0, poll_err=0) at
socket.c:831
       this = (transport_t *) 0x61d228
       priv = (socket_private_t *) 0x615540
       ret = 0
#9  0x00007ffd0df8784a in event_dispatch_epoll (event_pool=0x609930)
   at event.c:804
       events = (struct epoll_event *) 0x6166e0
       i = 1
       ret = 2
       __FUNCTION__ = "event_dispatch_epoll"
#10 0x0000000000404533 in main (argc=4, argv=0x7fff5b5ccac8)
   at glusterfsd.c:1425
       ctx = (glusterfs_ctx_t *) 0x608010
       cmd_args = <value optimized out>
       stbuf = {st_dev = 0, st_ino = 140734726193120, st_nlink = 0,
 st_mode = 236629549, st_uid = 32765, st_gid = 0, pad0 = 0, st_rdev =
0,
 st_size = 0, st_blksize = 140724840091158, st_blocks =
140734726194672,
 st_atim = {tv_sec = 140724840082928, tv_nsec = 140734726194735},
st_mtim = {
   tv_sec = 140734726194720, tv_nsec = 140734726194712}, st_ctim = {
   tv_sec = 140724842257208, tv_nsec = 4199943}, __unused = {0, 0,
   140724829864661}}
       tmp_logfile = '\0' <repeats 1023 times>
       tmp_logfile_dyn = <value optimized out>
       tmp_logfilebase = <value optimized out>
       timestr = '\0' <repeats 255 times>
       utime = 1271949306
       tm = <value optimized out>
       ret = 0
       lim = {rlim_cur = 18446744073709551615,
 rlim_max = 18446744073709551615}
       specfp = <value optimized out>
       graph = (xlator_t *) 0x609e70
       trav = <value optimized out>
       fuse_volume_found = 0
       xl_count = <value optimized out>
       pipe_fd = {6, 7}
       gf_success = 0
       gf_failure = -1
       __FUNCTION__ = "main"

Thread 2 (process 13561):
#0  0x00007ffd0d874b81 in nanosleep () from /lib/libc.so.6
No symbol table info available.
#1  0x00007ffd0d8a8584 in usleep () from /lib/libc.so.6
No symbol table info available.
#2  0x00007ffd0df79d63 in gf_timer_proc (ctx=0x608010) at timer.c:177
       now = 1272055239910113
       now_tv = {tv_sec = 1272055239, tv_usec = 910113}
       event = (gf_timer_t *) 0x7ffcfe587110
       reg = (gf_timer_registry_t *) 0x612fe0
       __FUNCTION__ = "gf_timer_proc"
#3  0x00007ffd0db3f3f7 in start_thread () from /lib/libpthread.so.0
No symbol table info available.
#4  0x00007ffd0d8aeb4d in clone () from /lib/libc.so.6
No symbol table info available.
#5  0x0000000000000000 in ?? ()
No symbol table info available.

Thread 1 (process 13565):
#0  0x00007ffd0c968d22 in ra_fstat (frame=0x127e200, this=0x60fbe0,
   fd=0x7ffd05e1ffe0) at read-ahead.c:750
       iter_fd = (fd_t *) 0x12af1f0
       inode = <value optimized out>
       tmp_file = 140724576339264
#1  0x00007ffd0df7411b in default_fstat (frame=<value optimized out>,
   this=0x6104e0, fd=0x7ffd05e1ffe0) at defaults.c:673
       _new = (call_frame_t *) 0x127e200
       old_THIS = (xlator_t *) 0x6104e0
#2  0x00007ffd0c5570a3 in qr_fstat (frame=0x127e710, this=0x610e90,
   fd=0x7ffd05e1ffe0) at quick-read.c:1184
       _new = (call_frame_t *) 0x127e200
       old_THIS = (xlator_t *) 0x610e90
       qr_fd_ctx = <value optimized out>
       need_open = 0 '\0'
       can_wind = 1 '\001'
       need_unwind = 0 '\0'
       value = 140724702175744
       ret = <value optimized out>
       op_ret = <value optimized out>
       op_errno = -1
       stub = <value optimized out>
       loc = {path = 0x0, name = 0x0, ino = 0, inode = 0x0, parent =
0x0}
       path = <value optimized out>
       flags = 32768
#3  0x00007ffd0c346adb in wb_fstat_helper (frame=<value optimized out>,
   this=0x611850, fd=0x7ffd05e1ffe0) at write-behind.c:698
       _new = (call_frame_t *) 0x127e200
       old_THIS = (xlator_t *) 0x611850
#4  0x00007ffd0df7cf60 in call_resume (stub=0x122e9c0) at
call-stub.c:2607
       old_THIS = (xlator_t *) 0x611850
       __FUNCTION__ = "call_resume"
#5  0x00007ffd0c349938 in wb_resume_other_requests (frame=0x11e8130,
   file=0x7ffd05e251c0, other_requests=0x40a0cf60) at
write-behind.c:1621
       request = (wb_request_t *) 0x11e7c00
       dummy = (wb_request_t *) 0x40a0cf30
       fops_removed = 0
       wind = 1 '\001'
#6  0x00007ffd0c348251 in wb_process_queue (frame=0x11e8130,
   file=0x7ffd05e251c0, flush_all=1 '\001') at write-behind.c:1817
       winds = {next = 0x40a0cf80, prev = 0x40a0cf80}
       unwinds = {next = 0x40a0cf70, prev = 0x40a0cf70}
       other_requests = {next = 0x11e7c30, prev = 0x11e7c30}
       size = 131072
       conf = (wb_conf_t *) 0x6133d0
       count = <value optimized out>
       ret = <value optimized out>
#7  0x00007ffd0c34a87a in wb_fstat (frame=0x11e8130,
   this=<value optimized out>, fd=0x7ffd05e1ffe0) at write-behind.c:755
       file = (wb_file_t *) 0x7ffcfe624540
       tmp_file = 140724702171584
       stub = (call_stub_t *) 0x122e9c0
       request = (wb_request_t *) 0x0
       ret = <value optimized out>
       __FUNCTION__ = "wb_fstat"
#8  0x00007ffd0df7411b in default_fstat (frame=<value optimized out>,
   this=0x612180, fd=0x7ffd05e1ffe0) at defaults.c:673
       _new = (call_frame_t *) 0x127e200
       old_THIS = (xlator_t *) 0x612180
#9  0x00007ffd0bf23a36 in fuse_getattr (this=<value optimized out>,
   finh=<value optimized out>, msg=<value optimized out>) at
fuse-bridge.c:812
       _new = (call_frame_t *) 0x127e200
       size = 131072
       conf = (wb_conf_t *) 0x6133d0
       count = <value optimized out>
       ret = <value optimized out>
#7  0x00007ffd0c34a87a in wb_fstat (frame=0x11e8130,
   this=<value optimized out>, fd=0x7ffd05e1ffe0) at write-behind.c:755
       file = (wb_file_t *) 0x7ffcfe624540
       tmp_file = 140724702171584
       stub = (call_stub_t *) 0x122e9c0
       request = (wb_request_t *) 0x0
       ret = <value optimized out>
       __FUNCTION__ = "wb_fstat"
#8  0x00007ffd0df7411b in default_fstat (frame=<value optimized out>,
   this=0x612180, fd=0x7ffd05e1ffe0) at defaults.c:673
       _new = (call_frame_t *) 0x127e200
       old_THIS = (xlator_t *) 0x612180
#9  0x00007ffd0bf23a36 in fuse_getattr (this=<value optimized out>,
   finh=<value optimized out>, msg=<value optimized out>) at
fuse-bridge.c:812
       _new = (call_frame_t *) 0x127e200
---Type <return> to continue, or q <return> to quit---
       old_THIS = (xlator_t *) 0x609e70
       frame = <value optimized out>
       xl = (xlator_t *) 0x612180
       state = (fuse_state_t *) 0x12afa90
       fd = (fd_t *) 0x7ffd05e1ffe0
       ret = <value optimized out>
       __FUNCTION__ = "fuse_getattr"
#10 0x00007ffd0bf246b6 in fuse_thread_proc (data=<value optimized out>)
   at fuse-bridge.c:3190
       mount_point = <value optimized out>
       this = (xlator_t *) 0x609e70
       priv = (fuse_private_t *) 0x6131c0
       res = 56
       iobuf = (struct iobuf *) 0x7ffcfe450e40
       finh = (fuse_in_header_t *) 0x12afb80
       iov_in = {{iov_base = 0x12afb80, iov_len = 80}, {
   iov_base = 0x7ffcfb0c0000, iov_len = 131072}}
       msg = (void *) 0x0
       ret = <value optimized out>
       now = {tv_sec = 1271949306, tv_usec = 169347}
       timeout = {tv_sec = 1271949307, tv_nsec = 169347000}
       __FUNCTION__ = "fuse_thread_proc"
#11 0x00007ffd0db3f3f7 in start_thread () from /lib/libpthread.so.0
No symbol table info available.
#12 0x00007ffd0d8aeb4d in clone () from /lib/libc.so.6
No symbol table info available.
#13 0x0000000000000000 in ?? ()
No symbol table info available.

Comment 1 Anand Avati 2010-05-04 07:32:08 UTC
PATCH: http://patches.gluster.com/patch/3214 in master (performance/read-ahead: don't set ra_file in fd->ctx unless all memebers of ra_file is initialized)

Comment 2 Anand Avati 2010-05-04 07:32:12 UTC
PATCH: http://patches.gluster.com/patch/3212 in release-2.0 (performance/read-ahead: don't set ra_file in fd->ctx unless all memebers of ra_file is initialized)

Comment 3 Anand Avati 2010-05-04 07:32:16 UTC
PATCH: http://patches.gluster.com/patch/3213 in release-3.0 (performance/read-ahead: don't set ra_file in fd->ctx unless all memebers of ra_file is initialized)