Hide Forgot
*** This bug has been marked as a duplicate of bug 1223 ***
Glusterfsd core dumped while running nfstests log file ------- cat gs2.log [2010-08-05 08:05:21.123316] N [server-handshake.c:518:server_setvolume] server-tcp: accepted client from 10.198.110.16:1021 [2010-08-05 08:05:26.158972] N [server-handshake.c:518:server_setvolume] server-tcp: accepted client from 10.214.231.112:1023 [2010-08-05 08:05:28.817348] N [server-handshake.c:518:server_setvolume] server-tcp: accepted client from 10.192.134.144:1021 [2010-08-05 08:05:32.58233] N [server-handshake.c:518:server_setvolume] server-tcp: accepted client from 10.192.141.187:1021 [2010-08-05 08:21:22.606596] E [posix.c:572:posix_stat] posix: lstat on /iozone.tmp failed: No such file or directory pending frames: patchset: v3.1.0qa3 signal received: 11 time of crash: 2010-08-05 09:59:04 configuration details: argp 1 backtrace 1 dlfcn 1 fdatasync 1 libpthread 1 llistxattr 1 setfsid 1 spinlock 1 epoll.h 1 xattr.h 1 st_atim.tv_nsec 1 package-string: glusterfs 3.1.0qa3 [2010-08-05 09:59:04.418648] N [server-handshake.c:518:server_setvolume] server-tcp: accepted client from 10.192.141.187:1021 /lib64/libc.so.6[0x2aaaab791f30] /lib64/libpthread.so.0(pthread_mutex_lock+0x13)[0x2aaaab54e343] /opt/glusterfs/3.1.0qa3//lib/libglusterfs.so.0(iobuf_get+0x18)[0x2aaaaacf8578] /opt/glusterfs/3.1.0qa3//lib/libgfrpc.so.0(rpc_transport_same_process_pollin_alloc+0x16a)[0x2aaaaaf1b18a] /opt/glusterfs/3.1.0qa3//lib/libgfrpc.so.0(rpc_transport_submit_reply+0x73)[0x2aaaaaf1b603] /opt/glusterfs/3.1.0qa3//lib/libgfrpc.so.0(rpcsvc_conn_submit+0x7f)[0x2aaaaaf17d1f] /opt/glusterfs/3.1.0qa3//lib/libgfrpc.so.0(rpcsvc_submit_generic+0x152)[0x2aaaaaf18b22] /opt/glusterfs/3.1.0qa3//lib/glusterfs/3.1.0qa3/xlator/protocol/server.so(server_submit_reply+0xc2)[0x2aaaacb6a132] /opt/glusterfs/3.1.0qa3//lib/glusterfs/3.1.0qa3/xlator/protocol/server.so(server_writev_cbk+0x304)[0x2aaaacb78854] /opt/glusterfs/3.1.0qa3//lib/glusterfs/3.1.0qa3/xlator/performance/io-threads.so(iot_writev_cbk+0x9d)[0x2aaaac9554cd] /opt/glusterfs/3.1.0qa3//lib/glusterfs/3.1.0qa3/xlator/features/locks.so(pl_writev_cbk+0x9d)[0x2aaaac745d5d] /opt/glusterfs/3.1.0qa3//lib/libglusterfs.so.0(default_writev_cbk+0x9f)[0x2aaaaace245f] /opt/glusterfs/3.1.0qa3//lib/glusterfs/3.1.0qa3/xlator/storage/posix.so(posix_writev+0x25d)[0x2aaaac32515d] /opt/glusterfs/3.1.0qa3//lib/glusterfs/3.1.0qa3/xlator/features/access-control.so(ac_writev_resume+0x119)[0x2aaaac537599] /opt/glusterfs/3.1.0qa3//lib/libglusterfs.so.0(call_resume+0x87d)[0x2aaaaacea79d] /opt/glusterfs/3.1.0qa3//lib/glusterfs/3.1.0qa3/xlator/features/access-control.so(ac_writev_fstat_cbk+0x7b)[0x2aaaac53929b] /opt/glusterfs/3.1.0qa3//lib/glusterfs/3.1.0qa3/xlator/storage/posix.so(posix_fstat+0x11a)[0x2aaaac3243da] /opt/glusterfs/3.1.0qa3//lib/glusterfs/3.1.0qa3/xlator/features/access-control.so(ac_writev+0x104)[0x2aaaac53ade4] /opt/glusterfs/3.1.0qa3//lib/glusterfs/3.1.0qa3/xlator/features/locks.so(pl_writev+0x174)[0x2aaaac747e24] /opt/glusterfs/3.1.0qa3//lib/glusterfs/3.1.0qa3/xlator/performance/io-threads.so(iot_writev_wrapper+0x119)[0x2aaaac958589] /opt/glusterfs/3.1.0qa3//lib/libglusterfs.so.0(call_resume+0x87d)[0x2aaaaacea79d] /opt/glusterfs/3.1.0qa3//lib/glusterfs/3.1.0qa3/xlator/performance/io-threads.so(iot_worker+0xd6)[0x2aaaac95d9c6] /lib64/libpthread.so.0[0x2aaaab54c407] /lib64/libc.so.6(clone+0x6d)[0x2aaaab835b0d] --------- ------ (gdb) bt full #0 0x00002aaaab54e343 in pthread_mutex_lock () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00002aaaaacf8578 in iobuf_get (iobuf_pool=0x0) at iobuf.c:356 iobuf = (struct iobuf *) 0x0 iobuf_arena = <value optimized out> #2 0x00002aaaaaf1b18a in rpc_transport_same_process_pollin_alloc (this=0x639970, rpchdr=0x418048c0, rpchdrcount=1, proghdr=0x41804930, proghdrcount=1, progpayload=0x0, progpayloadcount=0, rsp=0x0, is_request=0 '\0') at rpc-transport.c:769 msg = (rpc_transport_pollin_t *) 0x639c90 progpayloadlen = 0 vectored = <value optimized out> hdr = <value optimized out> progpayloadbuf = <value optimized out> iobuf = <value optimized out> __FUNCTION__ = "rpc_transport_same_process_pollin_alloc" #3 0x00002aaaaaf1b603 in rpc_transport_submit_reply (this=0x0, reply=<value optimized out>) at rpc-transport.c:1091 ret = <value optimized out> pollin = <value optimized out> handover_msg = <value optimized out> __FUNCTION__ = "rpc_transport_submit_reply" #4 0x00002aaaaaf17d1f in rpcsvc_conn_submit (conn=0x0, hdrvec=<value optimized out>, hdrcount=72, proghdr=0x41804930, proghdrcount=1, progpayload=0x1, progpayloadcount=0, iobref=0x639c40, priv=0x2aaab0065010) at rpcsvc.c:1374 ret = <value optimized out> reply = {msg = {rpchdr = 0x418048c0, rpchdrcount = 1, proghdr = 0x41804930, proghdrcount = 1, progpayload = 0x0, progpayloadcount = 0, iobref = 0x639c40}, private = 0x0} __FUNCTION__ = "rpcsvc_conn_submit" #5 0x00002aaaaaf18b22 in rpcsvc_submit_generic (req=0x2aaaad2b5940, proghdr=0x41804930, hdrcount=1, payload=0x0, payloadcount=0, iobref=0x639c40) at rpcsvc.c:1535 ret = <value optimized out> i = <value optimized out> replyiob = (struct iobuf *) 0x629898 recordhdr = {iov_base = 0x2aaaac29a000, iov_len = 28} msglen = 200 new_iobref = 0 '\0' __FUNCTION__ = "rpcsvc_submit_generic" #6 0x00002aaaacb6a132 in server_submit_reply (frame=0x2aaab005b378, req=0x2aaaad2b5940, arg=<value optimized out>, payload=0x0, payloadcount=0, iobref=0x639c40, sfunc=0x2aaaab12f540 <xdr_serialize_writev_rsp>) at server.c:123 iob = (struct iobuf *) 0x629488 ret = <value optimized out> Missing separate debuginfos, use: debuginfo-install gcc.x86_64 glibc.x86_64 ---Type <return> to continue, or q <return> to quit--- rsp = {iov_base = 0x2aaaabf5a000, iov_len = 200} state = (server_state_t *) 0x2aaab00599f0 new_iobref = 1 '\001' __FUNCTION__ = "server_submit_reply" #7 0x00002aaaacb78854 in server_writev_cbk (frame=0x2aaab005b378, cookie=<value optimized out>, this=0x0, op_ret=16843520, op_errno=0, prebuf=0x1, postbuf=0x41804c30) at server3_1-fops.c:1235 rsp = {gfs_id = 0, op_ret = 65536, op_errno = 0, prestat = {ia_ino = 39846415, ia_gen = 5501895600582451836, ia_dev = 2064, mode = 33188, ia_nlink = 1, ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 254177280, ia_blksize = 4096, ia_blocks = 496944, ia_atime = 1281016557, ia_atime_nsec = 0, ia_mtime = 1281016744, ia_mtime_nsec = 0, ia_ctime = 1281016744, ia_ctime_nsec = 0}, poststat = {ia_ino = 39846415, ia_gen = 5501895600582451836, ia_dev = 2064, mode = 33188, ia_nlink = 1, ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 254242816, ia_blksize = 4096, ia_blocks = 497072, ia_atime = 1281016557, ia_atime_nsec = 0, ia_mtime = 1281016744, ia_mtime_nsec = 0, ia_ctime = 1281016744, ia_ctime_nsec = 0}} req = (rpcsvc_request_t *) 0x2aaaad2b5940 __FUNCTION__ = "server_writev_cbk" #8 0x00002aaaac9554cd in iot_writev_cbk (frame=0x2aaab0059e40, cookie=<value optimized out>, this=<value optimized out>, op_ret=65536, op_errno=0, prebuf=0x41804ca0, postbuf=0x41804c30) at io-threads.c:945 _parent = (call_frame_t *) 0x2aaab005b378 old_THIS = (xlator_t *) 0x631680 __FUNCTION__ = "iot_writev_cbk" #9 0x00002aaaac745d5d in pl_writev_cbk (frame=0x64c6a0, cookie=<value optimized out>, this=<value optimized out>, op_ret=65536, op_errno=0, prebuf=0x41804ca0, postbuf=0x41804c30) at posix.c:497 _parent = (call_frame_t *) 0x2aaab0059e40 old_THIS = (xlator_t *) 0x630870 __FUNCTION__ = "pl_writev_cbk" #10 0x00002aaaaace245f in default_writev_cbk (frame=0x64b9f0, cookie=<value optimized out>, this=<value optimized out>, op_ret=65536, op_errno=0, prebuf=0x41804ca0, postbuf=0x41804c30) at defaults.c:565 fn = (ret_fn_t) 0x2aaaac745cc0 <pl_writev_cbk> _parent = (call_frame_t *) 0x64c6a0 old_THIS = (xlator_t *) 0x62f540 __FUNCTION__ = "default_writev_cbk" #11 0x00002aaaac32515d in posix_writev (frame=0x639bc0, this=<value optimized out>, fd=0x2aaab0065f40, vector=0x638950, count=1, offset=254177280, iobref=0x2aaab0065010) at posix.c:2550 _parent = (call_frame_t *) 0x64b9f0 old_THIS = (xlator_t *) 0x62e2c0 op_ret = 65536 op_errno = 0 _fd = 111 pfd = <value optimized out> ---Type <return> to continue, or q <return> to quit--- preop = {ia_ino = 39846415, ia_gen = 5501895600582451836, ia_dev = 2064, ia_type = IA_IFREG, ia_prot = {suid = 0 '\0', sgid = 0 '\0', sticky = 0 '\0', owner = {read = 1 '\001', write = 1 '\001', exec = 0 '\0'}, group = {read = 1 '\001', write = 0 '\0', exec = 0 '\0'}, other = {read = 1 '\001', write = 0 '\0', exec = 0 '\0'}}, ia_nlink = 1, ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 254177280, ia_blksize = 4096, ia_blocks = 496944, ia_atime = 1281016557, ia_atime_nsec = 0, ia_mtime = 1281016744, ia_mtime_nsec = 0, ia_ctime = 1281016744, ia_ctime_nsec = 0} postop = {ia_ino = 39846415, ia_gen = 5501895600582451836, ia_dev = 2064, ia_type = IA_IFREG, ia_prot = {suid = 0 '\0', sgid = 0 '\0', sticky = 0 '\0', owner = {read = 1 '\001', write = 1 '\001', exec = 0 '\0'}, group = {read = 1 '\001', write = 0 '\0', exec = 0 '\0'}, other = {read = 1 '\001', write = 0 '\0', exec = 0 '\0'}}, ia_nlink = 1, ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 254242816, ia_blksize = 4096, ia_blocks = 497072, ia_atime = 1281016557, ia_atime_nsec = 0, ia_mtime = 1281016744, ia_mtime_nsec = 0, ia_ctime = 1281016744, ia_ctime_nsec = 0} ret = <value optimized out> tmp_pfd = 46912652881232 __FUNCTION__ = "posix_writev" #12 0x00002aaaac537599 in ac_writev_resume (frame=<value optimized out>, this=0x62f540, fd=0x2aaab0065f40, vector=0x638950, count=1, offset=254177280, iobref=0x2aaab0065010) at access-control.c:1468 _new = (call_frame_t *) 0x0 old_THIS = (xlator_t *) 0x62f540 __FUNCTION__ = "ac_writev_resume" #13 0x00002aaaaacea79d in call_resume (stub=0x639970) at call-stub.c:2235 old_THIS = (xlator_t *) 0x62f540 __FUNCTION__ = "call_resume" #14 0x00002aaaac53929b in ac_writev_fstat_cbk (frame=0x64b9f0, cookie=<value optimized out>, this=<value optimized out>, op_ret=0, op_errno=0, buf=0x41804e60) at access-control.c:1493 stub = (call_stub_t *) 0x639970 __FUNCTION__ = "ac_writev_fstat_cbk" #15 0x00002aaaac3243da in posix_fstat (frame=0x64bd50, this=<value optimized out>, fd=<value optimized out>) at posix.c:3853 _parent = (call_frame_t *) 0x64b9f0 old_THIS = (xlator_t *) 0x62e2c0 op_ret = 0 op_errno = 0 buf = {ia_ino = 39846415, ia_gen = 5501895600582451836, ia_dev = 2064, ia_type = IA_IFREG, ia_prot = {suid = 0 '\0', sgid = 0 '\0', sticky = 0 '\0', owner = {read = 1 '\001', write = 1 '\001', exec = 0 '\0'}, group = {read = 1 '\001', write = 0 '\0', exec = 0 '\0'}, other = {read = 1 '\001', write = 0 '\0', exec = 0 '\0'}}, ia_nlink = 1, ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 254177280, ia_blksize = 4096, ia_blocks = 496944, ia_atime = 1281016557, ia_atime_nsec = 0, ia_mtime = 1281016744, ia_mtime_nsec = 0, ia_ctime = 1281016744, ia_ctime_nsec = 0} tmp_pfd = 46912652881232 ret = <value optimized out> __FUNCTION__ = "posix_fstat" #16 0x00002aaaac53ade4 in ac_writev (frame=<value optimized out>, this=0x62f540, fd=0x2aaab0065f40, vector=<value optimized out>, count=<value optimized out>, offset=<value optimized out>, iobref=0x2aaab0065010) at access-control.c:1522 ---Type <return> to continue, or q <return> to quit--- _new = (call_frame_t *) 0x0 old_THIS = (xlator_t *) 0x62f540 stub = <value optimized out> __FUNCTION__ = "ac_writev" #17 0x00002aaaac747e24 in pl_writev (frame=0x64c6a0, this=0x630870, fd=0x2aaab0065f40, vector=0x2aaab005d1e0, count=1, offset=254177280, iobref=0x2aaab0065010) at posix.c:736 _new = (call_frame_t *) 0x0 old_THIS = (xlator_t *) 0x630870 pl_inode = <value optimized out> rw = <value optimized out> region = {list = {next = 0x0, prev = 0x0}, fl_type = 0, fl_start = 0, fl_end = 0, blocked = 0, user_flock = {l_type = 0, l_whence = 0, l_start = 0, l_len = 0, l_pid = 0}, this = 0x0, fd_num = 0, frame = 0x0, transport = 0x0, client_pid = 0, owner = 0} op_ret = <value optimized out> op_errno = <value optimized out> __FUNCTION__ = "pl_writev" #18 0x00002aaaac958589 in iot_writev_wrapper (frame=<value optimized out>, this=0x631680, fd=0x2aaab0065f40, vector=0x2aaab005d1e0, count=1, offset=254177280, iobref=0x2aaab0065010) at io-threads.c:955 _new = (call_frame_t *) 0x0 old_THIS = (xlator_t *) 0x631680 __FUNCTION__ = "iot_writev_wrapper" #19 0x00002aaaaacea79d in call_resume (stub=0x2aaab0094500) at call-stub.c:2235 old_THIS = (xlator_t *) 0x631680 __FUNCTION__ = "call_resume" #20 0x00002aaaac95d9c6 in iot_worker (data=0x6350c0) at io-threads.c:118 conf = <value optimized out> this = <value optimized out> stub = (call_stub_t *) 0x2aaab0094500 sleep_till = {tv_sec = 1281016864, tv_nsec = 0} ret = <value optimized out> timeout = 0 '\0' bye = 0 '\0' __FUNCTION__ = "iot_worker" #21 0x00002aaaab54c407 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #22 0x00002aaaab835b0d in clone () from /lib64/libc.so.6 No symbol table info available. (gdb)