Bug 763028 (GLUSTER-1296)

Glusterfsd core dumped while running nfstests 
log file
-------
cat gs2.log
[2010-08-05 08:05:21.123316] N [server-handshake.c:518:server_setvolume] server-tcp: accepted client from 10.198.110.16:1021
[2010-08-05 08:05:26.158972] N [server-handshake.c:518:server_setvolume] server-tcp: accepted client from 10.214.231.112:1023
[2010-08-05 08:05:28.817348] N [server-handshake.c:518:server_setvolume] server-tcp: accepted client from 10.192.134.144:1021
[2010-08-05 08:05:32.58233] N [server-handshake.c:518:server_setvolume] server-tcp: accepted client from 10.192.141.187:1021
[2010-08-05 08:21:22.606596] E [posix.c:572:posix_stat] posix: lstat on /iozone.tmp failed: No such file or directory
pending frames:

patchset: v3.1.0qa3
signal received: 11
time of crash: 2010-08-05 09:59:04
configuration details:
argp 1
backtrace 1
dlfcn 1
fdatasync 1
libpthread 1
llistxattr 1
setfsid 1
spinlock 1
epoll.h 1
xattr.h 1
st_atim.tv_nsec 1
package-string: glusterfs 3.1.0qa3
[2010-08-05 09:59:04.418648] N [server-handshake.c:518:server_setvolume] server-tcp: accepted client from 10.192.141.187:1021
/lib64/libc.so.6[0x2aaaab791f30]
/lib64/libpthread.so.0(pthread_mutex_lock+0x13)[0x2aaaab54e343]
/opt/glusterfs/3.1.0qa3//lib/libglusterfs.so.0(iobuf_get+0x18)[0x2aaaaacf8578]
/opt/glusterfs/3.1.0qa3//lib/libgfrpc.so.0(rpc_transport_same_process_pollin_alloc+0x16a)[0x2aaaaaf1b18a]
/opt/glusterfs/3.1.0qa3//lib/libgfrpc.so.0(rpc_transport_submit_reply+0x73)[0x2aaaaaf1b603]
/opt/glusterfs/3.1.0qa3//lib/libgfrpc.so.0(rpcsvc_conn_submit+0x7f)[0x2aaaaaf17d1f]
/opt/glusterfs/3.1.0qa3//lib/libgfrpc.so.0(rpcsvc_submit_generic+0x152)[0x2aaaaaf18b22]
/opt/glusterfs/3.1.0qa3//lib/glusterfs/3.1.0qa3/xlator/protocol/server.so(server_submit_reply+0xc2)[0x2aaaacb6a132]
/opt/glusterfs/3.1.0qa3//lib/glusterfs/3.1.0qa3/xlator/protocol/server.so(server_writev_cbk+0x304)[0x2aaaacb78854]
/opt/glusterfs/3.1.0qa3//lib/glusterfs/3.1.0qa3/xlator/performance/io-threads.so(iot_writev_cbk+0x9d)[0x2aaaac9554cd]
/opt/glusterfs/3.1.0qa3//lib/glusterfs/3.1.0qa3/xlator/features/locks.so(pl_writev_cbk+0x9d)[0x2aaaac745d5d]
/opt/glusterfs/3.1.0qa3//lib/libglusterfs.so.0(default_writev_cbk+0x9f)[0x2aaaaace245f]
/opt/glusterfs/3.1.0qa3//lib/glusterfs/3.1.0qa3/xlator/storage/posix.so(posix_writev+0x25d)[0x2aaaac32515d]
/opt/glusterfs/3.1.0qa3//lib/glusterfs/3.1.0qa3/xlator/features/access-control.so(ac_writev_resume+0x119)[0x2aaaac537599]
/opt/glusterfs/3.1.0qa3//lib/libglusterfs.so.0(call_resume+0x87d)[0x2aaaaacea79d]
/opt/glusterfs/3.1.0qa3//lib/glusterfs/3.1.0qa3/xlator/features/access-control.so(ac_writev_fstat_cbk+0x7b)[0x2aaaac53929b]
/opt/glusterfs/3.1.0qa3//lib/glusterfs/3.1.0qa3/xlator/storage/posix.so(posix_fstat+0x11a)[0x2aaaac3243da]
/opt/glusterfs/3.1.0qa3//lib/glusterfs/3.1.0qa3/xlator/features/access-control.so(ac_writev+0x104)[0x2aaaac53ade4]
/opt/glusterfs/3.1.0qa3//lib/glusterfs/3.1.0qa3/xlator/features/locks.so(pl_writev+0x174)[0x2aaaac747e24]
/opt/glusterfs/3.1.0qa3//lib/glusterfs/3.1.0qa3/xlator/performance/io-threads.so(iot_writev_wrapper+0x119)[0x2aaaac958589]
/opt/glusterfs/3.1.0qa3//lib/libglusterfs.so.0(call_resume+0x87d)[0x2aaaaacea79d]
/opt/glusterfs/3.1.0qa3//lib/glusterfs/3.1.0qa3/xlator/performance/io-threads.so(iot_worker+0xd6)[0x2aaaac95d9c6]
/lib64/libpthread.so.0[0x2aaaab54c407]
/lib64/libc.so.6(clone+0x6d)[0x2aaaab835b0d]
---------
------
(gdb) bt full
#0  0x00002aaaab54e343 in pthread_mutex_lock () from /lib64/libpthread.so.0
No symbol table info available.
#1  0x00002aaaaacf8578 in iobuf_get (iobuf_pool=0x0) at iobuf.c:356
	iobuf = (struct iobuf *) 0x0
	iobuf_arena = <value optimized out>
#2  0x00002aaaaaf1b18a in rpc_transport_same_process_pollin_alloc (this=0x639970, rpchdr=0x418048c0, rpchdrcount=1, proghdr=0x41804930, proghdrcount=1, 
    progpayload=0x0, progpayloadcount=0, rsp=0x0, is_request=0 '\0') at rpc-transport.c:769
	msg = (rpc_transport_pollin_t *) 0x639c90
	progpayloadlen = 0
	vectored = <value optimized out>
	hdr = <value optimized out>
	progpayloadbuf = <value optimized out>
	iobuf = <value optimized out>
	__FUNCTION__ = "rpc_transport_same_process_pollin_alloc"
#3  0x00002aaaaaf1b603 in rpc_transport_submit_reply (this=0x0, reply=<value optimized out>) at rpc-transport.c:1091
	ret = <value optimized out>
	pollin = <value optimized out>
	handover_msg = <value optimized out>
	__FUNCTION__ = "rpc_transport_submit_reply"
#4  0x00002aaaaaf17d1f in rpcsvc_conn_submit (conn=0x0, hdrvec=<value optimized out>, hdrcount=72, proghdr=0x41804930, proghdrcount=1, progpayload=0x1, 
    progpayloadcount=0, iobref=0x639c40, priv=0x2aaab0065010) at rpcsvc.c:1374
	ret = <value optimized out>
	reply = {msg = {rpchdr = 0x418048c0, rpchdrcount = 1, proghdr = 0x41804930, proghdrcount = 1, progpayload = 0x0, progpayloadcount = 0, iobref = 0x639c40}, 
  private = 0x0}
	__FUNCTION__ = "rpcsvc_conn_submit"
#5  0x00002aaaaaf18b22 in rpcsvc_submit_generic (req=0x2aaaad2b5940, proghdr=0x41804930, hdrcount=1, payload=0x0, payloadcount=0, iobref=0x639c40) at rpcsvc.c:1535
	ret = <value optimized out>
	i = <value optimized out>
	replyiob = (struct iobuf *) 0x629898
	recordhdr = {iov_base = 0x2aaaac29a000, iov_len = 28}
	msglen = 200
	new_iobref = 0 '\0'
	__FUNCTION__ = "rpcsvc_submit_generic"
#6  0x00002aaaacb6a132 in server_submit_reply (frame=0x2aaab005b378, req=0x2aaaad2b5940, arg=<value optimized out>, payload=0x0, payloadcount=0, iobref=0x639c40, 
    sfunc=0x2aaaab12f540 <xdr_serialize_writev_rsp>) at server.c:123
	iob = (struct iobuf *) 0x629488
	ret = <value optimized out>
Missing separate debuginfos, use: debuginfo-install gcc.x86_64 glibc.x86_64
---Type <return> to continue, or q <return> to quit---  
	rsp = {iov_base = 0x2aaaabf5a000, iov_len = 200}
	state = (server_state_t *) 0x2aaab00599f0
	new_iobref = 1 '\001'
 	__FUNCTION__ = "server_submit_reply"
#7  0x00002aaaacb78854 in server_writev_cbk (frame=0x2aaab005b378, cookie=<value optimized out>, this=0x0, op_ret=16843520, op_errno=0, prebuf=0x1, 
    postbuf=0x41804c30) at server3_1-fops.c:1235
	rsp = {gfs_id = 0, op_ret = 65536, op_errno = 0, prestat = {ia_ino = 39846415, ia_gen = 5501895600582451836, ia_dev = 2064, mode = 33188, ia_nlink = 1, 
    ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 254177280, ia_blksize = 4096, ia_blocks = 496944, ia_atime = 1281016557, ia_atime_nsec = 0, 
    ia_mtime = 1281016744, ia_mtime_nsec = 0, ia_ctime = 1281016744, ia_ctime_nsec = 0}, poststat = {ia_ino = 39846415, ia_gen = 5501895600582451836, ia_dev = 2064, 
    mode = 33188, ia_nlink = 1, ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 254242816, ia_blksize = 4096, ia_blocks = 497072, ia_atime = 1281016557, 
    ia_atime_nsec = 0, ia_mtime = 1281016744, ia_mtime_nsec = 0, ia_ctime = 1281016744, ia_ctime_nsec = 0}}
	req = (rpcsvc_request_t *) 0x2aaaad2b5940
	__FUNCTION__ = "server_writev_cbk"
#8  0x00002aaaac9554cd in iot_writev_cbk (frame=0x2aaab0059e40, cookie=<value optimized out>, this=<value optimized out>, op_ret=65536, op_errno=0, 
    prebuf=0x41804ca0, postbuf=0x41804c30) at io-threads.c:945
	_parent = (call_frame_t *) 0x2aaab005b378
	old_THIS = (xlator_t *) 0x631680
 
	__FUNCTION__ = "iot_writev_cbk"
#9  0x00002aaaac745d5d in pl_writev_cbk (frame=0x64c6a0, cookie=<value optimized out>, this=<value optimized out>, op_ret=65536, op_errno=0, prebuf=0x41804ca0, 
    postbuf=0x41804c30) at posix.c:497
	_parent = (call_frame_t *) 0x2aaab0059e40
	old_THIS = (xlator_t *) 0x630870

	__FUNCTION__ = "pl_writev_cbk"
#10 0x00002aaaaace245f in default_writev_cbk (frame=0x64b9f0, cookie=<value optimized out>, this=<value optimized out>, op_ret=65536, op_errno=0, prebuf=0x41804ca0, 
    postbuf=0x41804c30) at defaults.c:565
	fn = (ret_fn_t) 0x2aaaac745cc0 <pl_writev_cbk>
	_parent = (call_frame_t *) 0x64c6a0
	old_THIS = (xlator_t *) 0x62f540
	__FUNCTION__ = "default_writev_cbk"
#11 0x00002aaaac32515d in posix_writev (frame=0x639bc0, this=<value optimized out>, fd=0x2aaab0065f40, vector=0x638950, count=1, offset=254177280, 
    iobref=0x2aaab0065010) at posix.c:2550
	_parent = (call_frame_t *) 0x64b9f0
	old_THIS = (xlator_t *) 0x62e2c0
	op_ret = 65536
	op_errno = 0
	_fd = 111
	pfd = <value optimized out>
---Type <return> to continue, or q <return> to quit---   
	preop = {ia_ino = 39846415, ia_gen = 5501895600582451836, ia_dev = 2064, ia_type = IA_IFREG, ia_prot = {suid = 0 '\0', sgid = 0 '\0', sticky = 0 '\0', 
    owner = {read = 1 '\001', write = 1 '\001', exec = 0 '\0'}, group = {read = 1 '\001', write = 0 '\0', exec = 0 '\0'}, other = {read = 1 '\001', write = 0 '\0', 
      exec = 0 '\0'}}, ia_nlink = 1, ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 254177280, ia_blksize = 4096, ia_blocks = 496944, ia_atime = 1281016557, 
  ia_atime_nsec = 0, ia_mtime = 1281016744, ia_mtime_nsec = 0, ia_ctime = 1281016744, ia_ctime_nsec = 0}
	postop = {ia_ino = 39846415, ia_gen = 5501895600582451836, ia_dev = 2064, ia_type = IA_IFREG, ia_prot = {suid = 0 '\0', sgid = 0 '\0', sticky = 0 '\0', 
    owner = {read = 1 '\001', write = 1 '\001', exec = 0 '\0'}, group = {read = 1 '\001', write = 0 '\0', exec = 0 '\0'}, other = {read = 1 '\001', write = 0 '\0', 
      exec = 0 '\0'}}, ia_nlink = 1, ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 254242816, ia_blksize = 4096, ia_blocks = 497072, ia_atime = 1281016557, 
  ia_atime_nsec = 0, ia_mtime = 1281016744, ia_mtime_nsec = 0, ia_ctime = 1281016744, ia_ctime_nsec = 0}
	ret = <value optimized out>
	tmp_pfd = 46912652881232
	__FUNCTION__ = "posix_writev"
#12 0x00002aaaac537599 in ac_writev_resume (frame=<value optimized out>, this=0x62f540, fd=0x2aaab0065f40, vector=0x638950, count=1, offset=254177280, 
    iobref=0x2aaab0065010) at access-control.c:1468
	_new = (call_frame_t *) 0x0
	old_THIS = (xlator_t *) 0x62f540
	__FUNCTION__ = "ac_writev_resume"
#13 0x00002aaaaacea79d in call_resume (stub=0x639970) at call-stub.c:2235
	old_THIS = (xlator_t *) 0x62f540
	__FUNCTION__ = "call_resume"
#14 0x00002aaaac53929b in ac_writev_fstat_cbk (frame=0x64b9f0, cookie=<value optimized out>, this=<value optimized out>, op_ret=0, op_errno=0, buf=0x41804e60)
    at access-control.c:1493
	stub = (call_stub_t *) 0x639970
	__FUNCTION__ = "ac_writev_fstat_cbk"
#15 0x00002aaaac3243da in posix_fstat (frame=0x64bd50, this=<value optimized out>, fd=<value optimized out>) at posix.c:3853
	_parent = (call_frame_t *) 0x64b9f0
	old_THIS = (xlator_t *) 0x62e2c0
	op_ret = 0
	op_errno = 0
	buf = {ia_ino = 39846415, ia_gen = 5501895600582451836, ia_dev = 2064, ia_type = IA_IFREG, ia_prot = {suid = 0 '\0', sgid = 0 '\0', sticky = 0 '\0', 
    owner = {read = 1 '\001', write = 1 '\001', exec = 0 '\0'}, group = {read = 1 '\001', write = 0 '\0', exec = 0 '\0'}, other = {read = 1 '\001', write = 0 '\0', 
      exec = 0 '\0'}}, ia_nlink = 1, ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 254177280, ia_blksize = 4096, ia_blocks = 496944, ia_atime = 1281016557, 
  ia_atime_nsec = 0, ia_mtime = 1281016744, ia_mtime_nsec = 0, ia_ctime = 1281016744, ia_ctime_nsec = 0}
	tmp_pfd = 46912652881232
	ret = <value optimized out>
	__FUNCTION__ = "posix_fstat"
#16 0x00002aaaac53ade4 in ac_writev (frame=<value optimized out>, this=0x62f540, fd=0x2aaab0065f40, vector=<value optimized out>, count=<value optimized out>, 
    offset=<value optimized out>, iobref=0x2aaab0065010) at access-control.c:1522
---Type <return> to continue, or q <return> to quit---
	_new = (call_frame_t *) 0x0
	old_THIS = (xlator_t *) 0x62f540
	stub = <value optimized out>
	__FUNCTION__ = "ac_writev"
#17 0x00002aaaac747e24 in pl_writev (frame=0x64c6a0, this=0x630870, fd=0x2aaab0065f40, vector=0x2aaab005d1e0, count=1, offset=254177280, iobref=0x2aaab0065010)
    at posix.c:736
	_new = (call_frame_t *) 0x0
	old_THIS = (xlator_t *) 0x630870
	pl_inode = <value optimized out>
	rw = <value optimized out>
	region = {list = {next = 0x0, prev = 0x0}, fl_type = 0, fl_start = 0, fl_end = 0, blocked = 0, user_flock = {l_type = 0, l_whence = 0, l_start = 0, 
    l_len = 0, l_pid = 0}, this = 0x0, fd_num = 0, frame = 0x0, transport = 0x0, client_pid = 0, owner = 0}
	op_ret = <value optimized out>
	op_errno = <value optimized out>
	__FUNCTION__ = "pl_writev"
#18 0x00002aaaac958589 in iot_writev_wrapper (frame=<value optimized out>, this=0x631680, fd=0x2aaab0065f40, vector=0x2aaab005d1e0, count=1, offset=254177280, 
    iobref=0x2aaab0065010) at io-threads.c:955
	_new = (call_frame_t *) 0x0
	old_THIS = (xlator_t *) 0x631680
	__FUNCTION__ = "iot_writev_wrapper"
#19 0x00002aaaaacea79d in call_resume (stub=0x2aaab0094500) at call-stub.c:2235
	old_THIS = (xlator_t *) 0x631680
	__FUNCTION__ = "call_resume"
#20 0x00002aaaac95d9c6 in iot_worker (data=0x6350c0) at io-threads.c:118
	conf = <value optimized out>
	this = <value optimized out>
	stub = (call_stub_t *) 0x2aaab0094500
	sleep_till = {tv_sec = 1281016864, tv_nsec = 0}
	ret = <value optimized out>
	timeout = 0 '\0'
	bye = 0 '\0'
	__FUNCTION__ = "iot_worker"
#21 0x00002aaaab54c407 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#22 0x00002aaaab835b0d in clone () from /lib64/libc.so.6
No symbol table info available.
(gdb)