Lakshmi, what qa release is this?

(In reply to comment #2)
> Lakshmi, what qa release is this?
Its running with 4x3 distributed-replicate with 3.1.0qa5.

While running 4x3 distributed-replicate with qa5 ,one server gnfs crashed and then  all other gnfs servers  say rpcmount export: RPC: Timed out with showmount -e command

(gdb) 
(gdb) bt
#0  0x00002aaaab56ba9b in pthread_once () from /lib64/libpthread.so.0
#1  0x00002aaaab86224c in backtrace () from /lib64/libc.so.6
#2  0x00002aaaaaceb323 in gf_print_trace (signum=11) at common-utils.c:365
#3  <signal handler called>
#4  0x00002aaaab7f0edb in free () from /lib64/libc.so.6
#5  0x00002aaaaad02d86 in __gf_free (free_ptr=0x1) at mem-pool.c:240
#6  0x00002aaaacc7f661 in dht_local_wipe (this=0x63d050, local=0x2aaac216aba0) at dht-helper.c:211
#7  0x00002aaaacc944ad in dht_readv_cbk (frame=0x2aaaac33bca0, cookie=0x2aaaac337ea0, this=0x63d050, op_ret=4096, op_errno=0, vector=0x7fff794bcc70, count=1, 
    stbuf=0x7fff794bcc00, iobref=0x2aaac6006580) at dht-common.c:1971
#8  0x00002aaaaca31d41 in afr_readv_cbk (frame=0x2aaaac337ea0, cookie=0x0, this=0x63ba50, op_ret=4096, op_errno=0, vector=0x7fff794bcc70, count=1, 
    buf=0x7fff794bcc00, iobref=0x2aaac6006580) at afr-inode-read.c:809
#9  0x00002aaaac805a58 in client3_1_readv_cbk (req=0x2aaab0aa2420, iov=0x2aaab0aa2460, count=2, myframe=0x2aaaac33b5a0) at client3_1-fops.c:2027
#10 0x00002aaaaaf31c0c in rpc_clnt_handle_reply (clnt=0x643740, pollin=0x2aaac5f89f90) at rpc-clnt.c:692
#11 0x00002aaaaaf31f1d in rpc_clnt_notify (trans=0x643840, mydata=0x643770, event=RPC_TRANSPORT_MSG_RECEIVED, data=0x2aaac5f89f90) at rpc-clnt.c:801
#12 0x00002aaaaaf2ff39 in rpc_transport_notify (this=0x643840, event=RPC_TRANSPORT_MSG_RECEIVED, data=0x2aaac5f89f90) at rpc-transport.c:1123
#13 0x00002aaaaffb65e5 in socket_event_poll_in (this=0x643840) at socket.c:1411
#14 0x00002aaaaffb68e8 in socket_event_handler (fd=9, idx=2, data=0x643840, poll_in=1, poll_out=0, poll_err=0) at socket.c:1521
#15 0x00002aaaaad02165 in event_dispatch_epoll_handler (event_pool=0x629330, events=0x648730, i=5) at event.c:812
#16 0x00002aaaaad02338 in event_dispatch_epoll (event_pool=0x629330) at event.c:876
#17 0x00002aaaaad02608 in event_dispatch (event_pool=0x629330) at event.c:984
#18 0x00000000004050ba in main (argc=7, argv=0x7fff794bd2b8) at glusterfsd.c:1297
(gdb) bt full thread apply
No symbol "thread" in current context.
(gdb) thread apply all bt full

Thread 4 (Thread 1084229968 (LWP 14283)):
#0  0x00002aaaab56e268 in do_sigwait () from /lib64/libpthread.so.0
No symbol table info available.
#1  0x00002aaaab56e30d in sigwait () from /lib64/libpthread.so.0
No symbol table info available.
#2  0x0000000000404aa1 in glusterfs_sigwaiter (arg=0x7fff794bd0d0) at glusterfsd.c:1067
	set = {__val = {46912507365891, 46912496173445, 5, 0, 0, 0, 46912509592168, 1084229968, 0, 46912496197090, 1084229968, 46912507372438, 1084231112, 24, 
    1084228048, 1084229968}}
	ret = 0
	sig = 0
	__FUNCTION__ = "glusterfs_sigwaiter"
#3  0x00002aaaab566407 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#4  0x00002aaaab84fb0d in clone () from /lib64/libc.so.6
No symbol table info available.

Thread 3 (Thread 1085282640 (LWP 14284)):
#0  0x00002aaaab56ba9b in pthread_once () from /lib64/libpthread.so.0
No symbol table info available.
#1  0x00002aaaab86224c in backtrace () from /lib64/libc.so.6
No symbol table info available.
#2  0x00002aaaab7e60ef in __libc_message () from /lib64/libc.so.6
No symbol table info available.
#3  0x00002aaaab7edfec in _int_malloc () from /lib64/libc.so.6
No symbol table info available.
#4  0x00002aaaab7ef5a3 in calloc () from /lib64/libc.so.6
No symbol table info available.
#5  0x00002aaaaaab515c in _dl_new_object () from /lib64/ld-linux-x86-64.so.2
No symbol table info available.
#6  0x00002aaaaaab0c4c in _dl_map_object_from_fd () from /lib64/ld-linux-x86-64.so.2
No symbol table info available.
#7  0x00002aaaaaab2f03 in _dl_map_object () from /lib64/ld-linux-x86-64.so.2
No symbol table info available.
#8  0x00002aaaaaabc18d in dl_open_worker () from /lib64/ld-linux-x86-64.so.2
No symbol table info available.
#9  0x00002aaaaaab8306 in _dl_catch_error () from /lib64/ld-linux-x86-64.so.2
---Type <return> to continue, or q <return> to quit---
No symbol table info available.
#10 0x00002aaaaaabbb7c in _dl_open () from /lib64/ld-linux-x86-64.so.2
No symbol table info available.
#11 0x00002aaaab885560 in do_dlopen () from /lib64/libc.so.6
No symbol table info available.
#12 0x00002aaaaaab8306 in _dl_catch_error () from /lib64/ld-linux-x86-64.so.2
No symbol table info available.
#13 0x00002aaaab8856c7 in __libc_dlopen_mode () from /lib64/libc.so.6
No symbol table info available.
#14 0x00002aaaab8620ba in init () from /lib64/libc.so.6
No symbol table info available.
#15 0x00002aaaab56baa3 in pthread_once () from /lib64/libpthread.so.0
No symbol table info available.
#16 0x00002aaaab86224c in backtrace () from /lib64/libc.so.6
No symbol table info available.
#17 0x00002aaaaaceb323 in gf_print_trace (signum=11) at common-utils.c:365
	array = {0x0, 0x2aaaab7bf1cd, 0x40aff600, 0x2aaaaad131d0, 0x40aff480, 0x1400000000, 0x40aff430, 0x2aaab8000020, 0x64, 0x2aaac20e1890, 0x69, 0x0, 
  0x2aaac20be860, 0x2aaaab7f0f2c, 0x69, 0x0, 0x40aff480, 0x40aff5c0, 0x64, 0x2aaac20e1890, 0x69, 0x2aaaab7eab39, 0x12c, 0x1, 0x11, 0x40afff11, 0x40aff5c0, 0x12, 
  0x40aff480, 0xffffffee, 0x0, 0x40aff5b0, 0x2aaaacef4bf7, 0x2aaaacef4bf7, 0x12, 0x2aaaab7bf1cd, 0x40aff040, 0x2aaaab7e87a6, 0x2525252525252525, 0x1, 0x40aff600, 
  0x2aaaaad131ea, 0x2aaaaad131eb, 0xcc, 0x1000, 0x2aaaab7be27e, 0x10000002f, 0x2aaaaad131ce, 0x40aff0f0, 0xffffffffffffffff, 0x750000002f, 0x2aaaacef4bdd, 
  0x40aff610, 0x2aaaaad131e8, 0x62aa50, 0xb, 0x1000000000, 0x0, 0x0, 0xffffffffffffffff, 0xcd00000000, 0x2aaaaad131e8, 0x0, 0x40aff4d8, 0x40b02000, 0xb, 0x1, 0x0, 
  0x0, 0x0, 0x0, 0x0, 0x629120, 0x2aaab8000020, 0x40aff0f0, 0x73000000, 0x0, 0x40b02000, 0x1000, 0x8, 0xffffffff00001000, 0x2aaaacef4bf6, 0x40aff3a0, 0x8, 
  0x2525252525252525, 0x40aff780, 0x0, 0x40aff770, 0x2aaaacef38e3, 0x2, 0x1, 0x2aaaaad131e6, 0x2525252525252525, 0xfffffff7, 0x2, 0x40aff7b0, 0x2aaaaad131e5, 
  0x2aaaaad131e7, 0x9, 0x2aaaab7bf1cd, 0x0, 0x40aff7e0, 0x40aff2a0, 0x2aaaaad131cf, 0x0, 0x2aaaab7bf1cd, 0x40aff940, 0x2aaaaad131d0, 0x40aff7c0, 0x40aff648, 
  0x3eeeeeeee, 0xb, 0x0, 0xffffffffffffffff, 0x5b40aff7f0, 0x2aaaaad131d1, 0x0, 0x40aff688, 0x500000000, 0xb, 0xa00000000, 0x0, 0x0, 0x0, 0x0, 0x0, 0xa00000020, 0x0, 
  0x0, 0x73000000, 0x100000000, 0x0, 0x2525252525252525, 0x40aff900, 0x0, 0x40aff8f0, 0x2aaaacef554e, 0x2aaaacef554e, 0x0, 0x2aaaab7bf1cd, 0x40affa50, 
  0x2aaaab7e87a6, 0x2525252525252525, 0x1, 0x40aff940, 0x2aaaaad131ea, 0x2aaaaad131eb, 0xbe, 0x1000, 0x2aaaab7be27e, 0x100000000, 0x2aaaaad131ce, 0x40aff430, 
  0xffffffffffffffff, 0x6300000000, 0x2aaaacef5529, 0x40aff950, 0x2aaaaad131e8, 0x62aa50, 0xb, 0xa00000000, 0x0, 0x0, 0xffffffffffffffff, 0xbf00000000, 
  0x2aaaaad131e8, 0x0, 0x40aff818, 0x0, 0xb, 0x2aaa00000001, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2525252525252520, 0x1, 0x40affa50, 0x73d131ea, 0x2aaaaad131eb, 0x5b, 0x1000, 
  0x1, 0x100000080, 0x2aaaacef554d, 0x40aff540, 0xffffffff00000001, 0x1800000204, 0x2aaaacef3afb, 0x40affa60, 0x2aaaaad131e8, 0x62aa50, 0xb, 0xffffffff00000000, 
  0x2aaaaad131e9, 0x0, 0xffffffffffffffff, 0x5c00000000, 0x2aaaaad131e8}
	size = 46912719814688
	tm = (struct tm *) 0x2aaaabad0ce0
	msg = "signal received: 11\n", '\0' <repeats 1003 times>
	timestr = "2010-08-13 17:11:14\n", '\0' <repeats 235 times>
	utime = 1281733874
	ret = 20
---Type <return> to 
#19 0x00002aaaaacefb1f in __list_splice (list=0x2aaac216b100, head=0x40affc10) at list.h:105
No locals.
#20 0x00002aaaaacefae7 in list_splice_init (list=0x2aaac216b100, head=0x40affc10) at list.h:129
No locals.
#21 0x00002aaaaacefa30 in inode_table_prune (table=0x2aaac216b070) at inode.c:1146
	ret = 0
	purge = {next = 0x40affc10, prev = 0x40affc10}
	del = (inode_t *) 0x0
	tmp = (inode_t *) 0x0
	entry = (inode_t *) 0x0
#22 0x00002aaaaacee516 in inode_unref (inode=0x2aaac2165470) at inode.c:462
	table = (inode_table_t *) 0x2aaac216b070
#23 0x00002aaaacec294e in nfs_entry_loc_fill (itable=0x63f5a0, ino=656081220, gen=5504804526097918454, entry=0x2aaac2126f90 "fileop_dir_0_67_47", loc=0x2aaaaac504c8, 
    how=2) at nfs-common.c:379
	parent = (inode_t *) 0x2aaac1fa32f0
	entryinode = (inode_t *) 0x2aaac2165470
	ret = -2
	resolvedpath = 0x0
	pret = 0
#24 0x00002aaaacee5015 in nfs3_fh_resolve_entry_hard (cs=0x2aaaaac500e0) at nfs3-helpers.c:2942
	ret = -14
	nfu = {uid = 0, gids = {0 <repeats 17 times>}, ngrps = 1}
	__FUNCTION__ = "nfs3_fh_resolve_entry_hard"
#25 0x00002aaaacee5232 in nfs3_fh_resolve_entry (cs=0x2aaaaac500e0) at nfs3-helpers.c:2995
	ret = -14
#26 0x00002aaaacee537b in nfs3_fh_resolve_and_resume (cs=0x2aaaaac500e0, fh=0x40affed0, entry=0x40afff10 "fileop_dir_0_67_47", 
    resum_fn=0x2aaaaced5838 <nfs3_mkdir_resume>) at nfs3-helpers.c:3022
	ret = -14
#27 0x00002aaaaced5d68 in nfs3_mkdir (req=0x2aaab804bef0, dirfh=0x40affed0, name=0x40afff10 "fileop_dir_0_67_47", sattr=0x40affe88) at nfs3.c:2439
	vol = (xlator_t *) 0x63d050
	stat = NFS3ERR_SERVERFAULT
	ret = -14
	nfs3 = (struct nfs3_state *) 0x65a600
	cs = (nfs3_call_state_t *) 0x2aaaaac500e0
	__FUNCTION__ = "nfs3_mkdir"
#28 0x00002aaaaced5ece in nfs3svc_mkdir (req=0x2aaab804bef0) at nfs3.c:2473
	name = "fileop_dir_0_67_47", '\0' <repeats 14 times>, "\020\000\000\0000\000\000\000 \001?@\000\000\000\000`\000?@", '\0' <repeats 12 times>, "\004", '\0' <repeats 23 times>, " \000\000?\a\000\000\000????\000\000\000\000 \000\000??*\000\000\220\030\016ª*\000\000\220\001?@\000\000\000\000.\000\000\000\000\000\000\000?\002?@\000\000\000\000 Z???*\000\000,\017\177??*\000\000.\000\000\000\000\000\000\000?\002?@\000\000\000\000 Z???*\000\000??~??*\000\000/\000\000\000\000\000\000\000/\000\000\000\000\000\000\000\220\030\016ª*\000\000?N~"...
	dirfh = {ident = ":O", hashcount = 5, xlatorid = 0, gen = 5504804526097918454, ino = 656081220, entryhash = {257, 25445, 41377, 992, 36035, 
    0 <repeats 16 times>}}
	args = {where = {dir = {data = {data_len = 32, data_val = 0x40affed0 ":O\005"}}, name = 0x40afff10 "fileop_dir_0_67_47"}, attributes = {mode = {set_it = 1, 
      set_mode3_u = {mode = 493}}, uid = {set_it = 0, set_uid3_u = {uid = 0}}, gid = {set_it = 0, set_gid3_u = {gid = 0}}, size = {set_it = 0, set_size3_u = {
        size = 0}}, atime = {set_it = DONT_CHANGE, set_atime_u = {atime = {seconds = 0, nseconds = 0}}}, mtime = {set_it = DONT_CHANGE, set_mtime_u = {mtime = {
          seconds = 0, nseconds = 0}}}}}
	ret = -1
	__FUNCTION__ = "nfs3svc_mkdir"
#29 0x00002aaaaceebaec in nfs_rpcsvc_handle_rpc_call (conn=0x2aaab8017f10) at ../../../../xlators/nfs/lib/src//rpcsvc.c:1901
	actor = (rpcsvc_actor_t *) 0x2aaaad104140
	req = (rpcsvc_request_t *) 0x2aaab804bef0
	ret = -1
#30 0x00002aaaaceec7ee in nfs_rpcsvc_record_update_state (conn=0x2aaab8017f10, dataread=0) at ../../../../xlators/nfs/lib/src//rpcsvc.c:2383
	rs = (rpcsvc_record_state_t *) 0x2aaab8017f28
	svc = (rpcsvc_t *) 0x0
	__FUNCTION__ = "nfs_rpcsvc_record_update_state"
#31 0x00002aaaaceec944 in nfs_rpcsvc_conn_data_poll_in (conn=0x2aaab8017f10) at ../../../../xlators/nfs/lib/src//rpcsvc.c:2426
	dataread = 128
	readsize = 128
	readaddr = 0x2aaabcc40000 "lf?X"
	ret = -1
	__FUNCTION__ = "nfs_rpcsvc_conn_data_poll_in"
#32 0x00002aaaaceecd68 in nfs_rpcsvc_conn_data_handler (fd=21, idx=4, data=0x2aaab8017f10, poll_in=1, poll_out=0, poll_err=0)
    at ../../../../xlators/nfs/lib/src//rpcsvc.c:2555
	conn = (rpcsvc_conn_t *) 0x2aaab8017f10
	ret = 0
#33 0x00002aaaaad02165 in event_dispatch_epoll_handler (event_pool=0x63f2f0, events=0x6596f0, i=1) at event.c:812
	event_data = (struct event_data *) 0x659700
	handler = (event_handler_t) 0x2aaaaceecce1 <nfs_rpcsvc_conn_data_handler>
	data = (void *) 0x2aaab8017f10
	idx = 4
	ret = -1
	__FUNCTION__ = "event_dispatch_epoll_handler"
#34 0x00002aaaaad02338 in event_dispatch_epoll (event_pool=0x63f2f0) at event.c:876
	events = (struct epoll_event *) 0x6596f0
	size = 2
	i = 1
	ret = 0
	__FUNCTION__ = "event_dispatch_epoll"
#35 0x00002aaaaad02608 in event_dispatch (event_pool=0x63f2f0) at event.c:984
	ret = -1
	__FUNCTION__ = "event_dispatch"
#36 0x00002aaaacee8236 in nfs_rpcsvc_stage_proc (arg=0x62b300) at ../../../../xlators/nfs/lib/src//rpcsvc.c:64
	stg = (rpcsvc_stage_t *) 0x62b300
#37 0x00002aaaab566407 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#38 0x00002aaaab84fb0d in clone () from /lib64/libc.so.6
No symbol table info available.

Thread 2 (Thread 1095772496 (LWP 14285)):
#0  0x00002aaaab85be4e in __lll_lock_wait_private () from /lib64/libc.so.6
No symbol table info available.
#1  0x00002aaaab7f20f7 in _L_lock_14679 () from /lib64/libc.so.6
No symbol table info available.
#2  0x00002aaaab7f0f21 in free () from /lib64/libc.so.6
No symbol table info available.
#3  0x00002aaaaad02d86 in __gf_free (free_ptr=0x2aaac215cdd0) at mem-pool.c:240
	req_size = 0
	ptr = 0x0
	type = 0
	xl = (xlator_t *) 0x0
	__PRETTY_FUNCTION__ = "__gf_free"
#4  0x00002aaaaace03ef in loc_wipe (loc=0x2aaac2127ae8) at xlator.c:1004
No locals.
#5  0x00002aaaacc7f4ef in dht_local_wipe (this=0x63d050, local=0x2aaac2127ae0) at dht-helper.c:175
No locals.
#6  0x00002aaaacc8a5bc in dht_lookup_everywhere_cbk (frame=0x2aaaac344020, cookie=0x2aaaac3350a0, this=0x63d050, op_ret=-1, op_errno=2, inode=0x0, 
    buf=0x2aaac6004b90, xattr=0x0, postparent=0x2aaac6004c60) at dht-common.c:483
#6  0x00002aaaacc8a5bc in dht_lookup_everywhere_cbk (frame=0x2aaaac344020, cookie=0x2aaaac3350a0, this=0x63d050, op_ret=-1, op_errno=2, inode=0x0, 
    buf=0x2aaac6004b90, xattr=0x0, postparent=0x2aaac6004c60) at dht-common.c:483
---Type <return> to continue, or q <return> to quit---
	__local = (dht_local_t *) 0x2aaac2127ae0
	__xl = (xlator_t *) 0x63d050
	conf = (dht_conf_t *) 0x63f920
	local = (dht_local_t *) 0x2aaac2127ae0
	this_call_cnt = 0
	prev = (call_frame_t *) 0x2aaaac3350a0
	is_linkfile = 0
	is_dir = 0
	subvol = (xlator_t *) 0x63ba50
	loc = (loc_t *) 0x2aaac2127ae8
	link_subvol = (xlator_t *) 0x0
	hashed_subvol = (xlator_t *) 0x63ba50
	cached_subvol = (xlator_t *) 0x0
	ret = -1
	__FUNCTION__ = "dht_lookup_everywhere_cbk"
#7  0x00002aaaaca5f4db in afr_lookup_done (frame=0x2aaaac3350a0, this=0x63ba50, lookup_buf=0x2aaac6004b90) at afr-common.c:706
	fn = (fop_lookup_cbk_t) 0x2aaaacc89c9e <dht_lookup_everywhere_cbk>
	_parent = (call_frame_t *) 0x2aaaac344020
	old_THIS = (xlator_t *) 0x63ba50
	__local = (afr_local_t *) 0x2aaac6004a40
	__this = (xlator_t *) 0x63ba50
	unwind = 1
	source = -1
	sh_type_str = '\0' <repeats 255 times>
	local = (afr_local_t *) 0x2aaac6004a40
	__FUNCTION__ = "afr_lookup_done"
#8  0x00002aaaaca5fb28 in afr_fresh_lookup_cbk (frame=0x2aaaac3350a0, cookie=0x0, this=0x63ba50, op_ret=-1, op_errno=0, inode=0x2aaac2159c10, buf=0x41501e00, 
    xattr=0x0, postparent=0x41501d90) at afr-common.c:868
	local = (afr_local_t *) 0x2aaac6004a40
	priv = (afr_private_t *) 0x63fcb0
	lookup_buf = (struct iatt *) 0x2aaac6004b90
	call_count = 0
	child_index = 0
	first_up_child = -1
	__FUNCTION__ = "afr_fresh_lookup_cbk"
#9  0x00002aaaac805720 in client3_1_lookup_cbk (req=0x2aaab0aa3ce0, iov=0x41502000, count=1, myframe=0x2aaaac339ea0) at client3_1-fops.c:1955
	fn = (fop_lookup_cbk_t) 0x2aaaaca5f547 <afr_fresh_lookup_cbk>
	_parent = (call_frame_t *) 0x2aaaac3350a0
	old_THIS = (xlator_t *) 0x637510
	local = (clnt_local_t *) 0x2aaac60155c0
	frame = (call_frame_t *) 0x2aaaac339ea0
	ret = 0
	rsp = {gfs_id = 0, op_ret = -1, op_errno = 0, stat = {ia_ino = 0, ia_gen = 0, ia_dev = 0, mode = 0, ia_nlink = 0, ia_uid = 0, ia_gid = 0, ia_rdev = 0, 
    ia_size = 0, ia_blksize = 0, ia_blocks = 0, ia_atime = 0, ia_atime_nsec = 0, ia_mtime = 0, ia_mtime_nsec = 0, ia_ctime = 0, ia_ctime_nsec = 0}, postparent = {
    ia_ino = 0, ia_gen = 0, ia_dev = 0, mode = 0, ia_nlink = 0, ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 0, ia_blksize = 0, ia_blocks = 0, ia_atime = 0, 
    ia_atime_nsec = 0, ia_mtime = 0, ia_mtime_nsec = 0, ia_ctime = 0, ia_ctime_nsec = 0}, dict = {dict_len = 0, dict_val = 0x0}}
	stbuf = {ia_ino = 0, ia_gen = 0, ia_dev = 0, ia_type = IA_INVAL, ia_prot = {suid = 0 '\0', sgid = 0 '\0', sticky = 0 '\0', owner = {read = 0 '\0', 
      write = 0 '\0', exec = 0 '\0'}, group = {read = 0 '\0', write = 0 '\0', exec = 0 '\0'}, other = {read = 0 '\0', write = 0 '\0', exec = 0 '\0'}}, ia_nlink = 0, 
  ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 0, ia_blksize = 0, ia_blocks = 0, ia_atime = 0, ia_atime_nsec = 0, ia_mtime = 0, ia_mtime_nsec = 0, ia_ctime = 0, 
  ia_ctime_nsec = 0}
	postparent = {ia_ino = 0, ia_gen = 0, ia_dev = 0, ia_type = IA_INVAL, ia_prot = {suid = 0 '\0', sgid = 0 '\0', sticky = 0 '\0', owner = {read = 0 '\0', 
      write = 0 '\0', exec = 0 '\0'}, group = {read = 0 '\0', write = 0 '\0', exec = 0 '\0'}, other = {read = 0 '\0', write = 0 '\0', exec = 0 '\0'}}, ia_nlink = 0, 
  ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 0, ia_blksize = 0, ia_blocks = 0, ia_atime = 0, ia_atime_nsec = 0, ia_mtime = 0, ia_mtime_nsec = 0, ia_ctime = 0, 
  ia_ctime_nsec = 0}
	op_errno = 0
	oldino = 0
	oldgen = 0
	xattr = (dict_t *) 0x0
	inode = (inode_t *) 0x2aaac2159c10
	buf = 0x0
	__FUNCTION__ = "client3_1_lookup_cbk"
#10 0x00002aaaaaf30c5f in call_bail (data=0x643740) at rpc-clnt.c:197
	clnt = (struct rpc_clnt *) 0x643740
	conn = (rpc_clnt_connection_t *) 0x643770
	current = {tv_sec = 1281735677, tv_usec = 608979}
	list = {next = 0x2aaab0ce16b0, prev = 0x2aaab0ce2130}
	saved_frame = (struct saved_frame *) 0x0
	trav = (struct saved_frame *) 0x2aaab0ce16b0
	tmp = (struct saved_frame *) 0x2aaab0ce1410
	frame_sent_tm = {tm_sec = 14, tm_min = 11, tm_hour = 17, tm_mday = 13, tm_mon = 7, tm_year = 110, tm_wday = 5, tm_yday = 224, tm_isdst = 1, 
  tm_gmtoff = -14400, tm_zone = 0x6291f0 "EDT"}
	frame_sent = "2010-08-13 17:11:14", '\0' <repeats 12 times>
	timeout = {tv_sec = 10, tv_usec = 0}
	iov = {iov_base = 0x0, iov_len = 0}
	__FUNCTION__ = "call_bail"
#11 0x00002aaaaaced5ee in gf_timer_proc (ctx=0x629010) at timer.c:176
	at = 1281735677598624
	need_cbk = 1 '\001'
	now = 1281735677607066
	now_tv = {tv_sec = 1281735677, tv_usec = 607066}
	event = (gf_timer_t *) 0x660c9c0
	reg = (gf_timer_registry_t *) 0x642b00
	__FUNCTION__ = "gf_timer_proc"
#12 0x00002aaaab566407 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#13 0x00002aaaab84fb0d in clone () from /lib64/libc.so.6
No symbol table info available.

Thread 1 (Thread 46912513061360 (LWP 14282)):
#0  0x00002aaaab56ba9b in pthread_once () from /lib64/libpthread.so.0
No symbol table info available.
#1  0x00002aaaab86224c in backtrace () from /lib64/libc.so.6
No symbol table info available.
#2  0x00002aaaaaceb323 in gf_print_trace (signum=11) at common-utils.c:365
	array = {0x2c, 0x2, 0x1, 0x2aaaaad131e6, 0x2525252525252525, 0xfffffffa, 0x2, 0x7fff794bc070, 0x2aaaaad131e5, 0x2aaaaad131e7, 0x6, 0x2aaaab7bf1cd, 0x0, 
  0x7fff794bc0a0, 0x7fff794bbb60, 0x2aaaaad131cf, 0x0, 0x2aaaab7bf1cd, 0x7fff794bc200, 0x2aaaaad131d0, 0x7fff794bc080, 0x7fff794bc0e0, 0x2aaaacef4efe, 
  0x2aaaacef4efe, 0x0, 0xffffffffffffffff, 0x56794bc0b0, 0x2aaaaad131d1, 0x0, 0x7fff794bbf48, 0x500000000, 0x0, 0xa00000000, 0x0, 0x0, 0x0, 0x0, 0x0, 0xa00000020, 
  0x0, 0x0, 0x73000000, 0x100000000, 0x0, 0x2525252525252525, 0x7fff794bc1c0, 0x0, 0x7fff794bc1b0, 0x2aaaacef656d, 0x2aaaacef656d, 0x0, 0x2aaaab7bf1cd, 0x0, 
  0x2aaaab7e87a6, 0x2525252525252525, 0x1, 0x7fff794bc200, 0x2aaaaad131ea, 0x2aaaaad131eb, 0x84, 0x0, 0x2aaaab7be27e, 0x1aad131e5, 0x2aaaaad131ce, 0x7fff794bbcf0, 
  0xffffffffffffffff, 0x2e00000000, 0x2aaaacef654e, 0x7fff794bc210, 0x2aaaaad131e8, 0x62aa50, 0x0, 0xa00000000, 0x0, 0x0, 0xffffffffffffffff, 0x8500000000, 
  0x2aaaaad131e8, 0x0, 0x7fff794bc0d8, 0x794bc260, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x20, 0x0, 0x0, 0x73000000, 0xa00000020, 0x0, 0x0, 0x2, 0x100000000, 
  0x2aaaacef656c, 0x25252525, 0x7fff00000002, 0x0, 0x7fff794bc360, 0x2aaaacef674f, 0x2aaaacef674f, 0x0, 0x2aaaab7bf1cd, 0xffffffff25252525, 0x2aaaaad131e9, 
  0x25252525, 0x2, 0x1, 0x2aaaaad131e6, 0x2525252525252525, 0xfffffff7, 0x2, 0x7fff794bc3d0, 0x2aaaaad131e5, 0x2aaaaad131e7, 0x9, 0x2aaaab7bf1cd, 0x0, 
  0x7fff794bc400, 0x7fff794bbec0, 0x2aaaaad131cf, 0x0, 0x2aaaab7bf1cd, 0x7fff794bc560, 0x2aaaaad131d0, 0x7fff794bc3e0, 0xffffffffffffffff, 0x6800000000, 
  0x2aaaaad131e8, 0x0, 0xffffffffffffffff, 0x4c794bc410, 0x2aaaaad131d1, 0x0, 0x7fff794bc2a8, 0x500000000, 0x0, 0xa00000000, 0x0, 0x0, 0x0, 0x0, 0x0, 0xa00000020, 
  0x0, 0x0, 0x73000000, 0x100000000, 0x0, 0x2525252525252525, 0x7fff794bc520, 0x0, 0x7fff794bc510, 0x2aaaacef395f, 0x2aaaacef395f, 0x0, 0x2aaaab7bf1cd, 
  0xffffffff794bc130, 0x2aaaab7e87a6, 0x2525252525252525, 0x1, 0x7fff794bc560, 0x2aaaaad131ea, 0x2aaaaad131eb, 0x55, 0x0, 0x2aaaab7be27e, 0x100000018, 
  0x2aaaaad131ce, 0x7fff794bc050, 0xffffffffffffffff, 0x900644ec0, 0x2aaaacef395d, 0x7fff794bc570, 0x2aaaaad131e8, 0x62aa50, 0x0, 0xa00000000, 0x0, 0x0, 
  0xffffffffffffffff, 0x5600000000, 0x2aaaaad131e8, 0x0, 0x7fff794bc438, 0xc606b590, 0x0, 0x3000000001, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3000000020, 0x7fff794bc2f0, 
  0x7fff794bc230, 0x7306b590}
	size = 46912510032122
	tm = (struct tm *) 0x2aaaabad0ce0
	msg = "signal received: 11\n", '\0' <repeats 1003 times>
	timestr = "2010-08-13 17:11:14\n", '\0' <repeats 235 times>
	utime = 1281733874
	ret = 20
	fd = 4
#3  <signal handler called>
No symbol table info available.
#4  0x00002aaaab7f0edb in free () from /lib64/libc.so.6
No symbol table info available.
#5  0x00002aaaaad02d86 in __gf_free (free_ptr=0x1) at mem-pool.c:240
	req_size = 0
	ptr = 0x0
	type = 0
	xl = (xlator_t *) 0x0
	__PRETTY_FUNCTION__ = "__gf_free"
#6  0x00002aaaacc7f661 in dht_local_wipe (this=0x63d050, local=0x2aaac216aba0) at dht-helper.c:211
No locals.
#7  0x00002aaaacc944ad in dht_readv_cbk (frame=0x2aaaac33bca0, cookie=0x2aaaac337ea0, this=0x63d050, op_ret=4096, op_errno=0, vector=0x7fff794bcc70, count=1, 
    stbuf=0x7fff794bcc00, iobref=0x2aaac6006580) at dht-common.c:1971
	__local = (dht_local_t *) 0x2aaac216aba0
	__xl = (xlator_t *) 0x63d050
	local = (dht_local_t *) 0x2aaac216aba0
	__FUNCTION__ = "dht_readv_cbk"
#8  0x00002aaaaca31d41 in afr_readv_cbk (frame=0x2aaaac337ea0, cookie=0x0, this=0x63ba50, op_ret=4096, op_errno=0, vector=0x7fff794bcc70, count=1, 
    buf=0x7fff794bcc00, iobref=0x2aaac6006580) at afr-inode-read.c:809
	fn = (fop_readv_cbk_t) 0x2aaaacc942d4 <dht_readv_cbk>
	_parent = (call_frame_t *) 0x2aaaac33bca0
	old_THIS = (xlator_t *) 0x63ba50
	__local = (afr_local_t *) 0x2aaac21240f0
	__this = (xlator_t *) 0x63ba50
	priv = (afr_private_t *) 0x63fcb0
	local = (afr_local_t *) 0x2aaac21240f0
	children = (xlator_t **) 0x63fd80
	unwind = 1
	last_tried = -1
	this_try = -1
---Type <return> to cont
	read_child = 0
	__FUNCTION__ = "afr_readv_cbk"
#9  0x00002aaaac805a58 in client3_1_readv_cbk (req=0x2aaab0aa2420, iov=0x2aaab0aa2460, count=2, myframe=0x2aaaac33b5a0) at client3_1-fops.c:2027
	fn = (fop_readv_cbk_t) 0x2aaaaca31666 <afr_readv_cbk>
	_parent = (call_frame_t *) 0x2aaaac337ea0
	old_THIS = (xlator_t *) 0x637510
	frame = (call_frame_t *) 0x2aaaac33b5a0
	iobref = (struct iobref *) 0x2aaac6006580
	vector = {{iov_base = 0x2aaabcd60000, iov_len = 4096}, {iov_base = 0x0, iov_len = 0} <repeats 15 times>}
	stat = {ia_ino = 23792738, ia_gen = 5504804526097433583, ia_dev = 0, ia_type = IA_IFREG, ia_prot = {suid = 0 '\0', sgid = 0 '\0', sticky = 0 '\0', owner = {
      read = 1 '\001', write = 1 '\001', exec = 0 '\0'}, group = {read = 1 '\001', write = 0 '\0', exec = 0 '\0'}, other = {read = 1 '\001', write = 0 '\0', 
      exec = 0 '\0'}}, ia_nlink = 1, ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 104857600, ia_blksize = 4096, ia_blocks = 205016, ia_atime = 1281733874, 
  ia_atime_nsec = 0, ia_mtime = 1281693041, ia_mtime_nsec = 0, ia_ctime = 1281693041, ia_ctime_nsec = 0}
	rsp = {gfs_id = 0, op_ret = 4096, op_errno = 0, stat = {ia_ino = 1982728, ia_gen = 5504804526097433583, ia_dev = 2064, mode = 33188, ia_nlink = 1, 
    ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 104857600, ia_blksize = 4096, ia_blocks = 205016, ia_atime = 1281733874, ia_atime_nsec = 0, ia_mtime = 1281693041, 
    ia_mtime_nsec = 0, ia_ctime = 1281693041, ia_ctime_nsec = 0}, size = 4096}
	ret = 112
	rspcount = 1
	i = 0
	__FUNCTION__ = "client3_1_readv_cbk"
#10 0x00002aaaaaf31c0c in rpc_clnt_handle_reply (clnt=0x643740, pollin=0x2aaac5f89f90) at rpc-clnt.c:692
	conn = (rpc_clnt_connection_t *) 0x643770
	saved_frame = (struct saved_frame *) 0x2aaab0ce1b10
	request_info = (rpc_request_info_t *) 0x2aaac6027b40
	ret = 0
	req = (struct rpc_req *) 0x2aaab0aa2420
	__FUNCTION__ = "rpc_clnt_handle_reply"
#11 0x00002aaaaaf31f1d in rpc_clnt_notify (trans=0x643840, mydata=0x643770, event=RPC_TRANSPORT_MSG_RECEIVED, data=0x2aaac5f89f90) at rpc-clnt.c:801
	conn = (rpc_clnt_connection_t *) 0x643770
	clnt = (struct rpc_clnt *) 0x643740
	ret = -1
	req_info = (rpc_request_info_t *) 0x0
	pollin = (rpc_transport_pollin_t *) 0x2aaac5f89f90
	tv = {tv_sec = 0, tv_usec = 0}
#12 0x00002aaaaaf2ff39 in rpc_transport_notify (this=0x643840, event=RPC_TRANSPORT_MSG_RECEIVED, data=0x2aaac5f89f90) at rpc-transport.c:1123
	ret = -1
#13 0x00002aaaaffb65e5 in socket_event_poll_in (this=0x643840) at socket.c:1411
---Type <return> to continue, or q <return> to quit---
	conn = (rpc_clnt_connection_t *) 0x643770
	clnt = (struct rpc_clnt *) 0x643740
	ret = -1
	req_info = (rpc_request_info_t *) 0x0
	pollin = (rpc_transport_pollin_t *) 0x2aaac5f89f90
	tv = {tv_sec = 0, tv_usec = 0}
#12 0x00002aaaaaf2ff39 in rpc_transport_notify (this=0x643840, event=RPC_TRANSPORT_MSG_RECEIVED, data=0x2aaac5f89f90) at rpc-transport.c:1123
	ret = -1
#13 0x00002aaaaffb65e5 in socket_event_poll_in (this=0x643840) at socket.c:1411
---Type <return> to continue, or q <return> to quit---
	ret = 0
	pollin = (rpc_transport_pollin_t *) 0x2aaac5f89f90
#14 0x00002aaaaffb68e8 in socket_event_handler (fd=9, idx=2, data=0x643840, poll_in=1, poll_out=0, poll_err=0) at socket.c:1521
	this = (rpc_transport_t *) 0x643840
	priv = (socket_private_t *) 0x643c20
	ret = 0
	__FUNCTION__ = "socket_event_handler"
#15 0x00002aaaaad02165 in event_dispatch_epoll_handler (event_pool=0x629330, events=0x648730, i=5) at event.c:812
	event_data = (struct event_data *) 0x648770
	handler = (event_handler_t) 0x2aaaaffb6806 <socket_event_handler>
	data = (void *) 0x643840
	idx = 2
	ret = -1
	__FUNCTION__ = "event_dispatch_epoll_handler"
#16 0x00002aaaaad02338 in event_dispatch_epoll (event_pool=0x629330) at event.c:876
	events = (struct epoll_event *) 0x648730
	size = 7
	i = 5
	ret = 0
	__FUNCTION__ = "event_dispatch_epoll"
#17 0x00002aaaaad02608 in event_dispatch (event_pool=0x629330) at event.c:984
	ret = -1
	__FUNCTION__ = "event_dispatch"
#18 0x00000000004050ba in main (argc=7, argv=0x7fff794bd2b8) at glusterfsd.c:1297
	ctx = (glusterfs_ctx_t *) 0x629010
	ret = 0
#0  0x00002aaaab56ba9b in pthread_once () from /lib64/libpthread.so.0
(gdb)