This service will be undergoing maintenance at 00:00 UTC, 2016-09-28. It is expected to last about 1 hours

Bug 763635 (GLUSTER-1903)

Summary: add-brick with fileop running on nfs mntpt crashed nfs-server
Product: [Community] GlusterFS Reporter: Lakshmipathi G <lakshmipathi>
Component: nfsAssignee: Shehjar Tikoo <shehjart>
Status: CLOSED CURRENTRELEASE QA Contact:
Severity: high Docs Contact:
Priority: low    
Version: 3.1-alphaCC: amarts, gluster-bugs
Target Milestone: ---   
Target Release: ---   
Hardware: All   
OS: Linux   
Whiteboard:
Fixed In Version: Doc Type: Bug Fix
Doc Text:
Story Points: ---
Clone Of: Environment:
Last Closed: Type: ---
Regression: RTP Mount Type: nfs
Documentation: --- CRM:
Verified Versions: Category: ---
oVirt Team: --- RHEL 7.3 requirements from Atomic Host:

Description Lakshmipathi G 2010-10-11 06:25:06 EDT
Started fileop on nfsmount pt and added 2 bricks to existing  2 birck afr cluster. after some time fileop hanged. following core found -
-----------------------------------------
# bt full
#0  0x00002aaaad96b01a in nfs3svc_access_cbk (frame=0x2aaaac558400, cookie=0x63d7b8, this=0x63e918, op_ret=0, op_errno=117, buf=0x11c9090) at nfs3.c:1343
	status = NFS3_OK
	cs = (nfs3_call_state_t *) 0x2aaaaac4d03c
#1  0x00002aaaad95a654 in nfs_fop_stat_cbk (frame=0x2aaaac558400, cookie=0x63d7b8, this=0x63e918, op_ret=0, op_errno=117, buf=0x11c9090) at nfs-fops.c:356
	nfl = (struct nfs_fop_local *) 0x2aaaaab26514
	progcbk = (fop_stat_cbk_t) 0x2aaaad96afc8 <nfs3svc_access_cbk>
#2  0x00002aaaad7360bf in io_stats_stat_cbk (frame=0x2aaaac349a00, cookie=0x2aaaac34a138, this=0x63d7b8, op_ret=0, op_errno=117, buf=0x11c9090) at io-stats.c:487
	fn = (fop_stat_cbk_t) 0x2aaaad95a5a2 <nfs_fop_stat_cbk>
	_parent = (call_frame_t *) 0x2aaaac558400
	old_THIS = (xlator_t *) 0x63d7b8
	__FUNCTION__ = "io_stats_stat_cbk"
#3  0x00002aaaaace5fc6 in default_stat_cbk (frame=0x2aaaac34a138, cookie=0x2aaaac3472d0, this=0x63c628, op_ret=0, op_errno=117, buf=0x11c9090) at defaults.c:55
	fn = (fop_stat_cbk_t) 0x2aaaad735f57 <io_stats_stat_cbk>
	_parent = (call_frame_t *) 0x2aaaac349a00
	old_THIS = (xlator_t *) 0x63c628
	__FUNCTION__ = "default_stat_cbk"
#4  0x00002aaaaace5fc6 in default_stat_cbk (frame=0x2aaaac3472d0, cookie=0x2aaaac347c18, this=0x63b498, op_ret=0, op_errno=117, buf=0x11c9090) at defaults.c:55
	fn = (fop_stat_cbk_t) 0x2aaaaace5ea7 <default_stat_cbk>
	_parent = (call_frame_t *) 0x2aaaac34a138
	old_THIS = (xlator_t *) 0x63b498
	__FUNCTION__ = "default_stat_cbk"
#5  0x00002aaaaace5fc6 in default_stat_cbk (frame=0x2aaaac347c18, cookie=0x2aaaac349b08, this=0x63a2d8, op_ret=0, op_errno=117, buf=0x11c9090) at defaults.c:55
	fn = (fop_stat_cbk_t) 0x2aaaaace5ea7 <default_stat_cbk>
	_parent = (call_frame_t *) 0x2aaaac3472d0
	old_THIS = (xlator_t *) 0x63a2d8
	__FUNCTION__ = "default_stat_cbk"
#6  0x00002aaaacef6bf9 in wb_stat_cbk (frame=0x2aaaac349b08, cookie=0x2aaaac34a660, this=0x639148, op_ret=0, op_errno=117, buf=0x11c9090) at write-behind.c:629
	fn = (fop_stat_cbk_t) 0x2aaaaace5ea7 <default_stat_cbk>
	_parent = (call_frame_t *) 0x2aaaac347c18
	old_THIS = (xlator_t *) 0x639148
	local = (wb_local_t *) 0x11c7858
	request = (wb_request_t *) 0x0
	process_frame = (call_frame_t *) 0x0
	file = (wb_file_t *) 0x0
	ret = -1
	fd = (fd_t *) 0x0
	__FUNCTION__ = "wb_stat_cbk"
Missing separate debuginfos, use: debuginfo-install gcc.x86_64 glibc.x86_64
---Type <return> to continue, or q <return> to quit---
#7  0x00002aaaaccd1070 in dht_attr_cbk (frame=0x2aaaac34a660, cookie=0x2aaaac34724c, this=0x637f18, op_ret=0, op_errno=0, stbuf=0x7fffaadebfe0) at dht-common.c:1179
	fn = (fop_stat_cbk_t) 0x2aaaacef6a5e <wb_stat_cbk>
	_parent = (call_frame_t *) 0x2aaaac349b08
	old_THIS = (xlator_t *) 0x637f18
	__local = (dht_local_t *) 0x11c9028
	__xl = (xlator_t *) 0x637f18
	local = (dht_local_t *) 0x11c9028
	this_call_cnt = 0
	prev = (call_frame_t *) 0x2aaaac34724c
	__FUNCTION__ = "dht_attr_cbk"
#8  0x00002aaaaca69ff6 in afr_stat_cbk (frame=0x2aaaac34724c, cookie=0x0, this=0x637358, op_ret=0, op_errno=0, buf=0x7fffaadebfe0) at afr-inode-read.c:227
	fn = (fop_stat_cbk_t) 0x2aaaaccd0ddb <dht_attr_cbk>
	_parent = (call_frame_t *) 0x2aaaac34a660
	old_THIS = (xlator_t *) 0x637358
	__local = (afr_local_t *) 0x11d0888
	__this = (xlator_t *) 0x637358
	priv = (afr_private_t *) 0x6473e8
	local = (afr_local_t *) 0x11d0888
	children = (xlator_t **) 0x647528
	unwind = 1
	last_tried = -1
	this_try = -1
	read_child = 0
	__FUNCTION__ = "afr_stat_cbk"
#9  0x00002aaaac835f76 in client3_1_stat_cbk (req=0x2aaab1277db0, iov=0x2aaab1277df0, count=1, myframe=0x2aaaac349f28) at client3_1-fops.c:367
	fn = (fop_stat_cbk_t) 0x2aaaaca69b8f <afr_stat_cbk>
	_parent = (call_frame_t *) 0x2aaaac34724c
	old_THIS = (xlator_t *) 0x634578
	rsp = {op_ret = 0, op_errno = 0, stat = {ia_gfid = "\2075��\036\035@�\220^\227I���\201", ia_ino = 1000643, ia_dev = 2049, mode = 16877, ia_nlink = 2, 
    ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 4096, ia_blksize = 4096, ia_blocks = 16, ia_atime = 1286780939, ia_atime_nsec = 0, ia_mtime = 1286780404, 
    ia_mtime_nsec = 0, ia_ctime = 1286780939, ia_ctime_nsec = 0}}
	frame = (call_frame_t *) 0x2aaaac349f28
	iatt = {ia_ino = 176163504, ia_gfid = {135 '\207', 53 '5', 174 '�', 238 '�', 30 '\036', 29 '\035', 64 '@', 234 '�', 144 '\220', 94 '^', 151 '\227', 73 'I', 
    219 '�', 227 '�', 218 '�', 129 '\201'}, ia_dev = 2049, ia_type = IA_IFDIR, ia_prot = {suid = 0 '\0', sgid = 0 '\0', sticky = 0 '\0', owner = {read = 1 '\001', 
      write = 1 '\001', exec = 1 '\001'}, group = {read = 1 '\001', write = 0 '\0', exec = 1 '\001'}, other = {read = 1 '\001', write = 0 '\0', exec = 1 '\001'}}, 
  ia_nlink = 2, ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 4096, ia_blksize = 4096, ia_blocks = 16, ia_atime = 1286780939, ia_atime_nsec = 0, 
  ia_mtime = 1286780404, ia_mtime_nsec = 0, ia_ctime = 1286780939, ia_ctime_nsec = 0}
---Type <return> to continue, or q <return> to quit---
	ret = 108
	__FUNCTION__ = "client3_1_stat_cbk"
#10 0x00002aaaaaf42385 in rpc_clnt_handle_reply (clnt=0x64b978, pollin=0x11c2938) at rpc-clnt.c:752
	conn = (rpc_clnt_connection_t *) 0x64b9a8
	saved_frame = (struct saved_frame *) 0x2aaab14bb620
	ret = 0
	req = (struct rpc_req *) 0x2aaab1277db0
	xid = 177086
	__FUNCTION__ = "rpc_clnt_handle_reply"
#11 0x00002aaaaaf426bc in rpc_clnt_notify (trans=0x64bb98, mydata=0x64b9a8, event=RPC_TRANSPORT_MSG_RECEIVED, data=0x11c2938) at rpc-clnt.c:865
	conn = (rpc_clnt_connection_t *) 0x64b9a8
	clnt = (struct rpc_clnt *) 0x64b978
	ret = -1
	req_info = (rpc_request_info_t *) 0x0


===============
tail -100 /etc/glusterd/logs/nfs.log 
[2010-10-11 03:00:16.924962] I [afr-self-heal-common.c:1526:afr_self_heal_completion_cbk] afr44-replicate-1: background  meta-data data entry self-heal completed on /fileop/fileop_L1_9/fileop_L1_9_L2_12/fileop_dir_9_12_16
[2010-10-11 03:00:17.218161] I [afr-common.c:662:afr_lookup_done] afr44-replicate-1: entries are missing in lookup of /fileop/fileop_L1_9/fileop_L1_9_L2_13.
[2010-10-11 03:00:17.218196] I [afr-common.c:716:afr_lookup_done] afr44-replicate-1: background  meta-data data entry self-heal triggered. path: /fileop/fileop_L1_9/fileop_L1_9_L2_13
[2010-10-11 03:00:17.225350] I [afr-self-heal-common.c:1526:afr_self_heal_completion_cbk] afr44-replicate-1: background  meta-data data entry self-heal completed on /fileop/fileop_L1_9/fileop_L1_9_L2_13
[2010-10-11 03:00:18.74176] I [afr-common.c:662:afr_lookup_done] afr44-replicate-1: entries are missing in lookup of /fileop/fileop_L1_9/fileop_L1_9_L2_14.
[2010-10-11 03:00:18.74223] I [afr-common.c:716:afr_lookup_done] afr44-replicate-1: background  meta-data data entry self-heal triggered. path: /fileop/fileop_L1_9/fileop_L1_9_L2_14
[2010-10-11 03:00:18.81421] I [afr-self-heal-common.c:1526:afr_self_heal_completion_cbk] afr44-replicate-1: background  meta-data data entry self-heal completed on /fileop/fileop_L1_9/fileop_L1_9_L2_14
[2010-10-11 03:00:18.897167] I [afr-common.c:662:afr_lookup_done] afr44-replicate-1: entries are missing in lookup of /fileop/fileop_L1_9/fileop_L1_9_L2_14/fileop_dir_9_14_24.
[2010-10-11 03:00:18.897215] I [afr-common.c:716:afr_lookup_done] afr44-replicate-1: background  meta-data data entry self-heal triggered. path: /fileop/fileop_L1_9/fileop_L1_9_L2_14/fileop_dir_9_14_24
[2010-10-11 03:00:18.904389] I [afr-self-heal-common.c:1526:afr_self_heal_completion_cbk] afr44-replicate-1: background  meta-data data entry self-heal completed on /fileop/fileop_L1_9/fileop_L1_9_L2_14/fileop_dir_9_14_24
[2010-10-11 03:00:19.892111] I [afr-common.c:662:afr_lookup_done] afr44-replicate-1: entries are missing in lookup of /fileop/fileop_L1_9/fileop_L1_9_L2_15/fileop_dir_9_15_22.
[2010-10-11 03:00:19.892160] I [afr-common.c:716:afr_lookup_done] afr44-replicate-1: background  meta-data data entry self-heal triggered. path: /fileop/fileop_L1_9/fileop_L1_9_L2_15/fileop_dir_9_15_22
[2010-10-11 03:00:19.898980] I [afr-self-heal-common.c:1526:afr_self_heal_completion_cbk] afr44-replicate-1: background  meta-data data entry self-heal completed on /fileop/fileop_L1_9/fileop_L1_9_L2_15/fileop_dir_9_15_22
[2010-10-11 03:00:19.981787] I [afr-common.c:662:afr_lookup_done] afr44-replicate-1: entries are missing in lookup of /fileop/fileop_L1_9/fileop_L1_9_L2_16.
[2010-10-11 03:00:19.981851] I [afr-common.c:716:afr_lookup_done] afr44-replicate-1: background  meta-data data entry self-heal triggered. path: /fileop/fileop_L1_9/fileop_L1_9_L2_16
[2010-10-11 03:00:19.989161] I [afr-self-heal-common.c:1526:afr_self_heal_completion_cbk] afr44-replicate-1: background  meta-data data entry self-heal completed on /fileop/fileop_L1_9/fileop_L1_9_L2_16
[2010-10-11 03:00:20.927041] I [afr-common.c:662:afr_lookup_done] afr44-replicate-1: entries are missing in lookup of /fileop/fileop_L1_9/fileop_L1_9_L2_17.
[2010-10-11 03:00:20.927089] I [afr-common.c:716:afr_lookup_done] afr44-replicate-1: background  meta-data data entry self-heal triggered. path: /fileop/fileop_L1_9/fileop_L1_9_L2_17
[2010-10-11 03:00:20.935032] I [afr-self-heal-common.c:1526:afr_self_heal_completion_cbk] afr44-replicate-1: background  meta-data data entry self-heal completed on /fileop/fileop_L1_9/fileop_L1_9_L2_17
[2010-10-11 03:00:21.905648] I [afr-common.c:662:afr_lookup_done] afr44-replicate-1: entries are missing in lookup of /fileop/fileop_L1_9/fileop_L1_9_L2_18/fileop_dir_9_18_4.
[2010-10-11 03:00:21.905696] I [afr-common.c:716:afr_lookup_done] afr44-replicate-1: background  meta-data data entry self-heal triggered. path: /fileop/fileop_L1_9/fileop_L1_9_L2_18/fileop_dir_9_18_4
[2010-10-11 03:00:21.913255] I [afr-self-heal-common.c:1526:afr_self_heal_completion_cbk] afr44-replicate-1: background  meta-data data entry self-heal completed on /fileop/fileop_L1_9/fileop_L1_9_L2_18/fileop_dir_9_18_4
[2010-10-11 03:00:22.688766] I [afr-common.c:662:afr_lookup_done] afr44-replicate-1: entries are missing in lookup of /fileop/fileop_L1_9/fileop_L1_9_L2_19.
[2010-10-11 03:00:22.688813] I [afr-common.c:716:afr_lookup_done] afr44-replicate-1: background  meta-data data entry self-heal triggered. path: /fileop/fileop_L1_9/fileop_L1_9_L2_19
[2010-10-11 03:00:22.696131] I [afr-self-heal-common.c:1526:afr_self_heal_completion_cbk] afr44-replicate-1: background  meta-data data entry self-heal completed on /fileop/fileop_L1_9/fileop_L1_9_L2_19
[2010-10-11 03:00:24.889894] I [afr-common.c:662:afr_lookup_done] afr44-replicate-1: entries are missing in lookup of /fileop/fileop_L1_9/fileop_L1_9_L2_21/fileop_dir_9_21_3.
[2010-10-11 03:00:24.889942] I [afr-common.c:716:afr_lookup_done] afr44-replicate-1: background  meta-data data entry self-heal triggered. path: /fileop/fileop_L1_9/fileop_L1_9_L2_21/fileop_dir_9_21_3
[2010-10-11 03:00:24.897404] I [afr-self-heal-common.c:1526:afr_self_heal_completion_cbk] afr44-replicate-1: background  meta-data data entry self-heal completed on /fileop/fileop_L1_9/fileop_L1_9_L2_21/fileop_dir_9_21_3
[2010-10-11 03:00:25.676395] I [afr-common.c:662:afr_lookup_done] afr44-replicate-1: entries are missing in lookup of /fileop/fileop_L1_9/fileop_L1_9_L2_22.
[2010-10-11 03:00:25.676435] I [afr-common.c:716:afr_lookup_done] afr44-replicate-1: background  meta-data data entry self-heal triggered. path: /fileop/fileop_L1_9/fileop_L1_9_L2_22
[2010-10-11 03:00:25.683780] I [afr-self-heal-common.c:1526:afr_self_heal_completion_cbk] afr44-replicate-1: background  meta-data data entry self-heal completed on /fileop/fileop_L1_9/fileop_L1_9_L2_22
[2010-10-11 03:00:25.885730] I [afr-common.c:662:afr_lookup_done] afr44-replicate-1: entries are missing in lookup of /fileop/fileop_L1_9/fileop_L1_9_L2_22/fileop_dir_9_22_5.
[2010-10-11 03:00:25.885758] I [afr-common.c:716:afr_lookup_done] afr44-replicate-1: background  meta-data data entry self-heal triggered. path: /fileop/fileop_L1_9/fileop_L1_9_L2_22/fileop_dir_9_22_5
[2010-10-11 03:00:25.894345] I [afr-self-heal-common.c:1526:afr_self_heal_completion_cbk] afr44-replicate-1: background  meta-data data entry self-heal completed on /fileop/fileop_L1_9/fileop_L1_9_L2_22/fileop_dir_9_22_5
[2010-10-11 03:00:26.600536] I [afr-common.c:662:afr_lookup_done] afr44-replicate-1: entries are missing in lookup of /fileop/fileop_L1_9/fileop_L1_9_L2_23.
[2010-10-11 03:00:26.600583] I [afr-common.c:716:afr_lookup_done] afr44-replicate-1: background  meta-data data entry self-heal triggered. path: /fileop/fileop_L1_9/fileop_L1_9_L2_23
[2010-10-11 03:00:26.607781] I [afr-self-heal-common.c:1526:afr_self_heal_completion_cbk] afr44-replicate-1: background  meta-data data entry self-heal completed on /fileop/fileop_L1_9/fileop_L1_9_L2_23
[2010-10-11 03:00:27.888264] I [afr-common.c:662:afr_lookup_done] afr44-replicate-1: entries are missing in lookup of /fileop/fileop_L1_9/fileop_L1_9_L2_24/fileop_dir_9_24_14.
[2010-10-11 03:00:27.888312] I [afr-common.c:716:afr_lookup_done] afr44-replicate-1: background  meta-data data entry self-heal triggered. path: /fileop/fileop_L1_9/fileop_L1_9_L2_24/fileop_dir_9_24_14
[2010-10-11 03:00:27.916382] I [afr-self-heal-common.c:1526:afr_self_heal_completion_cbk] afr44-replicate-1: background  meta-data data entry self-heal completed on /fileop/fileop_L1_9/fileop_L1_9_L2_24/fileop_dir_9_24_14
[2010-10-11 03:00:28.308821] I [afr-common.c:716:afr_lookup_done] afr44-replicate-1: background  entry self-heal triggered. path: /
[2010-10-11 03:00:28.334940] I [afr-self-heal-common.c:1526:afr_self_heal_completion_cbk] afr44-replicate-1: background  entry self-heal completed on /
[2010-10-11 03:00:28.338916] I [afr-common.c:716:afr_lookup_done] afr44-replicate-1: background  meta-data entry self-heal triggered. path: /fileop
[2010-10-11 03:00:28.347372] W [afr-self-heal-common.c:584:afr_sh_pending_to_delta] afr_sh_pending_to_delta: Unable to get dict value.
[2010-10-11 03:00:28.347445] W [afr-self-heal-common.c:584:afr_sh_pending_to_delta] afr_sh_pending_to_delta: Unable to get dict value.
[2010-10-11 03:00:28.406515] I [afr-self-heal-common.c:1526:afr_self_heal_completion_cbk] afr44-replicate-1: background  meta-data entry self-heal completed on /fileop
[2010-10-11 03:08:43.593898] I [afr-common.c:716:afr_lookup_done] afr44-replicate-1: background  meta-data entry self-heal triggered. path: /fileop/fileop_L1_8
[2010-10-11 03:08:43.602373] W [afr-self-heal-common.c:584:afr_sh_pending_to_delta] afr_sh_pending_to_delta: Unable to get dict value.
[2010-10-11 03:08:43.602397] W [afr-self-heal-common.c:584:afr_sh_pending_to_delta] afr_sh_pending_to_delta: Unable to get dict value.
[2010-10-11 03:08:43.635254] I [afr-self-heal-common.c:1526:afr_self_heal_completion_cbk] afr44-replicate-1: background  meta-data entry self-heal completed on /fileop/fileop_L1_8
[2010-10-11 03:08:58.841162] I [afr-common.c:716:afr_lookup_done] afr44-replicate-1: background  meta-data entry self-heal triggered. path: /fileop/fileop_L1_8/fileop_L1_8_L2_23
[2010-10-11 03:08:58.850028] W [afr-self-heal-common.c:584:afr_sh_pending_to_delta] afr_sh_pending_to_delta: Unable to get dict value.
[2010-10-11 03:08:58.850049] W [afr-self-heal-common.c:584:afr_sh_pending_to_delta] afr_sh_pending_to_delta: Unable to get dict value.
[2010-10-11 03:08:58.921120] I [afr-self-heal-common.c:1526:afr_self_heal_completion_cbk] afr44-replicate-1: background  meta-data entry self-heal completed on /fileop/fileop_L1_8/fileop_L1_8_L2_23
[2010-10-11 03:08:59.206508] I [afr-common.c:716:afr_lookup_done] afr44-replicate-1: background  meta-data entry self-heal triggered. path: /fileop/fileop_L1_8/fileop_L1_8_L2_23/fileop_dir_8_23_11
[2010-10-11 03:08:59.214987] W [afr-self-heal-common.c:584:afr_sh_pending_to_delta] afr_sh_pending_to_delta: Unable to get dict value.
[2010-10-11 03:08:59.215006] W [afr-self-heal-common.c:584:afr_sh_pending_to_delta] afr_sh_pending_to_delta: Unable to get dict value.
[2010-10-11 03:08:59.233124] I [afr-self-heal-common.c:1526:afr_self_heal_completion_cbk] afr44-replicate-1: background  meta-data entry self-heal completed on /fileop/fileop_L1_8/fileop_L1_8_L2_23/fileop_dir_8_23_11
[2010-10-11 03:08:59.233148] I [dht-common.c:415:dht_revalidate_cbk] afr44-dht: mismatching layouts for /fileop/fileop_L1_8/fileop_L1_8_L2_23/fileop_dir_8_23_11
pending frames:

patchset: v3.1.0qa7-558-g2eb9861
signal received: 11
time of crash: 2010-10-11 03:08:59
configuration details:
argp 1
backtrace 1
dlfcn 1
fdatasync 1
libpthread 1
llistxattr 1
setfsid 1
spinlock 1
epoll.h 1
xattr.h 1
st_atim.tv_nsec 1
package-string: glusterfs 3.1.0qa44
/lib64/libc.so.6[0x2aaaab7bdf30]
/usr/local/lib/glusterfs/3.1.0qa44/xlator/nfs/server.so(nfs3svc_access_cbk+0x52)[0x2aaaad96b01a]
/usr/local/lib/glusterfs/3.1.0qa44/xlator/nfs/server.so(nfs_fop_stat_cbk+0xb2)[0x2aaaad95a654]
/usr/local/lib/glusterfs/3.1.0qa44/xlator/debug/io-stats.so(io_stats_stat_cbk+0x168)[0x2aaaad7360bf]
/usr/local/lib/libglusterfs.so.0(default_stat_cbk+0x11f)[0x2aaaaace5fc6]
/usr/local/lib/libglusterfs.so.0(default_stat_cbk+0x11f)[0x2aaaaace5fc6]
/usr/local/lib/libglusterfs.so.0(default_stat_cbk+0x11f)[0x2aaaaace5fc6]
/usr/local/lib/glusterfs/3.1.0qa44/xlator/performance/write-behind.so(wb_stat_cbk+0x19b)[0x2aaaacef6bf9]
/usr/local/lib/glusterfs/3.1.0qa44/xlator/cluster/distribute.so(dht_attr_cbk+0x295)[0x2aaaaccd1070]
/usr/local/lib/glusterfs/3.1.0qa44/xlator/cluster/replicate.so(afr_stat_cbk+0x467)[0x2aaaaca69ff6]
/usr/local/lib/glusterfs/3.1.0qa44/xlator/protocol/client.so(client3_1_stat_cbk+0x252)[0x2aaaac835f76]
/usr/local/lib/libgfrpc.so.0(rpc_clnt_handle_reply+0x1e5)[0x2aaaaaf42385]
/usr/local/lib/libgfrpc.so.0(rpc_clnt_notify+0x289)[0x2aaaaaf426bc]
/usr/local/lib/libgfrpc.so.0(rpc_transport_notify+0xeb)[0x2aaaaaf3fe56]
/usr/local/lib/glusterfs/3.1.0qa44/rpc-transport/socket.so(socket_event_poll_in+0x4b)[0x2aaab0e5653d]
/usr/local/lib/glusterfs/3.1.0qa44/rpc-transport/socket.so(socket_event_handler+0xfc)[0x2aaab0e568ae]
/usr/local/lib/libglusterfs.so.0[0x2aaaaad0ce10]
/usr/local/lib/libglusterfs.so.0[0x2aaaaad0cfff]
/usr/local/lib/libglusterfs.so.0(event_dispatch+0x81)[0x2aaaaad0d35b]
/usr/local/sbin/glusterfs(main+0xec)[0x405e49]
/lib64/libc.so.6(__libc_start_main+0xf4)[0x2aaaab7ab074]
/usr/local/sbin/glusterfs[0x402d69]
---------

	pollin = (rpc_transport_pollin_t *) 0x11c2938
	tv = {tv_sec = 0, tv_usec = 0}
#12 0x00002aaaaaf3fe56 in rpc_transport_notify (this=0x64bb98, event=RPC_TRANSPORT_MSG_RECEIVED, data=0x11c2938) at rpc-transport.c:1142
	ret = -1
#13 0x00002aaab0e5653d in socket_event_poll_in (this=0x64bb98) at socket.c:1619
	ret = 0
	pollin = (rpc_transport_pollin_t *) 0x11c2938
#14 0x00002aaab0e568ae in socket_event_handler (fd=9, idx=2, data=0x64bb98, poll_in=1, poll_out=0, poll_err=0) at socket.c:1733
	this = (rpc_transport_t *) 0x64bb98
	priv = (socket_private_t *) 0x64bfa8
	ret = 0
	__FUNCTION__ = "socket_event_handler"
#15 0x00002aaaaad0ce10 in event_dispatch_epoll_handler (event_pool=0x62d378, events=0x64e988, i=0) at event.c:812
	event_data = (struct event_data *) 0x64e98c
	handler = (event_handler_t) 0x2aaab0e567b2 <socket_event_handler>
	data = (void *) 0x64bb98
	idx = 2
	ret = -1
	__FUNCTION__ = "event_dispatch_epoll_handler"
#16 0x00002aaaaad0cfff in event_dispatch_epoll (event_pool=0x62d378) at event.c:876
	events = (struct epoll_event *) 0x64e988
	size = 1
	i = 0
---Type <return> to continue, or q <return> to quit---
	ret = 1
	__FUNCTION__ = "event_dispatch_epoll"
#17 0x00002aaaaad0d35b in event_dispatch (event_pool=0x62d378) at event.c:984
	ret = -1
	__FUNCTION__ = "event_dispatch"
#18 0x0000000000405e49 in main (argc=7, argv=0x7fffaadec5e8) at glusterfsd.c:1410
	ctx = (glusterfs_ctx_t *) 0x62b010
	ret = 0
Comment 1 Amar Tumballi 2010-10-12 23:28:59 EDT
Fixed with the patch http://patches.gluster.com/patch/5461/