Description of problem: glusterd process segfaulted due to double free (may be possible double fclose). [2012-02-28 02:58:57.285084] I [glusterd-utils.c:796:glusterd_volume_brickinfo_get] 0-management: Found brick [2012-02-28 02:58:57.285091] D [glusterd-utils.c:820:glusterd_volume_brickinfo_get] 0-: Returning 0 [2012-02-28 02:58:57.285286] D [glusterd-utils.c:3757:glusterd_friend_find_by_hostname] 0-management: Friend 10.1.11.130 found.. state: 3 [2012-02-28 02:58:57.285298] D [glusterd-utils.c:3840:glusterd_hostname_to_uuid] 0-: returning 0 [2012-02-28 02:58:57.285305] I [glusterd-utils.c:796:glusterd_volume_brickinfo_get] 0-management: Found brick [2012-02-28 02:58:57.285311] D [glusterd-utils.c:820:glusterd_volume_brickinfo_get] 0-: Returning 0 [2012-02-28 02:58:57.285319] D [glusterd-utils.c:3757:glusterd_friend_find_by_hostname] 0-management: Friend 10.1.11.131 found.. state: 3 [2012-02-28 02:58:57.285326] D [glusterd-utils.c:3840:glusterd_hostname_to_uuid] 0-: returning 0 [2012-02-28 02:58:57.285332] I [glusterd-utils.c:796:glusterd_volume_brickinfo_get] 0-management: Found brick [2012-02-28 02:58:57.285339] D [glusterd-utils.c:820:glusterd_volume_brickinfo_get] 0-: Returning 0 [2012-02-28 02:58:57.285346] D [glusterd-utils.c:3757:glusterd_friend_find_by_hostname] 0-management: Friend 10.1.11.144 found.. state: 3 [2012-02-28 02:58:57.285353] D [glusterd-utils.c:3840:glusterd_hostname_to_uuid] 0-: returning 0 [2012-02-28 02:58:57.285359] I [glusterd-utils.c:796:glusterd_volume_brickinfo_get] 0-management: Found brick [2012-02-28 02:58:57.285366] D [glusterd-utils.c:820:glusterd_volume_brickinfo_get] 0-: Returning 0 [2012-02-28 02:58:57.286255] D [glusterd-utils.c:3806:glusterd_friend_find_by_hostname] 0-management: Unable to find friend: 10.1.11.145 [2012-02-28 02:58:57.286302] D [glusterd-utils.c:216:glusterd_is_local_addr] 0-management: 10.1.11.145 [2012-02-28 02:58:57.286315] D [glusterd-utils.c:225:glusterd_is_local_addr] 0-management: 10.1.11.145 is local [2012-02-28 02:58:57.286325] D [glusterd-utils.c:3840:glusterd_hostname_to_uuid] 0-: returning 0 [2012-02-28 02:58:57.286332] I [glusterd-utils.c:796:glusterd_volume_brickinfo_get] 0-management: Found brick [2012-02-28 02:58:57.286339] D [glusterd-utils.c:820:glusterd_volume_brickinfo_get] 0-: Returning 0 [2012-02-28 02:58:57.286474] D [glusterd-volgen.c:2873:build_nfs_graph] 0-glusterd: Returning 0 [2012-02-28 02:58:57.291239] D [run.c:194:runner_log] 0-: Starting the nfs/glustershd services: /usr/local/sbin/glusterfs -f /etc/glusterd/nfs/nfs-server.vol -p /etc/glusterd/nfs/run/nfs.pid -l /usr/local/var/log/glusterfs/nfs.log *** glibc detected *** glusterd: double free or corruption (!prev): 0x0000000000edd830 *** pending frames: patchset: git://git.gluster.com/glusterfs.git signal received: 11 This is the backtrace of the crash #0 0x000000390fc0caab in pthread_once () from /lib64/libpthread.so.0 #1 0x000000390f4fb7e4 in backtrace () from /lib64/libc.so.6 #2 0x00007f3759846e89 in _gf_log_callingfn (domain=0x40d55d "", file=0x40d538 "../../../glusterfsd/src/glusterfsd.c", function=0x40e1b0 "cleanup_and_exit", line=783, level=GF_LOG_WARNING, fmt=0x40d980 "received signum (%d), shutting down") at ../../../libglusterfs/src/logging.c:380 #3 0x0000000000405bfd in cleanup_and_exit (signum=2) at ../../../glusterfsd/src/glusterfsd.c:782 #4 <signal handler called> #5 0x000000390f4f4edc in __lll_lock_wait_private () from /lib64/libc.so.6 #6 0x000000390f47bb68 in _L_lock_9159 () from /lib64/libc.so.6 #7 0x000000390f479502 in malloc () from /lib64/libc.so.6 #8 0x000000390f47f762 in strdup () from /lib64/libc.so.6 #9 0x000000390f49bc11 in tzset_internal () from /lib64/libc.so.6 #10 0x000000390f49bd89 in __tz_convert () from /lib64/libc.so.6 #11 0x00007f37598597a0 in gf_print_trace (signum=11) at ../../../libglusterfs/src/common-utils.c:408 #12 <signal handler called> #13 0x000000390f475284 in malloc_consolidate () from /lib64/libc.so.6 #14 0x000000390f4780e2 in _int_malloc () from /lib64/libc.so.6 #15 0x000000390f4791a8 in calloc () from /lib64/libc.so.6 #16 0x000000390f00acaf in _dl_new_object () from /lib64/ld-linux-x86-64.so.2 #17 0x000000390f00717e in _dl_map_object_from_fd () from /lib64/ld-linux-x86-64.so.2 #18 0x000000390f00831a in _dl_map_object () from /lib64/ld-linux-x86-64.so.2 #19 0x000000390f0128d2 in dl_open_worker () from /lib64/ld-linux-x86-64.so.2 #20 0x000000390f00e0a6 in _dl_catch_error () from /lib64/ld-linux-x86-64.so.2 #21 0x000000390f01238a in _dl_open () from /lib64/ld-linux-x86-64.so.2 #22 0x000000390f522f20 in do_dlopen () from /lib64/libc.so.6 #23 0x000000390f00e0a6 in _dl_catch_error () from /lib64/ld-linux-x86-64.so.2 #24 0x000000390f523077 in __libc_dlopen_mode () from /lib64/libc.so.6 #25 0x000000390f4fb6e5 in init () from /lib64/libc.so.6 #26 0x000000390fc0cab3 in pthread_once () from /lib64/libpthread.so.0 #27 0x000000390f4fb7e4 in backtrace () from /lib64/libc.so.6 #28 0x000000390f46f83b in __libc_message () from /lib64/libc.so.6 #29 0x000000390f475146 in malloc_printerr () from /lib64/libc.so.6 #30 0x000000390f4658cd in fclose@@GLIBC_2.2.5 () from /lib64/libc.so.6 #31 0x00007f37562e6f22 in volgen_write_volfile (graph=0x7fffbc1ddb70, filename=0x7fffbc1ddc60 "/etc/glusterd/glustershd/glustershd-server.vol") at ../../../../../xlators/mgmt/glusterd/src/glusterd-volgen.c:1208 #32 0x00007f37562eb862 in glusterd_create_global_volfile (builder=0x7f37562e9fd8 <build_shd_graph>, filepath=0x7fffbc1ddc60 "/etc/glusterd/glustershd/glustershd-server.vol", mod_dict=0xef4e80) at ../../../../../xlators/mgmt/glusterd/src/glusterd-volgen.c:3173 #33 0x00007f37562eb9ef in glusterd_create_shd_volfile () at ../../../../../xlators/mgmt/glusterd/src/glusterd-volgen.c:3214 #34 0x00007f37562cc834 in glusterd_reconfigure_nodesvc (create_volfile=0x7f37562eb90a <glusterd_create_shd_volfile>) at ../../../../../xlators/mgmt/glusterd/src/glusterd-utils.c:2888 #35 0x00007f37562cc87c in glusterd_reconfigure_shd () at ../../../../../xlators/mgmt/glusterd/src/glusterd-utils.c:2901 #36 0x00007f37562cc950 in glusterd_nodesvcs_batch_op (volinfo=0xe618f0, nfs_op=0x7f37562cc87e <glusterd_check_generate_start_nfs>, #37 0x00007f37562ccc59 in glusterd_nodesvcs_handle_reconfigure (volinfo=0xe618f0) at ../../../../../xlators/mgmt/glusterd/src/glusterd-utils.c:3027 #38 0x00007f37562bb806 in glusterd_op_set_volume (dict=0xe59950) at ../../../../../xlators/mgmt/glusterd/src/glusterd-op-sm.c:1035 #39 0x00007f37562bf424 in glusterd_op_commit_perform (op=GD_OP_SET_VOLUME, dict=0xe59950, op_errstr=0x7fffbc1dee90, rsp_dict=0xe8aac0) at ../../../../../xlators/mgmt/glusterd/src/glusterd-op-sm.c:2454 #40 0x00007f37562becd1 in glusterd_op_ac_commit_op (event=0xeaa2a0, ctx=0xe61850) at ../../../../../xlators/mgmt/glusterd/src/glusterd-op-sm.c:2253 #41 0x00007f37562c2605 in glusterd_op_sm () at ../../../../../xlators/mgmt/glusterd/src/glusterd-op-sm.c:3861 #42 0x00007f37562af0b5 in glusterd_handle_commit_op (req=0x7f375621c04c) at ../../../../../xlators/mgmt/glusterd/src/glusterd-handler.c:651 #43 0x00007f37596180a9 in rpcsvc_handle_rpc_call (svc=0xe56530, trans=0xe5fd40, msg=0xe5f160) at ../../../../rpc/rpc-lib/src/rpcsvc.c:514 #44 0x00007f375961844c in rpcsvc_notify (trans=0xe5fd40, mydata=0xe56530, event=RPC_TRANSPORT_MSG_RECEIVED, data=0xe5f160) at ../../../../rpc/rpc-lib/src/rpcsvc.c:610 #45 0x00007f375961dda8 in rpc_transport_notify (this=0xe5fd40, event=RPC_TRANSPORT_MSG_RECEIVED, data=0xe5f160) at ../../../../rpc/rpc-lib/src/rpc-transport.c:498 #46 0x00007f3756011270 in socket_event_poll_in (this=0xe5fd40) at ../../../../../rpc/rpc-transport/socket/src/socket.c:1686 #47 0x00007f37560117f4 in socket_event_handler (fd=8, idx=5, data=0xe5fd40, poll_in=1, poll_out=0, poll_err=0) at ../../../../../rpc/rpc-transport/socket/src/socket.c:1801 #48 0x00007f375987805c in event_dispatch_epoll_handler (event_pool=0xe4ac20, events=0xe5d900, i=0) at ../../../libglusterfs/src/event.c:794 #49 0x00007f375987827f in event_dispatch_epoll (event_pool=0xe4ac20) at ../../../libglusterfs/src/event.c:856 #50 0x00007f375987860a in event_dispatch (event_pool=0xe4ac20) at ../../../libglusterfs/src/event.c:956 #51 0x0000000000407dcc in main (argc=2, argv=0x7fffbc1df4d8) at ../../../glusterfsd/src/glusterfsd.c:1612 p f $1 = (FILE *) 0xedd830 (gdb) p *f $2 = {_flags = 0, _IO_read_ptr = 0xed84e0 "", _IO_read_end = 0x0, _IO_read_base = 0x0, _IO_write_base = 0x0, _IO_write_ptr = 0x0, _IO_write_end = 0x0, _IO_buf_base = 0x0, _IO_buf_end = 0x0, _IO_save_base = 0x0, _IO_backup_base = 0x0, _IO_save_end = 0x0, _markers = 0x0, _chain = 0xee68d0, _fileno = -1, _flags2 = 0, _old_offset = 0, _cur_column = 0, _vtable_offset = 0 '\000', _shortbuf = "", _lock = 0xedd910, _offset = -1, __pad1 = 0x0, __pad2 = 0xedd920, __pad3 = 0x0, __pad4 = 0x0, __pad5 = 0, _mode = -1, _unused2 = '\000' <repeats 19 times>} (gdb) l volgen_write_volfile 1155 } 1156 } 1157 1158 static int 1159 volgen_write_volfile (volgen_graph_t *graph, char *filename) 1160 { 1161 char *ftmp = NULL; 1162 FILE *f = NULL; 1163 int fd = 0; 1164 xlator_t *this = NULL; (gdb) 1165 1166 this = THIS; 1167 1168 if (gf_asprintf (&ftmp, "%s.tmp", filename) == -1) { 1169 ftmp = NULL; 1170 1171 goto error; 1172 } 1173 1174 fd = creat (ftmp, S_IRUSR | S_IWUSR); (gdb) 1175 if (fd < 0) { 1176 gf_log (this->name, GF_LOG_ERROR, "%s", 1177 strerror (errno)); 1178 goto error; 1179 } 1180 1181 close (fd); 1182 1183 f = fopen (ftmp, "w"); 1184 if (!f) (gdb) 1185 goto error; 1186 1187 if (glusterfs_graph_print_file (f, &graph->graph) == -1) 1188 goto error; 1189 1190 if (fclose (f) == -1) 1191 goto error; 1192 f = NULL; 1193 1194 if (rename (ftmp, filename) == -1) (gdb) 1195 goto error; 1196 1197 GF_FREE (ftmp); 1198 1199 volgen_apply_filters(filename); 1200 1201 return 0; 1202 1203 error: 1204 (gdb) 1205 if (ftmp) 1206 GF_FREE (ftmp); 1207 if (f) 1208 fclose (f); 1209 1210 gf_log (this->name, GF_LOG_ERROR, 1211 "failed to create volfile %s", filename); 1212 1213 return -1; 1214 } (gdb) Version-Release number of selected component (if applicable): How reproducible: Steps to Reproduce: 1. 2. 3. Actual results: Expected results: Additional info:
CHANGE: http://review.gluster.com/2843 (mgmt/glusterd: do not close the same fd twice) merged in master by Vijay Bellur (vijay)
Its fixed now since we are not doing fclose twice. Checked with glusterfs-3.3.0qa40.