glusterd hung while volume set operations were running in a loop. This is the backtrace when the process was attached to gdb #0 0x0000003c8460caab in pthread_once () from /lib64/libpthread.so.0 #1 0x0000003c83efb7e4 in backtrace () from /lib64/libc.so.6 #2 0x0000003c83e6f83b in __libc_message () from /lib64/libc.so.6 #3 0x0000003c83e75146 in malloc_printerr () from /lib64/libc.so.6 #4 0x0000003c83e75571 in malloc_consolidate () from /lib64/libc.so.6 #5 0x0000003c83e780e2 in _int_malloc () from /lib64/libc.so.6 #6 0x0000003c83e791a8 in calloc () from /lib64/libc.so.6 #7 0x0000003c83a0acaf in _dl_new_object () from /lib64/ld-linux-x86-64.so.2 #8 0x0000003c83a0717e in _dl_map_object_from_fd () from /lib64/ld-linux-x86-64.so.2 #9 0x0000003c83a0831a in _dl_map_object () from /lib64/ld-linux-x86-64.so.2 #10 0x0000003c83a128d2 in dl_open_worker () from /lib64/ld-linux-x86-64.so.2 #11 0x0000003c83a0e0a6 in _dl_catch_error () from /lib64/ld-linux-x86-64.so.2 #12 0x0000003c83a1238a in _dl_open () from /lib64/ld-linux-x86-64.so.2 #13 0x0000003c83f22f20 in do_dlopen () from /lib64/libc.so.6 #14 0x0000003c83a0e0a6 in _dl_catch_error () from /lib64/ld-linux-x86-64.so.2 #15 0x0000003c83f23077 in __libc_dlopen_mode () from /lib64/libc.so.6 #16 0x0000003c83efb6e5 in init () from /lib64/libc.so.6 #17 0x0000003c8460cab3 in pthread_once () from /lib64/libpthread.so.0 #18 0x0000003c83efb7e4 in backtrace () from /lib64/libc.so.6 #19 0x0000003c83e6f83b in __libc_message () from /lib64/libc.so.6 #20 0x0000003c83e75146 in malloc_printerr () from /lib64/libc.so.6 #21 0x0000003c83e658cd in fclose@@GLIBC_2.2.5 () from /lib64/libc.so.6 #22 0x00007fd33602e494 in volgen_write_volfile (graph=0x7fff2737ee20, filename=0x7fff2737de20 "/etc/glusterd/vols/mirror/mirror.10.1.11.125.export-mirror.vol") at glusterd-volgen.c:1095 #23 0x00007fd33602e5cd in glusterd_generate_brick_volfile (volinfo=0x1b9d150, brickinfo=<value optimized out>) at glusterd-volgen.c:2283 #24 0x00007fd33602e7e1 in generate_brick_volfiles (volinfo=0x1b9d150) at glusterd-volgen.c:2344 #25 0x00007fd33602f596 in glusterd_create_volfiles_and_notify_services (volinfo=0x1b9d150) at glusterd-volgen.c:2453 #26 0x00007fd336009408 in glusterd_op_set_volume (dict=<value optimized out>) at glusterd-op-sm.c:5538 #27 0x00007fd3360102a8 in glusterd_op_commit_perform (op=<value optimized out>, dict=0x1ba15a0, op_errstr=0x7fff27380170, rsp_dict=<value optimized out>) at glusterd-op-sm.c:7449 #28 0x00007fd3360116a3 in glusterd_op_ac_send_commit_op (event=<value optimized out>, ctx=<value optimized out>) at glusterd-op-sm.c:6627 #29 0x00007fd3360052ae in glusterd_op_sm () at glusterd-op-sm.c:8285 #30 0x00007fd33602294c in glusterd3_1_stage_op_cbk (req=<value optimized out>, iov=<value optimized out>, count=<value optimized out>, myframe=0x7fd336cf9314) at glusterd-rpc-ops.c:1091 #31 0x00007fd337c7b365 in rpc_clnt_handle_reply (clnt=0x1370cc0, pollin=0x1371950) at rpc-clnt.c:741 #32 0x00007fd337c7b5b9 in rpc_clnt_notify (trans=<value optimized out>, mydata=0x1370cf0, event=<value optimized out>, data=<value optimized out>) at rpc-clnt.c:854 #33 0x00007fd337c76458 in rpc_transport_notify (this=<value optimized out>, event=<value optimized out>, data=<value optimized out>) at rpc-transport.c:919 #34 0x00007fd335d1a254 in socket_event_poll_in (this=0x16299c0) at socket.c:1647 #35 0x00007fd335d1a337 in socket_event_handler (fd=<value optimized out>, idx=3, data=0x16299c0, poll_in=1, poll_out=0, poll_err=<value optimized out>) at socket.c:1762 #36 0x00007fd337ec2504 in event_dispatch_epoll_handler (event_pool=0x1362360) at event.c:794 #37 event_dispatch_epoll (event_pool=0x1362360) at event.c:856 #38 0x0000000000405e99 in main (argc=1, argv=0x7fff27380d78) at glusterfsd.c:1509 f 22 #22 0x00007fd33602e494 in volgen_write_volfile (graph=0x7fff2737ee20, filename=0x7fff2737de20 "/etc/glusterd/vols/mirror/mirror.10.1.11.125.export-mirror.vol") at glusterd-volgen.c:1095 1095 fclose (f); (gdb) p f $3 = (FILE *) 0x1ba0ee0 (gdb) p * A syntax error in expression, near `'. (gdb) p *f $4 = {_flags = 0, _IO_read_ptr = 0x1ba5130 "", _IO_read_end = 0x0, _IO_read_base = 0x0, _IO_write_base = 0x0, _IO_write_ptr = 0x0, _IO_write_end = 0x0, _IO_buf_base = 0x0, _IO_buf_end = 0x0, _IO_save_base = 0x0, _IO_backup_base = 0x0, _IO_save_end = 0x0, _markers = 0x0, _chain = 0x136d9a0, _fileno = -1, _flags2 = 0, _old_offset = 48, _cur_column = 0, _vtable_offset = -70 '\272', _shortbuf = "\001", _lock = 0x1ba0fc0, _offset = -1, __pad1 = 0x0, __pad2 = 0x1ba0fd0, __pad3 = 0x0, __pad4 = 0x1ba1060, __pad5 = 0, _mode = -1, _unused2 = '\000' <repeats 19 times>} (gdb) l 1090 error: 1091 1092 if (ftmp) 1093 GF_FREE (ftmp); 1094 if (f) 1095 fclose (f); 1096 1097 gf_log ("", GF_LOG_ERROR, "failed to create volfile %s", filename); 1098 1099 return -1; (gdb) l - 1080 goto error; 1081 f = NULL; 1082 1083 if (rename (ftmp, filename) == -1) 1084 goto error; 1085 1086 GF_FREE (ftmp); 1087 1088 return 0; 1089 (gdb)
Can the problem be reproduced with valgrind please?
not seen this in a long time, will be closing the bug as INSUFFICIENT_DATA.