Description of problem: glusterd generated a core file while running ./tests/bugs/cli/bug-1077682.t in centos running. Core and logs can be found in https://build.gluster.org/job/centos7-regression/5857/consoleFull
s/"centos running"/"centos regression"
(gdb) bt #0 0x00007fa56532b207 in raise () from ./lib64/libc.so.6 #1 0x00007fa56532c8f8 in abort () from ./lib64/libc.so.6 #2 0x00007fa56536dd27 in __libc_message () from ./lib64/libc.so.6 #3 0x00007fa56536de0e in __libc_fatal () from ./lib64/libc.so.6 #4 0x00007fa56536e183 in _IO_vtable_check () from ./lib64/libc.so.6 #5 0x00007fa565372c9b in _IO_cleanup () from ./lib64/libc.so.6 #6 0x00007fa56532eb1b in __run_exit_handlers () from ./lib64/libc.so.6 #7 0x00007fa56532ebb7 in exit () from ./lib64/libc.so.6 #8 0x0000000000409485 in cleanup_and_exit (signum=15) at /home/jenkins/root/workspace/centos7-regression/glusterfsd/src/glusterfsd.c:1659 #9 0x000000000040b093 in glusterfs_sigwaiter (arg=0x7fff6cf56250) at /home/jenkins/root/workspace/centos7-regression/glusterfsd/src/glusterfsd.c:2421 #10 0x00007fa565b2bdd5 in start_thread () from ./lib64/libpthread.so.0 #11 0x00007fa5653f2ead in clone () from ./lib64/libc.so.6 (gdb) t a a bt Thread 9 (LWP 15288): #0 0x00007fa565b32e3d in nanosleep () from ./lib64/libpthread.so.0 #1 0x00007fa566d11c77 in gf_timer_proc (data=0xe569d0) at /home/jenkins/root/workspace/centos7-regression/libglusterfs/src/timer.c:194 #2 0x00007fa565b2bdd5 in start_thread () from ./lib64/libpthread.so.0 #3 0x00007fa5653f2ead in clone () from ./lib64/libc.so.6 Thread 8 (LWP 15287): #0 0x00007fa565b2cf47 in pthread_join () from ./lib64/libpthread.so.0 #1 0x00007fa566d7be2b in event_dispatch_epoll (event_pool=0xe4ee40) at /home/jenkins/root/workspace/centos7-regression/libglusterfs/src/event-epoll.c:846 #2 0x00007fa566d38405 in event_dispatch (event_pool=0xe4ee40) at /home/jenkins/root/workspace/centos7-regression/libglusterfs/src/event.c:116 #3 0x000000000040c019 in main (argc=1, argv=0x7fff6cf574a8) at /home/jenkins/root/workspace/centos7-regression/glusterfsd/src/glusterfsd.c:2917 Thread 7 (LWP 15321): #0 0x00007fa565b2f965 in pthread_cond_wait@@GLIBC_2.3.2 () from ./lib64/libpthread.so.0 #1 0x00007fa55aeb1b50 in hooks_worker (args=0xe61d70) at /home/jenkins/root/workspace/centos7-regression/xlators/mgmt/glusterd/src/glusterd-hooks.c:527 #2 0x00007fa565b2bdd5 in start_thread () from ./lib64/libpthread.so.0 #3 0x00007fa5653f2ead in clone () from ./lib64/libc.so.6 Thread 6 (LWP 15293): #0 0x00007fa5653e9f73 in select () from ./lib64/libc.so.6 #1 0x00007fa566d9a526 in runner (arg=0xe5b3c0) at /home/jenkins/root/workspace/centos7-regression/contrib/timer-wheel/timer-wheel.c:186 #2 0x00007fa565b2bdd5 in start_thread () from ./lib64/libpthread.so.0 #3 0x00007fa5653f2ead in clone () from ./lib64/libc.so.6 Thread 5 (LWP 15290): #0 0x00007fa5653b9e2d in nanosleep () from ./lib64/libc.so.6 #1 0x00007fa5653b9cc4 in sleep () from ./lib64/libc.so.6 #2 0x00007fa566d399f8 in pool_sweeper (arg=0x0) at /home/jenkins/root/workspace/centos7-regression/libglusterfs/src/mem-pool.c:446 #3 0x00007fa565b2bdd5 in start_thread () from ./lib64/libpthread.so.0 #4 0x00007fa5653f2ead in clone () from ./lib64/libc.so.6 Thread 4 (LWP 15322): #0 0x00007fa565b324ed in __lll_lock_wait () from ./lib64/libpthread.so.0 #1 0x00007fa565b2ddcb in _L_lock_883 () from ./lib64/libpthread.so.0 #2 0x00007fa565b2dc98 in pthread_mutex_lock () from ./lib64/libpthread.so.0 #3 0x00007fa55aee982e in gd_peerinfo_find_from_hostname (hoststr=0x7fa548007520 "builder204.int.aws.gluster.org") at /home/jenkins/root/workspace/centos7-regression/xlators/mgmt/glusterd/src/glusterd-peer-utils.c:650 #4 0x00007fa55aee7935 in glusterd_peerinfo_find_by_hostname (hoststr=0x7fa548007520 "builder204.int.aws.gluster.org") at /home/jenkins/root/workspace/centos7-regression/xlators/mgmt/glusterd/src/glusterd-peer-utils.c:112 #5 0x00007fa55aee7b77 in glusterd_hostname_to_uuid (hostname=0x7fa548007520 "builder204.int.aws.gluster.org", uuid=0x7fa557cd2ab0 "") at /home/jenkins/root/workspace/centos7-regression/xlators/mgmt/glusterd/src/glusterd-peer-utils.c:154 #6 0x00007fa55ae024b7 in glusterd_volume_brickinfo_get (uuid=0x0, hostname=0x7fa548007520 "builder204.int.aws.gluster.org", path=0x7fa54800761f "/d/backends/patchy4", volinfo=0xed3760, brickinfo=0x7fa557cd5c10) at /home/jenkins/root/workspace/centos7-regression/xlators/mgmt/glusterd/src/glusterd-utils.c:1611 #7 0x00007fa55ae0273e in glusterd_volume_brickinfo_get_by_brick (brick=0x7fa548001bd5 "builder204.int.aws.gluster.org:/d/backends/patchy4", volinfo=0xed3760, brickinfo=0x7fa557cd5c10, construct_real_path=false) at /home/jenkins/root/workspace/centos7-regression/xlators/mgmt/glusterd/src/glusterd-utils.c:1655 #8 0x00007fa55addfb0f in get_brickinfo_from_brickid ( brickid=0x7fa550004640 "a2393496-9716-4c17-a016-8512a0b911a7:builder204.int.aws.gluster.org:/d/backends/patchy4", brickinfo=0x7fa557cd5c10) at /home/jenkins/root/workspace/centos7-regression/xlators/mgmt/glusterd/src/glusterd-handler.c:6010 --Type <RET> for more, q to quit, c to continue without paging-- #9 0x00007fa55addfbf6 in __glusterd_brick_rpc_notify (rpc=0x7fa550004700, mydata=0x7fa550004640, event=RPC_CLNT_DISCONNECT, data=0x0) at /home/jenkins/root/workspace/centos7-regression/xlators/mgmt/glusterd/src/glusterd-handler.c:6044 #10 0x00007fa55adccdb1 in glusterd_big_locked_notify (rpc=0x7fa550004700, mydata=0x7fa550004640, event=RPC_CLNT_DISCONNECT, data=0x0, notify_fn=0x7fa55addfb32 <__glusterd_brick_rpc_notify>) at /home/jenkins/root/workspace/centos7-regression/xlators/mgmt/glusterd/src/glusterd-handler.c:66 #11 0x00007fa55ade0581 in glusterd_brick_rpc_notify (rpc=0x7fa550004700, mydata=0x7fa550004640, event=RPC_CLNT_DISCONNECT, data=0x0) at /home/jenkins/root/workspace/centos7-regression/xlators/mgmt/glusterd/src/glusterd-handler.c:6199 #12 0x00007fa566a9f668 in rpc_clnt_handle_disconnect (clnt=0x7fa550004700, conn=0x7fa550004730) at /home/jenkins/root/workspace/centos7-regression/rpc/rpc-lib/src/rpc-clnt.c:826 #13 0x00007fa566a9f927 in rpc_clnt_notify (trans=0x7fa550004a80, mydata=0x7fa550004730, event=RPC_TRANSPORT_DISCONNECT, data=0x7fa550004a80) at /home/jenkins/root/workspace/centos7-regression/rpc/rpc-lib/src/rpc-clnt.c:887 #14 0x00007fa566a9ba5b in rpc_transport_notify (this=0x7fa550004a80, event=RPC_TRANSPORT_DISCONNECT, data=0x7fa550004a80) at /home/jenkins/root/workspace/centos7-regression/rpc/rpc-lib/src/rpc-transport.c:549 #15 0x00007fa559fedd8e in socket_event_poll_err (this=0x7fa550004a80, gen=1, idx=3) at /home/jenkins/root/workspace/centos7-regression/rpc/rpc-transport/socket/src/socket.c:1385 #16 0x00007fa559ff3f17 in socket_event_handler (fd=7, idx=3, gen=1, data=0x7fa550004a80, poll_in=1, poll_out=4, poll_err=16, event_thread_died=0 '\000') at /home/jenkins/root/workspace/centos7-regression/rpc/rpc-transport/socket/src/socket.c:3025 #17 0x00007fa566d7b680 in event_dispatch_epoll_handler (event_pool=0xe4ee40, event=0x7fa557cd6140) at /home/jenkins/root/workspace/centos7-regression/libglusterfs/src/event-epoll.c:648 #18 0x00007fa566d7bb99 in event_dispatch_epoll_worker (data=0xeed3e0) at /home/jenkins/root/workspace/centos7-regression/libglusterfs/src/event-epoll.c:761 #19 0x00007fa565b2bdd5 in start_thread () from ./lib64/libpthread.so.0 #20 0x00007fa5653f2ead in clone () from ./lib64/libc.so.6 Thread 3 (LWP 15292): #0 0x00007fa565b2fd12 in pthread_cond_timedwait@@GLIBC_2.3.2 () from ./lib64/libpthread.so.0 #1 0x00007fa566d51f02 in syncenv_task (proc=0xe57600) at /home/jenkins/root/workspace/centos7-regression/libglusterfs/src/syncop.c:517 #2 0x00007fa566d520f7 in syncenv_processor (thdata=0xe57600) at /home/jenkins/root/workspace/centos7-regression/libglusterfs/src/syncop.c:584 #3 0x00007fa565b2bdd5 in start_thread () from ./lib64/libpthread.so.0 #4 0x00007fa5653f2ead in clone () from ./lib64/libc.so.6 Thread 2 (LWP 15291): #0 0x00007fa565b2fd12 in pthread_cond_timedwait@@GLIBC_2.3.2 () from ./lib64/libpthread.so.0 #1 0x00007fa566d51f02 in syncenv_task (proc=0xe57240) at /home/jenkins/root/workspace/centos7-regression/libglusterfs/src/syncop.c:517 #2 0x00007fa566d520f7 in syncenv_processor (thdata=0xe57240) at /home/jenkins/root/workspace/centos7-regression/libglusterfs/src/syncop.c:584 #3 0x00007fa565b2bdd5 in start_thread () from ./lib64/libpthread.so.0 #4 0x00007fa5653f2ead in clone () from ./lib64/libc.so.6 Thread 1 (LWP 15289): #0 0x00007fa56532b207 in raise () from ./lib64/libc.so.6 #1 0x00007fa56532c8f8 in abort () from ./lib64/libc.so.6 #2 0x00007fa56536dd27 in __libc_message () from ./lib64/libc.so.6 #3 0x00007fa56536de0e in __libc_fatal () from ./lib64/libc.so.6 #4 0x00007fa56536e183 in _IO_vtable_check () from ./lib64/libc.so.6 #5 0x00007fa565372c9b in _IO_cleanup () from ./lib64/libc.so.6 #6 0x00007fa56532eb1b in __run_exit_handlers () from ./lib64/libc.so.6 #7 0x00007fa56532ebb7 in exit () from ./lib64/libc.so.6 #8 0x0000000000409485 in cleanup_and_exit (signum=15) at /home/jenkins/root/workspace/centos7-regression/glusterfsd/src/glusterfsd.c:1659 #9 0x000000000040b093 in glusterfs_sigwaiter (arg=0x7fff6cf56250) at /home/jenkins/root/workspace/centos7-regression/glusterfsd/src/glusterfsd.c:2421 #10 0x00007fa565b2bdd5 in start_thread () from ./lib64/libpthread.so.0 #11 0x00007fa5653f2ead in clone () from ./lib64/libc.so.6
Looking at the backtrace, I think this issue has been fixed by me some time ago. Closing this issue as current release. If you still happen to see this issue, please feel free to re-open.