Description of problem: ./tests/bugs/snapshot/bug-1399598-uss-with-ssl.t is generating crash. Version-Release number of selected component (if applicable): How reproducible: Always Steps to Reproduce: 1. Run .t in a loop for i in `seq 1 10 do; prove -vf ./tests/bugs/snapshot/bug-1399598-uss-with-ssl.t; done 2. 3. Actual results: Test case is crashing Expected results: Test case should not crash Additional info:
Hi, test case is generating below core dump, As we can see in thread 1 ssl ctx is NULL so ssl api are crashing. ssl ctx is NULL because rpc_clnt_disable is already called by the client and rpc_clnt_disable internally call ssl_teardown_connection to cleanup ssl connection. Thread 17 (Thread 0x7fad4e5f9700 (LWP 28214)): #0 0x00007fad73bdb17f in epoll_wait () from /lib64/libc.so.6 #1 0x00007fad7581451c in event_dispatch_epoll_worker (data=0x7fad44000ca0) at event-epoll.c:751 #2 0x00007fad74332594 in start_thread () from /lib64/libpthread.so.0 #3 0x00007fad73bdae5f in clone () from /lib64/libc.so.6 Thread 16 (Thread 0x7fad591d7700 (LWP 28209)): #0 0x00007fad7433893a in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fad757f40b6 in syncenv_task (proc=proc@entry=0x7fad5003a320) at syncop.c:517 #2 0x00007fad757f4e90 in syncenv_processor (thdata=0x7fad5003a320) at syncop.c:584 #3 0x00007fad74332594 in start_thread () from /lib64/libpthread.so.0 #4 0x00007fad73bdae5f in clone () from /lib64/libc.so.6 Thread 15 (Thread 0x7fad62bc2700 (LWP 28101)): #0 0x00007fad73bd27a7 in select () from /lib64/libc.so.6 #1 0x00007fad7582ac7d in runner (arg=0xb67550) at ../../contrib/timer-wheel/timer-wheel.c:186 #2 0x00007fad74332594 in start_thread () from /lib64/libpthread.so.0 #3 0x00007fad73bdae5f in clone () from /lib64/libc.so.6 Thread 14 (Thread 0x7fad61319700 (LWP 28112)): #0 0x00007fad743385ec in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fad4db83b4b in fini (this=<optimized out>) at client.c:2797 #2 0x00007fad757b4acc in xlator_fini_rec (xl=0x7fad400059a0) at xlator.c:667 #3 0x00007fad757b4a5e in xlator_fini_rec (xl=0x7fad40018dc0) at xlator.c:657 #4 0x00007fad757b4a5e in xlator_fini_rec (xl=0x7fad4001aa00) at xlator.c:657 #5 0x00007fad757b4a5e in xlator_fini_rec (xl=0x7fad4001c3d0) at xlator.c:657 #6 0x00007fad757b4a5e in xlator_fini_rec (xl=0x7fad4001dee0) at xlator.c:657 #7 0x00007fad757b4a5e in xlator_fini_rec (xl=0x7fad4001f950) at xlator.c:657 #8 0x00007fad757b4a5e in xlator_fini_rec (xl=0x7fad400213e0) at xlator.c:657 ---Type <return> to continue, or q <return> to quit--- #9 0x00007fad757b4a5e in xlator_fini_rec (xl=0x7fad40023590) at xlator.c:657 #10 0x00007fad757b4a5e in xlator_fini_rec (xl=0x7fad40025040) at xlator.c:657 #11 0x00007fad757b4a5e in xlator_fini_rec (xl=0x7fad40026b10) at xlator.c:657 #12 0x00007fad757b4a5e in xlator_fini_rec (xl=0x7fad400285c0) at xlator.c:657 #13 0x00007fad757b4a5e in xlator_fini_rec (xl=0x7fad4002a0a0) at xlator.c:657 #14 0x00007fad757b4a5e in xlator_fini_rec (xl=0x7fad4002b3f0) at xlator.c:657 #15 0x00007fad757b4a5e in xlator_fini_rec (xl=0x7fad4002d340) at xlator.c:657 #16 0x00007fad757b5c5a in xlator_tree_fini (xl=<optimized out>) at xlator.c:759 #17 0x00007fad757eef72 in glusterfs_graph_deactivate (graph=<optimized out>) at graph.c:427 #18 0x00007fad606dc443 in pub_glfs_fini (fs=0x7fad50001890) at glfs.c:1353 #19 0x00007fad6090ec95 in mgmt_get_snapinfo_cbk (req=<optimized out>, iov=<optimized out>, count=<optimized out>, myframe=<optimized out>) at snapview-server-mgmt.c:410 #20 0x00007fad755620e1 in rpc_clnt_handle_reply (clnt=clnt@entry=0xb8fa20, pollin=pollin@entry=0x7fad5c04db40) at rpc-clnt.c:772 #21 0x00007fad75562493 in rpc_clnt_notify (trans=0xb8fbf0, mydata=0xb8fa50, event=<optimized out>, data=0x7fad5c04db40) at rpc-clnt.c:941 #22 0x00007fad7555f253 in rpc_transport_notify (this=this@entry=0xb8fbf0, event=event@entry=RPC_TRANSPORT_MSG_RECEIVED, data=data@entry=0x7fad5c04db40) at rpc-transport.c:549 #23 0x00007fad621a6ef8 in socket_event_poll_in_async (xl=<optimized out>, async=0x7fad5c04dc68) at socket.c:2572 #24 0x00007fad621ae201 in gf_async (cbk=0x7fad621a6ed0 <socket_event_poll_in_async>, xl=<optimized out>, async=0x7fad5c04dc68) at ../../../../libglusterfs/src/glusterfs/async.h:189 #25 socket_event_poll_in (notify_handled=true, this=0xb8fbf0) at socket.c:2613 #26 socket_event_handler (fd=fd@entry=10, idx=idx@entry=1, gen=gen@entry=1, data=data@entry=0xb8fbf0, poll_in=<optimized out>, poll_out=<optimized out>, poll_err=<optimized out>, event_thread_died=0 '\000') at socket.c:3004 #27 0x00007fad7581461b in event_dispatch_epoll_handler (event=0x7fad61318024, event_pool=0xb58aa0) at event-epoll.c:648 #28 event_dispatch_epoll_worker (data=0xbb4760) at event-epoll.c:761 #29 0x00007fad74332594 in start_thread () from /lib64/libpthread.so.0 #30 0x00007fad73bdae5f in clone () from /lib64/libc.so.6 Thread 13 (Thread 0x7fad4f5fb700 (LWP 28212)): ---Type <return> to continue, or q <return> to quit--- #0 0x00007fad74333a2d in __pthread_timedjoin_ex () from /lib64/libpthread.so.0 #1 0x00007fad75813d07 in event_dispatch_epoll (event_pool=0x7fad50035260) at event-epoll.c:846 #2 0x00007fad606da404 in glfs_poller (data=<optimized out>) at glfs.c:727 #3 0x00007fad74332594 in start_thread () from /lib64/libpthread.so.0 #4 0x00007fad73bdae5f in clone () from /lib64/libc.so.6 Thread 12 (Thread 0x7fad4ffff700 (LWP 28211)): #0 0x00007fad7433c460 in nanosleep () from /lib64/libpthread.so.0 #1 0x00007fad757c6136 in gf_timer_proc (data=0x7fad5003e610) at timer.c:194 #2 0x00007fad74332594 in start_thread () from /lib64/libpthread.so.0 #3 0x00007fad73bdae5f in clone () from /lib64/libc.so.6 Thread 11 (Thread 0x7fad589d6700 (LWP 28210)): #0 0x00007fad7433893a in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fad757f40b6 in syncenv_task (proc=proc@entry=0x7fad5003a6e0) at syncop.c:517 #2 0x00007fad757f4e90 in syncenv_processor (thdata=0x7fad5003a6e0) at syncop.c:584 #3 0x00007fad74332594 in start_thread () from /lib64/libpthread.so.0 #4 0x00007fad73bdae5f in clone () from /lib64/libc.so.6 Thread 10 (Thread 0x7fad5a791700 (LWP 28132)): #0 0x00007fad7433893a in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fad5b9c4b5d in iot_worker (data=0x7fad5c035bc0) at io-threads.c:197 #2 0x00007fad74332594 in start_thread () from /lib64/libpthread.so.0 #3 0x00007fad73bdae5f in clone () from /lib64/libc.so.6 Thread 9 (Thread 0x7fad653c7700 (LWP 28096)): #0 0x00007fad7433c460 in nanosleep () from /lib64/libpthread.so.0 #1 0x00007fad757c6136 in gf_timer_proc (data=0xb60c70) at timer.c:194 #2 0x00007fad74332594 in start_thread () from /lib64/libpthread.so.0 #3 0x00007fad73bdae5f in clone () from /lib64/libc.so.6 ---Type <return> to continue, or q <return> to quit--- Thread 8 (Thread 0x7fad75c9b5c0 (LWP 28095)): #0 0x00007fad74333a2d in __pthread_timedjoin_ex () from /lib64/libpthread.so.0 #1 0x00007fad75813d07 in event_dispatch_epoll (event_pool=0xb58aa0) at event-epoll.c:846 #2 0x00000000004060f8 in main (argc=<optimized out>, argv=<optimized out>) at glusterfsd.c:2917 Thread 7 (Thread 0x7fad5a618700 (LWP 28208)): #0 0x00007fad743385ec in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fad7555c6e2 in rpcsvc_request_handler (arg=0x7fad6000c080) at rpcsvc.c:2189 #2 0x00007fad74332594 in start_thread () from /lib64/libpthread.so.0 #3 0x00007fad73bdae5f in clone () from /lib64/libc.so.6 Thread 6 (Thread 0x7fad61b1a700 (LWP 28111)): #0 0x00007fad73bdb17f in epoll_wait () from /lib64/libc.so.6 #1 0x00007fad7581451c in event_dispatch_epoll_worker (data=0xbb45e0) at event-epoll.c:751 #2 0x00007fad74332594 in start_thread () from /lib64/libpthread.so.0 #3 0x00007fad73bdae5f in clone () from /lib64/libc.so.6 Thread 5 (Thread 0x7fad633c3700 (LWP 28100)): #0 0x00007fad7433893a in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fad757f40b6 in syncenv_task (proc=proc@entry=0xb63c60) at syncop.c:517 #2 0x00007fad757f4e90 in syncenv_processor (thdata=0xb63c60) at syncop.c:584 #3 0x00007fad74332594 in start_thread () from /lib64/libpthread.so.0 #4 0x00007fad73bdae5f in clone () from /lib64/libc.so.6 Thread 4 (Thread 0x7fad63bc4700 (LWP 28099)): #0 0x00007fad7433893a in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fad757f40b6 in syncenv_task (proc=proc@entry=0xb638a0) at syncop.c:517 #2 0x00007fad757f4e90 in syncenv_processor (thdata=0xb638a0) at syncop.c:584 #3 0x00007fad74332594 in start_thread () from /lib64/libpthread.so.0 ---Type <return> to continue, or q <return> to quit--- #4 0x00007fad73bdae5f in clone () from /lib64/libc.so.6 Thread 3 (Thread 0x7fad643c5700 (LWP 28098)): #0 0x00007fad73ba76b0 in nanosleep () from /lib64/libc.so.6 #1 0x00007fad73ba758a in sleep () from /lib64/libc.so.6 #2 0x00007fad757e0c45 in pool_sweeper (arg=<optimized out>) at mem-pool.c:446 #3 0x00007fad74332594 in start_thread () from /lib64/libpthread.so.0 #4 0x00007fad73bdae5f in clone () from /lib64/libc.so.6 Thread 2 (Thread 0x7fad64bc6700 (LWP 28097)): #0 0x00007fad73b18c8c in sigtimedwait () from /lib64/libc.so.6 #1 0x00007fad7433cc5c in sigwait () from /lib64/libpthread.so.0 #2 0x0000000000406667 in glusterfs_sigwaiter (arg=<optimized out>) at glusterfsd.c:2414 #3 0x00007fad74332594 in start_thread () from /lib64/libpthread.so.0 #4 0x00007fad73bdae5f in clone () from /lib64/libc.so.6 Thread 1 (Thread 0x7fad4edfa700 (LWP 28213)): #0 0x00007fad73f3c654 in BIO_test_flags () from /lib64/libcrypto.so.1.1 #1 0x00007fad73f3d0d6 in BIO_copy_next_retry () from /lib64/libcrypto.so.1.1 #2 0x00007fad73f3a9fb in buffer_ctrl () from /lib64/libcrypto.so.1.1 #3 0x00007fad61f5c6b2 in ssl3_dispatch_alert () from /lib64/libssl.so.1.1 #4 0x00007fad61f700fc in ossl_statem_client_read_transition () from /lib64/libssl.so.1.1 #5 0x00007fad61f6ef59 in state_machine () from /lib64/libssl.so.1.1 #6 0x00007fad61f67135 in SSL_do_handshake () from /lib64/libssl.so.1.1 #7 0x00007fad621ac398 in ssl_complete_connection (this=this@entry=0x7fad40032c00) at socket.c:485 #8 0x00007fad621ac9ed in ssl_handle_client_connection_attempt (this=0x7fad40032c00) at socket.c:2812 #9 socket_complete_connection (this=0x7fad40032c00) at socket.c:2911 #10 socket_event_handler (fd=fd@entry=17, idx=idx@entry=1, gen=gen@entry=4, data=data@entry=0x7fad40032c00, poll_in=0, poll_out=4, poll_err=0, event_thread_died=0 '\000') at socket.c:2973 #11 0x00007fad7581461b in event_dispatch_epoll_handler (event=0x7fad4edf9024, event_pool=0x7fad50035260) at event-epoll.c:648 ---Type <return> to continue, or q <return> to quit--- #12 event_dispatch_epoll_worker (data=0x7fad44000b20) at event-epoll.c:761 #13 0x00007fad74332594 in start_thread () from /lib64/libpthread.so.0 #14 0x00007fad73bdae5f in clone () from /lib64/libc.so.6 (gdb) p priv->ssl_ssl $3 = (SSL *) 0x0 Thanks, Mohit Agrawal
REVIEW: https://review.gluster.org/22745 (rpc: test case bug-1399598-uss-with-ssl.t is generating crash) posted (#1) for review on master by MOHIT AGRAWAL
REVIEW: https://review.gluster.org/22750 (Revert \"rpc: implement reconnect back-off strategy\") posted (#4) for review on master by Amar Tumballi
REVIEW: https://review.gluster.org/22750 (Revert \"rpc: implement reconnect back-off strategy\") merged (#5) on master by Xavi Hernandez