glusterd crashed with following trace. #0 0x00002b42d9715f6c in rpcsvc_submit_generic (req=0x2aaaaad0903c, proghdr=0x7fffc4fbbde0, hdrcount=0, payload=0x0, payloadcount=0, iobref=0x5d1c038) at rpcsvc.c:1539 1539 gf_log (GF_RPCSVC, GF_LOG_ERROR, "failed to submit message " (gdb) bt #0 0x00002b42d9715f6c in rpcsvc_submit_generic (req=0x2aaaaad0903c, proghdr=0x7fffc4fbbde0, hdrcount=0, payload=0x0, payloadcount=0, iobref=0x5d1c038) at rpcsvc.c:1539 #1 0x00002b42d97162b1 in rpcsvc_error_reply (req=0x2b42d971dbfa) at rpcsvc.c:1577 #2 0x00002b42d97167b4 in rpcsvc_handle_rpc_call (svc=0x5cec2d8, trans=<value optimized out>, msg=0x5d1c778) at rpcsvc.c:998 #3 0x00002b42d9716a5b in rpcsvc_notify (trans=0x5d1b0b8, mydata=0x2b42d971dbfa, event=<value optimized out>, data=0x5d1c778) at rpcsvc.c:1088 #4 0x00002b42d971778c in rpc_transport_notify (this=0x2b42d971dbfa, event=3648120344, data=0x606) at rpc-transport.c:1142 #5 0x00002aaaaad9e36f in socket_event_poll_in (this=0x5d1b0b8) at socket.c:1619 #6 0x00002aaaaad9e4f8 in socket_event_handler (fd=<value optimized out>, idx=6, data=0x5d1b0b8, poll_in=1, poll_out=0, poll_err=0) at socket.c:1733 #7 0x00002b42d94ddee7 in event_dispatch_epoll_handler (event_pool=0x5ce5198) at event.c:812 #8 event_dispatch_epoll (event_pool=0x5ce5198) at event.c:876 #9 0x00000000004046ed in main (argc=1, argv=0x7fffc4fbc608) at glusterfsd.c:1410 (gdb) info thr 4 Thread 7686 0x000000391d40e838 in do_sigwait () from /lib64/libpthread.so.0 3 Thread 7687 0x000000391d40d91b in read () from /lib64/libpthread.so.0 2 Thread 7688 0x000000391d40d91b in read () from /lib64/libpthread.so.0 * 1 Thread 7685 0x00002b42d9715f6c in rpcsvc_submit_generic (req=0x2aaaaad0903c, proghdr=0x7fffc4fbbde0, hdrcount=0, payload=0x0, payloadcount=0, iobref=0x5d1c038) at rpcsvc.c:1539 (gdb) f -0 #0 0x00002b42d9715f6c in rpcsvc_submit_generic (req=0x2aaaaad0903c, proghdr=0x7fffc4fbbde0, hdrcount=0, payload=0x0, payloadcount=0, iobref=0x5d1c038) at rpcsvc.c:1539 1539 gf_log (GF_RPCSVC, GF_LOG_ERROR, "failed to submit message " (gdb) f 0 #0 0x00002b42d9715f6c in rpcsvc_submit_generic (req=0x2aaaaad0903c, proghdr=0x7fffc4fbbde0, hdrcount=0, payload=0x0, payloadcount=0, iobref=0x5d1c038) at rpcsvc.c:1539 1539 gf_log (GF_RPCSVC, GF_LOG_ERROR, "failed to submit message " (gdb) l 1534 ret = rpcsvc_transport_submit (trans, &recordhdr, 1, proghdr, hdrcount, 1535 payload, payloadcount, iobref, 1536 req->trans_private); 1537 1538 if (ret == -1) { 1539 gf_log (GF_RPCSVC, GF_LOG_ERROR, "failed to submit message " 1540 "(XID: 0x%lx, Program: %s, ProgVers: %d, Proc: %d) to " 1541 "rpc-transport (%s)", req->xid, req->prog->progname, 1542 req->prog->progver, req->procnum, trans->name); 1543 } else { (gdb) p *req $1 = {trans = 0x5d1b0b8, svc = 0x5cec2d8, prog = 0x0, xid = 1, prognum = 1238463, progver = 1, procnum = 5, type = 0, uid = 0, gid = 0, pid = 0, lk_owner = 0, gfs_id = 0, auxgids = {0 <repeats 16 times>}, auxgidcount = 0, msg = {{iov_base = 0x2b42da329080, iov_len = 12}, {iov_base = 0x0, iov_len = 0}}, count = 1, iobref = 0x5d1b9d8, rpc_status = 0, rpc_err = 1, auth_err = 0, txlist = {next = 0x2aaaaad0910c, prev = 0x2aaaaad0910c}, payloadsize = 0, cred = {flavour = 5, datalen = 88, authdata = '\000' <repeats 399 times>}, verf = {flavour = 0, datalen = 0, authdata = '\000' <repeats 399 times>}, private = 0x0, trans_private = 0x0} (gdb) I have archived the core file and logs.
This looks like a duplicate of 2481. This issue must not happen in versions >= 3.1.1. Closing this now, please re-open if it happens in the more recent versions of 3.1 release. *** This bug has been marked as a duplicate of bug 2481 ***