Bug 762186 (GLUSTER-454)

I observed atleast two crashes in protocol/client in client_readdirp_cbk and client_lookup_cbk. Both of these were crashes were due to frame being passed as argument already freed.

The guess I am doing is that either in replicate or distribute, STACK_DESTROY was called directly or indirectly through STACK_UNWIND on frame->root.

I've attached the logfiles for crash in client_lookup_cbk.

Here is the backtrace of crash in client_lookup_cbk.

#0  0xb7ebee48 in gf_print_trace (signum=11) at ../../../libglusterfs/src/common-utils.c:413
        tmp = (call_frame_t *) 0xb02774cc
        ctx = (glusterfs_ctx_t *) 0x8051008
        trav = (struct list_head *) 0xb0277458
        tm = (struct tm *) 0x0
        msg = "frame : type(1) op(WRITE)\n", '\0' <repeats 997 times>
        timestr = '\0' <repeats 255 times>
        utime = 0
        ret = 26
        fd = 4
#1  <signal handler called>
No symbol table info available.
#2  0xb7e8f276 in pthread_spin_lock () from /lib/libpthread.so.0
No symbol table info available.
#3  0xb7ec5524 in inode_ctx_get2 (inode=0xa9, xlator=0x58a68b7, value1=0xad1ff158, value2=0xad1ff150)
    at ../../../libglusterfs/src/inode.c:1213
        ret = 0
#4  0xb74a3495 in client_lookup_cbk (frame=0xaa0d8287, hdr=0x8309188, hdrlen=504, iobuf=0x0)
    at ../../../../../xlators/protocol/client/src/client-protocol.c:4897
        stbuf = {st_dev = 5412414050074099713, __pad1 = 0, __st_ino = 0, st_mode = 16877, st_nlink = 3, 
  st_uid = 0, st_gid = 0, st_rdev = 0, __pad2 = 0, st_size = 4096, st_blksize = 4096, st_blocks = 16, st_atim = {
    tv_sec = 1260236810, tv_nsec = 0}, st_mtim = {tv_sec = 1260176040, tv_nsec = 0}, st_ctim = {
    tv_sec = 1260236804, tv_nsec = 0}, st_ino = 1}
        postparent = {st_dev = 0, __pad1 = 0, __st_ino = 0, st_mode = 0, st_nlink = 0, st_uid = 0, st_gid = 0, 
  st_rdev = 0, __pad2 = 0, st_size = 0, st_blksize = 0, st_blocks = 0, st_atim = {tv_sec = 0, tv_nsec = 0}, 
  st_mtim = {tv_sec = 0, tv_nsec = 0}, st_ctim = {tv_sec = 0, tv_nsec = 0}, st_ino = 0}
        inode = (inode_t *) 0xa9
        xattr = (dict_t *) 0x0
        rsp = (gf_fop_lookup_rsp_t *) 0x83091f4
        op_ret = -1
        op_errno = 0
        dict_len = 0
        dictbuf = 0x0
        ret = -1
        gf_errno = 0
        local = (client_local_t *) 0xa9d070b7
        oldino = 0
        oldgen = 0
        __FUNCTION__ = "client_lookup_cbk"
#5  0xb74a79d2 in protocol_client_interpret (this=0x8058a68, trans=0x80941b8, hdr_p=0x8309188 "", hdrlen=504, 
    iobuf=0x0) at ../../../../../xlators/protocol/client/src/client-protocol.c:6507
        ret = -1
        frame = (call_frame_t *) 0xaa0d8287
        hdr = (gf_hdr_common_t *) 0x8309188
        callid = 236320
        type = 4
        op = 27
        conn = (client_connection_t *) 0x80944c0
        __FUNCTION__ = "protocol_client_interpret"
#6  0xb74a85ee in protocol_client_pollin (this=0x8058a68, trans=0x80941b8)
    at ../../../../../xlators/protocol/client/src/client-protocol.c:6805
        conf = (client_conf_t *) 0x8093ce0
        ret = 0
        iobuf = (struct iobuf *) 0x0
        hdr = 0x8309188 ""
        hdrlen = 504
#7  0xb74a8abd in notify (this=0x8058a68, event=2, data=0x80941b8)
    at ../../../../../xlators/protocol/client/src/client-protocol.c:6924
        i = 0
        ret = -1
        child_down = 1
        was_not_down = 0
        trans = (transport_t *) 0x80941b8
        conn = (client_connection_t *) 0x0
        conf = (client_conf_t *) 0x8093ce0
        parent = (xlator_list_t *) 0x0
        __FUNCTION__ = "notify"
#8  0xb7eb5720 in xlator_notify (xl=0x8058a68, event=2, data=0x80941b8) at ../../../libglusterfs/src/xlator.c:923
        old_THIS = (xlator_t *) 0xb7ee7ba0
        ret = 0
#9  0xb7ec2230 in transport_peerproc (trans_data=0x80941b8) at ../../../libglusterfs/src/transport.c:414
        trans = (transport_t *) 0x80941b8
        msg = (struct transport_msg *) 0x8174d70
#10 0xb7e8a383 in start_thread () from /lib/libpthread.so.0
No symbol table info available.
#11 0xb7e0f05e in clone () from /lib/libc.so.6
No symbol table info available.
(gdb) p *frame
$1 = {root = 0xab029000, parent = 0x66a58b7, next = 0x66a58aa, prev = 0xa6dac0aa, local = 0x0, this = 0x58a68b7, 
  ret = 0x44dd5408, ref_count = 183, lock = 256, cookie = 0x0, complete = _gf_false}
(gdb) p *frame->this
Cannot access memory at address 0x58a68b7


And here is the backtrace of crash in client_readdirp_cbk.

#0  0xb7553208 in client_readdirp_cbk (frame=0xa89072a7, hdr=0xa89c01e0, hdrlen=1316, iobuf=0x0)
    at ../../../../../xlators/protocol/client/src/client-protocol.c:4461
        fn = (ret_fn_t) 0x5093b008
        _parent = (call_frame_t *) 0x994668a8
        old_THIS = (xlator_t *) 0x0
        rsp = (gf_fop_readdirp_rsp_t *) 0xa89c024c
        op_ret = 11
        op_errno = 0
        buf_size = 1204
        entries = {{list = {next = 0xa8702268, prev = 0xa8715bc0}, {next = 0xa8702268, prev = 0xa8715bc0}}, 
  d_ino = 12773734187154396736, d_off = 7486, d_len = 2828271320, d_type = 3075854596, d_stat = {
    st_dev = 12147639625727274734, __pad1 = 22120, __st_ino = 3086215132, st_mode = 2974114408, 
    st_nlink = 3075854656, st_uid = 1, st_gid = 2974116752, st_rdev = 13210695157743444568, __pad2 = 0, 
    st_size = -5673020022532510832, st_blksize = -1219209821, st_blocks = 134829584, st_atim = {tv_sec = 0, 
      tv_nsec = -1219209931}, st_mtim = {tv_sec = 7486, tv_nsec = 0}, st_ctim = {tv_sec = 0, tv_nsec = 4}, 
    st_ino = 0}, d_name = 0xb1456248 "L\002\234¨\v"}
#1  0xb75589d2 in protocol_client_interpret (this=0x80580f8, trans=0x8095308, hdr_p=0xa89c01e0 "", hdrlen=1316, 
    iobuf=0x0) at ../../../../../xlators/protocol/client/src/client-protocol.c:6507
        ret = -1
        frame = (call_frame_t *) 0xa89072a7
        hdr = (gf_hdr_common_t *) 0xa89c01e0
        callid = 7486
        type = 4
        op = 44
        conn = (client_connection_t *) 0x8095610
        __FUNCTION__ = "protocol_client_interpret"
#2  0xb75595ee in protocol_client_pollin (this=0x80580f8, trans=0x8095308)
    at ../../../../../xlators/protocol/client/src/client-protocol.c:6805
        conf = (client_conf_t *) 0x8094e30
        ret = 0
        iobuf = (struct iobuf *) 0x0
        hdr = 0xa89c01e0 ""
        hdrlen = 1316
#3  0xb7559abd in notify (this=0x80580f8, event=2, data=0x8095308)
    at ../../../../../xlators/protocol/client/src/client-protocol.c:6924
        i = 0
        ret = -1
        child_down = 1
        was_not_down = 0
        trans = (transport_t *) 0x8095308
        conn = (client_connection_t *) 0x0
        conf = (client_conf_t *) 0x8094e30
        parent = (xlator_list_t *) 0x0
        __FUNCTION__ = "notify"
#4  0xb7f66720 in xlator_notify (xl=0x80580f8, event=2, data=0x8095308) at ../../../libglusterfs/src/xlator.c:923
        old_THIS = (xlator_t *) 0xb7f98ba0
        ret = 0
#5  0xb7f73230 in transport_peerproc (trans_data=0x8095308) at ../../../libglusterfs/src/transport.c:414
        trans = (transport_t *) 0x8095308
        msg = (struct transport_msg *) 0xa8989920
#6  0xb7f3b383 in start_thread () from /lib/libpthread.so.0
No symbol table info available.
#7  0xb7ec005e in clone () from /lib/libc.so.6
No symbol table info available.

(gdb) p *frame
$1 = {root = 0x99018000, parent = 0x994668a8, next = 0x994668a8, prev = 0x9901f4a8, local = 0xa8, 
  this = 0x580f800, ret = 0x5093b008, ref_count = 183, lock = 256, cookie = 0x0, complete = _gf_false}
(gdb) p *frame->this
Cannot access memory at address 0x580f800


I was running,
 while true; do dbench -t 100 10; done

on the mount point.

This bug is no longer reproducible on latest git pull. Please reopen if the bug is found again.