Bug 763513 (GLUSTER-1781) - Crash in lookup
Summary: Crash in lookup
Keywords:
Status: CLOSED CURRENTRELEASE
Alias: GLUSTER-1781
Product: GlusterFS
Classification: Community
Component: core
Version: mainline
Hardware: All
OS: Linux
low
high
Target Milestone: ---
Assignee: Amar Tumballi
QA Contact:
URL:
Whiteboard:
Depends On:
Blocks:
TreeView+ depends on / blocked
 
Reported: 2010-10-03 06:56 UTC by Pavan Vilas Sondur
Modified: 2015-12-01 16:45 UTC (History)
2 users (show)

Fixed In Version:
Doc Type: Bug Fix
Doc Text:
Clone Of:
Environment:
Last Closed:
Regression: ---
Mount Type: ---
Documentation: ---
CRM:
Verified Versions:


Attachments (Terms of Use)

Description Pavan Vilas Sondur 2010-10-03 06:56:59 UTC
gdb) bt full
#0  memcpy () at ../sysdeps/x86_64/memcpy.S:191
No locals.
#1  0x00007fcd9d88ac6d in xdrmem_putbytes (xdrs=0x7fffdd2e1f30, addr=0x0, len=<value optimized out>)
    at xdr_mem.c:152
No locals.
#2  0x00007fcd9d88a398 in xdr_opaque (xdrs=0x7fffdd2e1f30, cp=0x0, cnt=50) at xdr.c:501
        rndup = 2
        crud = "\000\000\000"
#3  0x00007fcd9df155ca in xdr_gfs3_lookup_rsp (xdrs=0x7fffdd2e1f30, objp=0x7fffdd2e2180)
    at ../../../../rpc/xdr/src/glusterfs3-xdr.c:640
No locals.
#4  0x00007fcd9e1397b1 in xdr_serialize_generic (outmsg=..., res=0x7fffdd2e2180, 
    proc=0x7fcd9df15508 <xdr_gfs3_lookup_req+237>) at ../../../../rpc/rpc-lib/src/rpc-common.c:36
        ret = -1
        xdr = {x_op = XDR_ENCODE, x_ops = 0x7fcd9dae2860, x_public = 0x7c30 <Address 0x7c30 out of bounds>, 
          x_private = 0x7fcd9d6fc0d4 "", x_base = 0x7fcd9d6fc000 "\377\377\377\377", x_handy = 130810}
#5  0x00007fcd9df17c4f in xdr_serialize_lookup_rsp (outmsg=..., rsp=0x7fffdd2e2180)
    at ../../../../rpc/xdr/src/glusterfs3.c:38
No locals.
#6  0x00007fcd9bbdeb2e in gfs_serialize_reply (req=0x7fcd9e71b03c, arg=0x7fffdd2e2180, 
    sfunc=0x7fcd9df17c13 <xdr_serialize_getspec_rsp+35>, outmsg=0x7fffdd2e2050)
    at ../../../../../xlators/protocol/server/src/server.c:61
        iob = 0xffca58
        retlen = -1
        __FUNCTION__ = "gfs_serialize_reply"
#7  0x00007fcd9bbdecde in server_submit_reply (frame=0x7fcd9cae10c4, req=0x7fcd9e71b03c, arg=0x7fffdd2e2180, 
    payload=0x0, payloadcount=0, iobref=0x100cb98, sfunc=0x7fcd9df17c13 <xdr_serialize_getspec_rsp+35>)
    at ../../../../../xlators/protocol/server/src/server.c:114
        iob = 0x0
        ret = -1
        rsp = {iov_base = 0x7fcd9d6fc000, iov_len = 131072}
        state = 0x100a0d8
        new_iobref = 1 '\001'
        __FUNCTION__ = "server_submit_reply"
#8  0x00007fcd9bbe820f in server_lookup_cbk (frame=0x7fcd9cae10c4, cookie=0x7fcd9cd0a144, this=0x1004058, 
    op_ret=-1, op_errno=61, inode=0x7fcd9914c2ac, stbuf=0x7fffdd2e2440, dict=0x100ca38, 
    postparent=0x7fffdd2e23d0) at ../../../../../xlators/protocol/server/src/server3_1-fops.c:163
        req = 0x7fcd9e71b03c
        state = 0x100a0d8
        root_inode = 0x0
---Type <return> to continue, or q <return> to quit---
        link_inode = 0x0
        fresh_loc = {path = 0x0, name = 0x0, ino = 0, inode = 0x0, parent = 0x0}
        rsp = {op_ret = -1, op_errno = 61, stat = {ia_gfid = '\000' <repeats 15 times>, ia_ino = 0, ia_dev = 0, 
            mode = 0, ia_nlink = 0, ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 0, ia_blksize = 0, 
            ia_blocks = 0, ia_atime = 0, ia_atime_nsec = 0, ia_mtime = 0, ia_mtime_nsec = 0, ia_ctime = 0, 
            ia_ctime_nsec = 0}, postparent = {ia_gfid = '\000' <repeats 15 times>, ia_ino = 4456449, 
            ia_dev = 2049, mode = 16877, ia_nlink = 22, ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 4096, 
            ia_blksize = 4096, ia_blocks = 8, ia_atime = 1286027394, ia_atime_nsec = 0, ia_mtime = 1286027398, 
            ia_mtime_nsec = 0, ia_ctime = 1286027398, ia_ctime_nsec = 0}, dict = {dict_len = 50, 
            dict_val = 0x0}}
        ret = -1
        __FUNCTION__ = "server_lookup_cbk"
#9  0x00007fcd9be10ed6 in pl_lookup_cbk (frame=0x7fcd9cd0a144, cookie=0x7fcd9cd0a1c8, this=0x1002e68, 
    op_ret=-1, op_errno=61, inode=0x7fcd9914c2ac, buf=0x7fffdd2e2440, dict=0x100ca38, postparent=0x7fffdd2e23d0)
    at ../../../../../xlators/features/locks/src/posix.c:1385
        fn = 0x7fcd9bbe7a13 <server_lookup_cbk>
        _parent = 0x7fcd9cae10c4
        old_THIS = 0x1002e68
        local = 0x0
        __FUNCTION__ = "pl_lookup_cbk"
#10 0x00007fcd9c0235df in posix_lookup (frame=0x7fcd9cd0a1c8, this=0x1001a38, loc=0x7fffdd2e25f0, 
    xattr_req=0x100a688) at ../../../../../xlators/storage/posix/src/posix.c:474
        fn = 0x7fcd9be10ce3 <pl_posixlk_xattr_fill+164>
        _parent = 0x7fcd9cd0a144
        old_THIS = 0x1001a38
        buf = {ia_ino = 4481044, ia_gfid = '\000' <repeats 15 times>, "\001", ia_dev = 2049, 
          ia_type = IA_IFDIR, ia_prot = {suid = 0 '\000', sgid = 0 '\000', sticky = 0 '\000', owner = {
              read = 1 '\001', write = 1 '\001', exec = 1 '\001'}, group = {read = 1 '\001', write = 0 '\000', 
              exec = 1 '\001'}, other = {read = 1 '\001', write = 0 '\000', exec = 1 '\001'}}, ia_nlink = 4, 
          ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 4096, ia_blksize = 4096, ia_blocks = 16, 
          ia_atime = 1286027413, ia_atime_nsec = 0, ia_mtime = 1286022903, ia_mtime_nsec = 0, 
          ia_ctime = 1286027413, ia_ctime_nsec = 0}
        real_path = 0x7fffdd2e2340 "/exports/exp5/"
        op_ret = -1
        entry_ret = 0
        op_errno = 61
        xattr = 0x100ca38
        pathdup = 0x100cb58 ""
        parentpath = 0x100cb58 ""
        postparent = {ia_ino = 4456449, ia_gfid = '\000' <repeats 15 times>, ia_dev = 2049, ia_type = IA_IFDIR, 
---Type <return> to continue, or q <return> to quit---
          ia_prot = {suid = 0 '\000', sgid = 0 '\000', sticky = 0 '\000', owner = {read = 1 '\001', 
              write = 1 '\001', exec = 1 '\001'}, group = {read = 1 '\001', write = 0 '\000', exec = 1 '\001'}, 
            other = {read = 1 '\001', write = 0 '\000', exec = 1 '\001'}}, ia_nlink = 22, ia_uid = 0, 
          ia_gid = 0, ia_rdev = 0, ia_size = 4096, ia_blksize = 4096, ia_blocks = 8, ia_atime = 1286027394, 
          ia_atime_nsec = 0, ia_mtime = 1286027398, ia_mtime_nsec = 0, ia_ctime = 1286027398, ia_ctime_nsec = 0}
        __FUNCTION__ = "posix_lookup"
#11 0x00007fcd9be11342 in pl_lookup (frame=0x7fcd9cd0a144, this=0x1002e68, loc=0x7fffdd2e25f0, 
    xattr_req=0x100a688) at ../../../../../xlators/features/locks/src/posix.c:1427
        _new = 0x7fcd9cd0a1c8
        old_THIS = 0x1002e68
        tmp_cbk = 0x7fcd9be10ce3 <pl_posixlk_xattr_fill+164>
        local = 0x100ca08
        ret = -1
        __FUNCTION__ = "pl_lookup"
#12 0x00007fcd9bbe7de9 in server_lookup_cbk (frame=0x7fcd9cae10c4, cookie=0x7fcd9cd0a03c, this=0x1004058, 
    op_ret=-1, op_errno=61, inode=0x7fcd9914c03c, stbuf=0x7fffdd2e2920, dict=0x100c758, 
    postparent=0x7fffdd2e28b0) at ../../../../../xlators/protocol/server/src/server3_1-fops.c:83
        _new = 0x7fcd9cd0a144
        old_THIS = 0x1004058
        tmp_cbk = 0x7fcd9bbe7a13 <server_lookup_cbk>
        req = 0x7fcd9e71b03c
        state = 0x100a0d8
        root_inode = 0x0
        link_inode = 0x0
        fresh_loc = {path = 0x100c9d8 "/", name = 0x100c9d9 "", ino = 0, inode = 0x7fcd9914c2ac, 
          parent = 0x7fcd9914c03c}
        rsp = {op_ret = 0, op_errno = 0, stat = {ia_gfid = '\000' <repeats 15 times>, ia_ino = 0, ia_dev = 0, 
            mode = 0, ia_nlink = 0, ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 0, ia_blksize = 0, 
            ia_blocks = 0, ia_atime = 0, ia_atime_nsec = 0, ia_mtime = 0, ia_mtime_nsec = 0, ia_ctime = 0, 
            ia_ctime_nsec = 0}, postparent = {ia_gfid = '\000' <repeats 15 times>, ia_ino = 0, ia_dev = 0, 
            mode = 0, ia_nlink = 0, ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 0, ia_blksize = 0, 
            ia_blocks = 0, ia_atime = 0, ia_atime_nsec = 0, ia_mtime = 0, ia_mtime_nsec = 0, ia_ctime = 0, 
            ia_ctime_nsec = 0}, dict = {dict_len = 0, dict_val = 0x0}}
        ret = -1
        __FUNCTION__ = "server_lookup_cbk"
#13 0x00007fcd9be10ed6 in pl_lookup_cbk (frame=0x7fcd9cd0a03c, cookie=0x7fcd9cd0a0c0, this=0x1002e68, 
    op_ret=-1, op_errno=61, inode=0x7fcd9914c03c, buf=0x7fffdd2e2920, dict=0x100c758, postparent=0x7fffdd2e28b0)
    at ../../../../../xlators/features/locks/src/posix.c:1385
        fn = 0x7fcd9bbe7a13 <server_lookup_cbk>
        _parent = 0x7fcd9cae10c4
---Type <return> to continue, or q <return> to quit---
        old_THIS = 0x1002e68
        local = 0x0
        __FUNCTION__ = "pl_lookup_cbk"
#14 0x00007fcd9c0235df in posix_lookup (frame=0x7fcd9cd0a0c0, this=0x1001a38, loc=0x100a0f8, 
    xattr_req=0x100a688) at ../../../../../xlators/storage/posix/src/posix.c:474
        fn = 0x7fcd9be10ce3 <pl_posixlk_xattr_fill+164>
        _parent = 0x7fcd9cd0a03c
        old_THIS = 0x1001a38
        buf = {ia_ino = 4481044, ia_gfid = '\000' <repeats 15 times>, "\001", ia_dev = 2049, 
          ia_type = IA_IFDIR, ia_prot = {suid = 0 '\000', sgid = 0 '\000', sticky = 0 '\000', owner = {
              read = 1 '\001', write = 1 '\001', exec = 1 '\001'}, group = {read = 1 '\001', write = 0 '\000', 
              exec = 1 '\001'}, other = {read = 1 '\001', write = 0 '\000', exec = 1 '\001'}}, ia_nlink = 4, 
          ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 4096, ia_blksize = 4096, ia_blocks = 16, 
          ia_atime = 1286027413, ia_atime_nsec = 0, ia_mtime = 1286022903, ia_mtime_nsec = 0, 
          ia_ctime = 1286027413, ia_ctime_nsec = 0}
        real_path = 0x7fffdd2e2820 "/exports/exp5/"
        op_ret = -1
        entry_ret = 0
        op_errno = 61
        xattr = 0x100c758
        pathdup = 0x100c8b8 ""
        parentpath = 0x100c8b8 ""
        postparent = {ia_ino = 4456449, ia_gfid = '\000' <repeats 15 times>, ia_dev = 2049, ia_type = IA_IFDIR, 
          ia_prot = {suid = 0 '\000', sgid = 0 '\000', sticky = 0 '\000', owner = {read = 1 '\001', 
              write = 1 '\001', exec = 1 '\001'}, group = {read = 1 '\001', write = 0 '\000', exec = 1 '\001'}, 
            other = {read = 1 '\001', write = 0 '\000', exec = 1 '\001'}}, ia_nlink = 22, ia_uid = 0, 
          ia_gid = 0, ia_rdev = 0, ia_size = 4096, ia_blksize = 4096, ia_blocks = 8, ia_atime = 1286027394, 
          ia_atime_nsec = 0, ia_mtime = 1286027398, ia_mtime_nsec = 0, ia_ctime = 1286027398, ia_ctime_nsec = 0}
        __FUNCTION__ = "posix_lookup"
#15 0x00007fcd9be11342 in pl_lookup (frame=0x7fcd9cd0a03c, this=0x1002e68, loc=0x100a0f8, xattr_req=0x100a688)
    at ../../../../../xlators/features/locks/src/posix.c:1427
        _new = 0x7fcd9cd0a0c0
        old_THIS = 0x1002e68
        tmp_cbk = 0x7fcd9be10ce3 <pl_posixlk_xattr_fill+164>
        local = 0x100c728
        ret = -1
        __FUNCTION__ = "pl_lookup"
#16 0x00007fcd9bbf352c in server_lookup_resume (frame=0x7fcd9cae10c4, bound_xl=0x1002e68)
    at ../../../../../xlators/protocol/server/src/server3_1-fops.c:2600
        _new = 0x7fcd9cd0a03c
---Type <return> to continue, or q <return> to quit---
        old_THIS = 0x1004058
        tmp_cbk = 0x7fcd9bbe7a13 <server_lookup_cbk>
        state = 0x100a0d8
        __FUNCTION__ = "server_lookup_resume"
#17 0x00007fcd9bbe1bf7 in server_resolve_done (frame=0x7fcd9cae10c4)
    at ../../../../../xlators/protocol/server/src/server-resolve.c:568
        state = 0x100a0d8
        bound_xl = 0x1002e68
#18 0x00007fcd9bbe1cf2 in server_resolve_all (frame=0x7fcd9cae10c4)
    at ../../../../../xlators/protocol/server/src/server-resolve.c:603
        state = 0x100a0d8
        this = 0x1004058
        __FUNCTION__ = "server_resolve_all"
#19 0x00007fcd9bbe1b8b in server_resolve (frame=0x7fcd9cae10c4)
    at ../../../../../xlators/protocol/server/src/server-resolve.c:550
        state = 0x100a0d8
        resolve = 0x100a1e8
#20 0x00007fcd9bbe1cc9 in server_resolve_all (frame=0x7fcd9cae10c4)
    at ../../../../../xlators/protocol/server/src/server-resolve.c:599
        state = 0x100a0d8
        this = 0x1004058
        __FUNCTION__ = "server_resolve_all"
#21 0x00007fcd9bbe19e7 in server_resolve_inode (frame=0x7fcd9cae10c4)
    at ../../../../../xlators/protocol/server/src/server-resolve.c:487
        state = 0x100a0d8
        ret = 0
        loc = 0x100a0f8
#22 0x00007fcd9bbe1b4c in server_resolve (frame=0x7fcd9cae10c4)
    at ../../../../../xlators/protocol/server/src/server-resolve.c:539
        state = 0x100a0d8
        resolve = 0x100a148
#23 0x00007fcd9bbe1c77 in server_resolve_all (frame=0x7fcd9cae10c4)
    at ../../../../../xlators/protocol/server/src/server-resolve.c:592
        state = 0x100a0d8
        this = 0x1004058
        __FUNCTION__ = "server_resolve_all"
#24 0x00007fcd9bbe1d8a in resolve_and_resume (frame=0x7fcd9cae10c4, fn=0x7fcd9bbf330b <server_stat_resume+553>)
    at ../../../../../xlators/protocol/server/src/server-resolve.c:622
        state = 0x100a0d8
#25 0x00007fcd9bbf9a11 in server_lookup (req=0x7fcd9e71b03c)
---Type <return> to continue, or q <return> to quit---
    at ../../../../../xlators/protocol/server/src/server3_1-fops.c:4993
        frame = 0x7fcd9cae10c4
        conn = 0x100afc8
        state = 0x100a0d8
        xattr_req = 0x100a688
        buf = 0x0
        args = {gfid = '\000' <repeats 15 times>, "\001", pargfid = '\000' <repeats 15 times>, flags = 0, 
          path = 0x7fffdd2eacc0 "/", bname = 0x7fffdd2e6cc0 "", dict = {dict_len = 291, 
            dict_val = 0x7fffdd2e2cc0 ""}}
        ret = 0
        path = "/", '\000' <repeats 16382 times>
        bname = '\000' <repeats 16383 times>
        dict_val = "\000\000\000\b\000\000\000\027\000\000\000\002glusterfs.entrylk-count\000\060\000\000\000\000\027\000\000\000\002glusterfs.inodelk-count\000\060\000\000\000\000 \000\000\000\003trusted.afr.dapawz-replace-brick\000\061\062\000\000\000\000\030\000\000\000\003trusted.afr.dapawz-locks\000\061\062\000\000\000\000\b\000\000\000\020gfid-req\000\177hP3\200\260E\217\237\275\365\004\253x\311\"\000\000\000\021\000\000\000\006glusterfs.conte"...
        __FUNCTION__ = "server_lookup"
#26 0x00007fcd9e12e6c3 in rpcsvc_handle_rpc_call (svc=0x10065d8, trans=0x1009ca8, msg=0x100a618)
    at ../../../../rpc/rpc-lib/src/rpcsvc.c:992
        actor = 0x7fcd9be04340
        req = 0x7fcd9e71b03c
        ret = -1
        port = 1014
        __FUNCTION__ = "rpcsvc_handle_rpc_call"
#27 0x00007fcd9e12ea69 in rpcsvc_notify (trans=0x1009ca8, mydata=0x10065d8, event=RPC_TRANSPORT_MSG_RECEIVED, 
    data=0x100a618) at ../../../../rpc/rpc-lib/src/rpcsvc.c:1087
        ret = -1
        msg = 0x100a618
        new_trans = 0x0
        svc = 0x10065d8
        __FUNCTION__ = "rpcsvc_notify"
#28 0x00007fcd9e134bd2 in rpc_transport_notify (this=0x1009ca8, event=RPC_TRANSPORT_MSG_RECEIVED, 
    data=0x100a618) at ../../../../rpc/rpc-lib/src/rpc-transport.c:1134
        ret = -1
#29 0x00007fcd9b7c7d5a in socket_event_poll_in (this=0x1009ca8)
    at ../../../../../rpc/rpc-transport/socket/src/socket.c:1619
        ret = 0
        pollin = 0x100a618
#30 0x00007fcd9b7c810d in socket_event_handler (fd=6, idx=1, data=0x1009ca8, poll_in=1, poll_out=0, poll_err=0)
---Type <return> to continue, or q <return> to quit---
    at ../../../../../rpc/rpc-transport/socket/src/socket.c:1733
        this = 0x1009ca8
        priv = 0xffd4c8
        ret = 0
        __FUNCTION__ = "socket_event_handler"
#31 0x00007fcd9e387c7f in event_dispatch_epoll_handler (event_pool=0xffbd88, events=0x1009078, i=0)
    at ../../../libglusterfs/src/event.c:785
        event_data = 0x100907c
        handler = 0x7fcd9b7c8007 <socket_event_handler>
        data = 0x1009ca8
        idx = 1
        ret = -1
        __FUNCTION__ = "event_dispatch_epoll_handler"
#32 0x00007fcd9e387e8f in event_dispatch_epoll (event_pool=0xffbd88) at ../../../libglusterfs/src/event.c:830
        events = 0x1009078
        size = 1
        i = 0
        ret = 1
        __FUNCTION__ = "event_dispatch_epoll"
#33 0x00007fcd9e3881f7 in event_unregister (event_pool=0x7fcd9e3881f7, fd=32767, idx=-584126208)
    at ../../../libglusterfs/src/event.c:944
        ret = 0
        __FUNCTION__ = "event_unregister"
#34 0x00000000004066fc in main (argc=5, argv=0x7fffdd2ef218) at ../../../glusterfsd/src/glusterfsd.c:141

Comment 1 Amar Tumballi 2010-10-04 01:23:47 UTC
Backtrace is bit confusing.

In xdr_gfs3_lookup_rsp(), there is no xdr_opaque() call. need to check it thoroughly.

Comment 2 Amar Tumballi 2010-10-05 06:40:58 UTC
http://patches.gluster.com/patch/5251/ should fix this issue.


Note You need to log in before you can comment on or make changes to this bug.