Bug 999356

Description of problem:
Crashes upon an ls -l of the filesystem on el5 i386 machines (I have no el5 x86_64 to compare against so I don't know if it's 32 bit or el5)

Version-Release number of selected component (if applicable):
3.4.0-8

How reproducible:
always

Steps to Reproduce:
1. Mount a volume
2. ls -l

Actual results:
(gdb) thread apply all bt full

Thread 6 (Thread 0xb2409b90 (LWP 16122)):
#0  0x00d75402 in __kernel_vsyscall ()
No symbol table info available.
#1  0x0026a95e in readv () from /lib/libc.so.6
No symbol table info available.
#2  0x006a6e36 in fuse_thread_proc (data=0x8089eb8) at fuse-bridge.c:4474
        mount_point = <value optimized out>
        priv = 0x808aae0
        res = 42
        iobuf = 0x8089400
        finh = 0x809d2b0
        iov_in = {{iov_base = 0x8235128, iov_len = 80}, {iov_base = 0xb7ad6000, iov_len = 131072}}
        msg = <value optimized out>
        fuse_ops = 0x6bb460
        pfd = {{fd = 9, events = 25, revents = 1}, {fd = 11, events = 25, revents = 1}}
        mount_finished = _gf_true
        __FUNCTION__ = "fuse_thread_proc"
#3  0x004bd912 in start_thread () from /lib/libpthread.so.0
No symbol table info available.
#4  0x0027260e in clone () from /lib/libc.so.6
No symbol table info available.

Thread 5 (Thread 0xb5033b90 (LWP 16077)):
#0  0x00d75402 in __kernel_vsyscall ()
No symbol table info available.
#1  0x004c5446 in nanosleep () from /lib/libpthread.so.0
No symbol table info available.
#2  0x00135cfa in gf_timer_proc (ctx=0x8079008) at timer.c:173
        now = 1377073384621059
        now_tv = {tv_sec = 1377073384, tv_usec = 621059}
        event = 0x80d53a8
        reg = 0x809b0b8
        sleepts = {tv_sec = 1, tv_nsec = 0}
        __FUNCTION__ = "gf_timer_proc"
#3  0x004bd912 in start_thread () from /lib/libpthread.so.0
No symbol table info available.
#4  0x0027260e in clone () from /lib/libc.so.6
No symbol table info available.

Thread 4 (Thread 0xb5a69b90 (LWP 16076)):
#0  0x00d75402 in __kernel_vsyscall ()
No symbol table info available.
#1  0x004c1ff2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
No symbol table info available.
#2  0x0015813a in syncenv_task (proc=0x8091668) at syncop.c:306
        env = 0x8091500
        task = <value optimized out>
        sleep_till = {tv_sec = 1377073795, tv_nsec = 0}
        ret = <value optimized out>
#3  0x0015a8d8 in syncenv_processor (thdata=0x8091668) at syncop.c:384
        env = 0x8091500
        task = <value optimized out>
#4  0x004bd912 in start_thread () from /lib/libpthread.so.0
No symbol table info available.
#5  0x0027260e in clone () from /lib/libc.so.6
No symbol table info available.

Thread 3 (Thread 0xb646ab90 (LWP 16074)):
#0  0x00d75402 in __kernel_vsyscall ()
No symbol table info available.
#1  0x004c1ff2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
No symbol table info available.
#2  0x0015813a in syncenv_task (proc=0x8091500) at syncop.c:306
        env = 0x8091500
        task = <value optimized out>
        sleep_till = {tv_sec = 1377073795, tv_nsec = 0}
        ret = <value optimized out>
#3  0x0015a8d8 in syncenv_processor (thdata=0x8091500) at syncop.c:384
        env = 0x8091500
        task = <value optimized out>
#4  0x004bd912 in start_thread () from /lib/libpthread.so.0
No symbol table info available.
#5  0x0027260e in clone () from /lib/libc.so.6
No symbol table info available.

Thread 2 (Thread 0xb6e6bb90 (LWP 16072)):
#0  0x00d75402 in __kernel_vsyscall ()
No symbol table info available.
#1  0x004c5d9e in do_sigwait () from /lib/libpthread.so.0
No symbol table info available.
#2  0x004c5e3f in sigwait () from /lib/libpthread.so.0
No symbol table info available.
#3  0x0804c3e0 in glusterfs_sigwaiter (arg=0xbfffd670) at glusterfsd.c:1604
        set = {__val = {18947, 0 <repeats 31 times>}}
        ret = -4
        sig = 0
#4  0x004bd912 in start_thread () from /lib/libpthread.so.0
No symbol table info available.
#5  0x0027260e in clone () from /lib/libc.so.6
No symbol table info available.

Thread 1 (Thread 0xb7ff68d0 (LWP 16067)):
#0  0x0011e25c in __glusterfs_this_location@plt () from /usr/lib/libglusterfs.so.0
No symbol table info available.
#1  0x00136b24 in inode_invalidate (inode=0xb25ec8b8) at inode.c:988
        ret = 0
        xl = 0x80af1e0
        old_THIS = 0x80b0d38
        __FUNCTION__ = "inode_invalidate"
#2  0x004ae47b in mdc_inode_iatt_set_validate (this=0x80b0d38, inode=0xb25ec8b8, prebuf=0x0, iatt=0xb358ba34) at md-cache.c:412
        ret = <value optimized out>
        mdc = 0xb1901a08
#3  0x004aeaa3 in mdc_inode_iatt_set (this=0x80b0d38, inode=0xb25ec8b8, iatt=0xb358ba34) at md-cache.c:427
No locals.
#4  0x004afce3 in mdc_lookup_cbk (frame=0xb72f4858, cookie=0xb72f4138, this=0x80b0d38, op_ret=0, op_errno=0, inode=0xb25ec8b8, stbuf=0xb358ba34, dict=0xb6f8e9c8, postparent=0xb358ba9c) at md-cache.c:708
        __FUNCTION__ = "mdc_lookup_cbk"
#5  0x00417f72 in qr_lookup_cbk (frame=0xb72f4138, cookie=0xb72f4198, this=0x80aff88, op_ret=0, op_errno=0, inode_ret=0xb25ec8b8, buf=0xb358ba34, xdata=0xb6f8e9c8, postparent=0xb358ba9c) at quick-read.c:438
        _parent = 0xb72f4858
        old_THIS = 0x80aff88
        content = 0xb1901808
        qr_inode = <value optimized out>
        inode = <value optimized out>
        __FUNCTION__ = "qr_lookup_cbk"
#6  0x004a0e8b in ioc_lookup_cbk (frame=0xb72f4198, cookie=0xb72f41f8, this=0x80af1e0, op_ret=0, op_errno=0, inode=0xb25ec8b8, stbuf=0xb358ba34, xdata=0xb6f8e9c8, postparent=0xb358ba9c) at io-cache.c:255
        _parent = 0xb72f4138
        old_THIS = 0x80af1e0
        ioc_inode = <value optimized out>
        tmp_ioc_inode = 0
        weight = <value optimized out>
        path = 0x81cf408 "\250\214 \b"
        local = <value optimized out>
        __FUNCTION__ = "ioc_lookup_cbk"
#7  0x00dbb1cd in dht_lookup_cbk (frame=0xb72f41f8, cookie=0xb72f4018, this=0x80ac7f8, op_ret=0, op_errno=0, inode=0xb25ec8b8, stbuf=0xb358ba34, xattr=0xb6f8e9c8, postparent=0xb358ba9c) at dht-common.c:1343
        _parent = 0xb72f4198
        old_THIS = 0x80ac7f8
        __xl = 0x80ac7f8
        subvol = <value optimized out>
        local = 0xb456169c
        loc = 0xb45616a0
        ret = <value optimized out>
        parent_layout = 0x0
        __FUNCTION__ = "dht_lookup_cbk"
#8  0x00484d95 in afr_lookup_done (frame=0xb72f4018, cookie=0x2, this=0x80a9d08, op_ret=0, op_errno=0, inode=0xb25ec8b8, buf=0xbfffd4ec, xattr=0xb6f8e9c8, postparent=0xbfffd484) at afr-common.c:1977
        local = 0xb358afc0
        ret = <value optimized out>
        fail_conflict = <value optimized out>
        gfid_miss_count = <value optimized out>
        __FUNCTION__ = "afr_lookup_done"
#9  afr_lookup_cbk (frame=0xb72f4018, cookie=0x2, this=0x80a9d08, op_ret=0, op_errno=0, inode=0xb25ec8b8, buf=0xbfffd4ec, xattr=0xb6f8e9c8, postparent=0xbfffd484) at afr-common.c:2208
        local = 0xb358afc0
        call_count = <value optimized out>
#10 0x003e9ba3 in client3_3_lookup_cbk (req=0xb2b0787c, iov=0xb2b0789c, count=1, myframe=0xb72f40d8) at client-rpc-fops.c:2630
        _parent = 0xb72f4018
        old_THIS = 0x80a39a0
        local = 0x81b40ac
        ret = <value optimized out>
        rsp = {op_ret = 0, op_errno = 0, stat = {ia_gfid = "\032OOk<>@\a\201~\"\020$\357\211\r", ia_ino = 9330932930692745485, ia_dev = 64774, mode = 33279, ia_nlink = 1, ia_uid = 0, ia_gid = 3, ia_rdev = 0, ia_size = 358, ia_blksize = 4096, ia_blocks = 1, ia_atime = 1366793782, ia_atime_nsec = 348057000, ia_mtime = 1348770744, ia_mtime_nsec = 171075000, ia_ctime = 1366793782, ia_ctime_nsec = 340093116}, postparent = {ia_gfid = '\000' <repeats 15 times>, "\001", ia_ino = 1, ia_dev = 64774, mode = 16877, ia_nlink = 241, ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 8192, ia_blksize = 4096, ia_blocks = 16, ia_atime = 1368267587, ia_atime_nsec = 0, ia_mtime = 1377044782, ia_mtime_nsec = 648798000, ia_ctime = 1377067006, ia_ctime_nsec = 70871863}, xdata = {xdata_len = 663, xdata_val = 0xb1900f60 ""}}
        stbuf = {ia_ino = 9330932930692745485, ia_gfid = "\032OOk<>@\a\201~\"\020$\357\211\r", ia_dev = 64774, ia_type = IA_IFREG, ia_prot = {suid = 0 '\000', sgid = 0 '\000', sticky = 0 '\000', owner = {read = 1 '\001', write = 1 '\001', exec = 1 '\001'}, group = {read = 1 '\001', write = 1 '\001', exec = 1 '\001'}, other = {read = 1 '\001', write = 1 '\001', exec = 1 '\001'}}, ia_nlink = 1, ia_uid = 0, ia_gid = 3, ia_rdev = 0, ia_size = 358, ia_blksize = 4096, ia_blocks = 1, ia_atime = 1366793782, ia_atime_nsec = 348057000, ia_mtime = 1348770744, ia_mtime_nsec = 171075000, ia_ctime = 1366793782, ia_ctime_nsec = 340093116}
        postparent = {ia_ino = 1, ia_gfid = '\000' <repeats 15 times>, "\001", ia_dev = 64774, ia_type = IA_IFDIR, ia_prot = {suid = 0 '\000', sgid = 0 '\000', sticky = 0 '\000', owner = {read = 1 '\001', write = 1 '\001', exec = 1 '\001'}, group = {read = 1 '\001', write = 0 '\000', exec = 1 '\001'}, other = {read = 1 '\001', write = 0 '\000', exec = 1 '\001'}}, ia_nlink = 241, ia_uid = 0, ia_gid = 0, ia_rdev = 0, ia_size = 8192, ia_blksize = 4096, ia_blocks = 16, ia_atime = 1368267587, ia_atime_nsec = 0, ia_mtime = 1377044782, ia_mtime_nsec = 648798000, ia_ctime = 1377067006, ia_ctime_nsec = 70871863}
        op_errno = 0
        xdata = 0xb6f8e9c8
        inode = 0xb25ec8b8
        this = 0x80a39a0
        __FUNCTION__ = "client3_3_lookup_cbk"
#11 0x00428682 in rpc_clnt_handle_reply (clnt=0x81b3a38, pollin=0xb1900468) at rpc-clnt.c:771
        conn = 0x81b3a54
        saved_frame = 0x81c69b8
        ret = 0
        req = 0xb2b0787c
        xid = 27
        __FUNCTION__ = "rpc_clnt_handle_reply"
#12 0x0042a0f6 in rpc_clnt_notify (trans=0x81ce970, mydata=0x81b3a54, event=RPC_TRANSPORT_MSG_RECEIVED, data=0xb1900468) at rpc-clnt.c:891
        clnt = 0x81b3a38
        ret = <value optimized out>
#13 0x0042486c in rpc_transport_notify (this=0x0, event=RPC_TRANSPORT_MSG_RECEIVED, data=0xb1900468) at rpc-transport.c:497
        ret = <value optimized out>
        __FUNCTION__ = "rpc_transport_notify"
#14 0x008cb60d in socket_event_poll_in (this=0x81ce970) at socket.c:2118
        ret = 0
        pollin = 0xb1900468
#15 0x008cc117 in socket_event_handler (fd=14, idx=4, data=0x81ce970, poll_in=1, poll_out=0, poll_err=0) at socket.c:2230
        this = 0x0
        priv = 0x81cf418
        ret = 0
        __FUNCTION__ = "socket_event_handler"
#16 0x00170247 in event_dispatch_epoll_handler (event_pool=0x8089728) at event-epoll.c:384
        __FUNCTION__ = "event_dispatch_epoll_handler"
#17 event_dispatch_epoll (event_pool=0x8089728) at event-epoll.c:445
        events = 0x809bac0
        i = 0
        ret = <value optimized out>
        __FUNCTION__ = "event_dispatch_epoll"
#18 0x00147001 in event_dispatch (event_pool=0x0) at event.c:113
        ret = <value optimized out>
        __FUNCTION__ = "event_dispatch"
#19 0x0804ea00 in main (argc=6, argv=0xbfffec44) at glusterfsd.c:1934
        ctx = 0x8079008
        ret = 0
        cmdlinestr = "/usr/sbin/glusterfs --attribute-timeout=0 --volfile-id=home --volfile-server=glusterfs --debug /mnt/gluster/home", '\000' <repeats 3983 times>
        __FUNCTION__ = "main"