Bug 798634

Description of problem:
Description of problem:
Created a distributed-replicate volume, on the fuse client was running dbench. The client got crashed after sometime at mdc_lookup. so remounted the client.
From a nfs client, started doing rmdir on the dbench created directories. This caused a crash on the nfs server.
Bug id: http://bugzilla.redhat.com/show_bug.cgi?id=798608

Subsequently crashed a glusterfs brick.

Core backtrace:
(gdb) bt
#0  0x0000003831e32a45 in raise () from /lib64/libc.so.6
#1  0x0000003831e34225 in abort () from /lib64/libc.so.6
#2  0x0000003831e2b9d5 in __assert_fail () from /lib64/libc.so.6
#3  0x00007f0d1906e228 in mq_fetch_child_size_and_contri (frame=0x7f0d1ca5c1d8, cookie=0x7f0d1cc612b0, this=0xde9100, op_ret=0, op_errno=0)
    at marker-quota.c:1782
#4  0x00007f0d1de2c9af in default_setxattr_cbk (frame=0x7f0d1cc612b0, cookie=0x7f0d1cc5e0f8, this=0xde7e30, op_ret=0, op_errno=0)
    at defaults.c:273
#5  0x00007f0d19492b7d in iot_setxattr_cbk (frame=0x7f0d1cc5e0f8, cookie=0x7f0d1cc62224, this=0xde6b60, op_ret=0, op_errno=0)
    at io-threads.c:1597
#6  0x00007f0d1de2c9af in default_setxattr_cbk (frame=0x7f0d1cc62224, cookie=0x7f0d1cc60698, this=0xde5940, op_ret=0, op_errno=0)
    at defaults.c:273
#7  0x00007f0d198c8162 in posix_acl_setxattr_cbk (frame=0x7f0d1cc60698, cookie=0x7f0d1cc62ae0, this=0xde46f0, op_ret=0, op_errno=0)
    at posix-acl.c:1779
#8  0x00007f0d19ae242c in posix_setxattr (frame=0x7f0d1cc62ae0, this=0xde3170, loc=0x7f0d1c92502c, dict=0x7f0cf0050980, flags=0)
    at posix.c:2368
#9  0x00007f0d198c8406 in posix_acl_setxattr (frame=0x7f0d1cc60698, this=0xde46f0, loc=0x7f0d1c92502c, xattr=0x7f0cf0050980, flags=0)
    at posix-acl.c:1798
#10 0x00007f0d1de34a81 in default_setxattr (frame=0x7f0d1cc62224, this=0xde5940, loc=0x7f0d1c92502c, dict=0x7f0cf0050980, flags=0)
    at defaults.c:846
#11 0x00007f0d19492dd7 in iot_setxattr_wrapper (frame=0x7f0d1cc5e0f8, this=0xde6b60, loc=0x7f0d1c92502c, dict=0x7f0cf0050980, flags=0)
    at io-threads.c:1606
#12 0x00007f0d1de4dcec in call_resume_wind (stub=0x7f0d1c924ff4) at call-stub.c:2309
#13 0x00007f0d1de552fc in call_resume (stub=0x7f0d1c924ff4) at call-stub.c:3938
#14 0x00007f0d194888cd in iot_worker (data=0xdfe2e0) at io-threads.c:138
#15 0x00000038322077e1 in start_thread () from /lib64/libpthread.so.0
#16 0x0000003831ee68ed in clone () from /lib64/libc.so.6
(gdb) f 3
#3  0x00007f0d1906e228 in mq_fetch_child_size_and_contri (frame=0x7f0d1ca5c1d8, cookie=0x7f0d1cc612b0, this=0xde9100, op_ret=0, op_errno=0)
    at marker-quota.c:1782
1782            GF_UUID_ASSERT (local->loc.gfid);
(gdb) l
1777            mq_set_ctx_updation_status (local->ctx, _gf_false);
1778
1779            if (uuid_is_null (local->loc.gfid))
1780                    uuid_copy (local->loc.gfid, local->loc.inode->gfid);
1781
1782            GF_UUID_ASSERT (local->loc.gfid);
1783
1784            STACK_WIND (frame, mq_update_inode_contribution, FIRST_CHILD(this),
1785                        FIRST_CHILD(this)->fops->lookup, &local->loc, newdict);
1786
(gdb) p local
$3 = (marker_local_t *) 0xdf3dbc
(gdb) p *local
$4 = {timebuf = {0, 0}, pid = 0, loc = {path = 0x7f0cfc07e470 "<gfid:26490608-2c8a-4bca-ad83-cc33d79df83e>/PARADOX/ENROLL.Y03", 
    name = 0x7f0cfc07e4a4 "ENROLL.Y03", inode = 0x7f0d1812e008, parent = 0x7f0d1812ccf4, gfid = '\000' <repeats 15 times>, 
    pargfid = "l\021I\003\034\266J謏\357\337\352\230\070", <incomplete sequence \323>}, parent_loc = {
    path = 0x7f0cfc076210 "<gfid:26490608-2c8a-4bca-ad83-cc33d79df83e>/PARADOX", name = 0x7f0cfc07623c "PARADOX", inode = 0x7f0d1812ccf4, 
    parent = 0x7f0d1812b5d4, gfid = "l\021I\003\034\266J謏\357\337\352\230\070", <incomplete sequence \323>, 
    pargfid = '\000' <repeats 15 times>}, next_lock_on = 0x0, uid = 0, gid = 0, ref = 1, ia_nlink = 0, lock = 1, mode = 0, err = 0, 
  stub = 0x0, contribution = 0, oplocal = 0x0, delta = 0, d_off = 0, sum = 0, size = 0, hl_count = 0, dentry_child_count = 0, fd = 0x0, 
  frame = 0x0, ctx = 0x7f0cfc063070, contri = 0x7f0cfc074af0}


Brick log:
[2012-02-29 05:20:25.478587] E [posix.c:1700:posix_create] 0-vol-posix: setting xattrs on /export/brick/.glusterfs/00/00/00000000-0000-0000-0
000-000000000001/00/4f4dfbbb~~~~~~~~FRKEIP10F63QI1GL90FU8E82 failed (Operation not supported)
[2012-02-29 05:20:25.492566] E [posix.c:1700:posix_create] 0-vol-posix: setting xattrs on /export/brick/.glusterfs/00/00/00000000-0000-0000-0
000-000000000001/00/4f4dfbbb~~~~~~~~DCLZZ8SQDG3WXIUX4NIWFTRF failed (Operation not supported)
[2012-02-29 05:20:25.494586] E [posix-helpers.c:645:posix_handle_pair] 0-vol-posix: setxattr on /export/brick/.glusterfs/1f/f4/1ff485c0-889e-
4b73-a1cd-7fd19fca4e5f failed: No such file or directory
[2012-02-29 05:20:25.643294] E [posix.c:1700:posix_create] 0-vol-posix: setting xattrs on /export/brick/.glusterfs/00/00/00000000-0000-0000-0
000-000000000001/00/4f4dfbbb~~~~~~~~O147VTJ3ANX5RRZ30U2M1A8N failed (Operation not supported)
[2012-02-29 05:20:25.670887] E [posix.c:1700:posix_create] 0-vol-posix: setting xattrs on /export/brick/.glusterfs/00/00/00000000-0000-0000-0
000-000000000001/00/4f4dfbbb~~~~~~~~UW8XTUABVX7YMECURTON057X failed (Operation not supported)
[2012-02-29 05:20:25.694651] W [marker-quota.c:1262:mq_get_parent_inode_local] (-->/usr/local/lib/glusterfs/3git/xlator/performance/io-thread
s.so(iot_inodelk_cbk+0x14d) [0x7f0d19495bab] (-->/usr/local/lib/libglusterfs.so.0(default_inodelk_cbk+0x14d) [0x7f0d1de2d6a0] (-->/usr/local/
lib/glusterfs/3git/xlator/features/marker.so(mq_inodelk_cbk+0x1bc) [0x7f0d1906c47a]))) 0-vol-marker: failed to build parent loc of <gfid:2649
0608-2c8a-4bca-ad83-cc33d79df83e>/PARADOX
[2012-02-29 05:20:25.696870] W [marker-quota.c:1262:mq_get_parent_inode_local] (-->/usr/local/lib/glusterfs/3git/xlator/performance/io-thread
s.so(iot_inodelk_cbk+0x14d) [0x7f0d19495bab] (-->/usr/local/lib/libglusterfs.so.0(default_inodelk_cbk+0x14d) [0x7f0d1de2d6a0] (-->/usr/local/
lib/glusterfs/3git/xlator/features/marker.so(mq_inodelk_cbk+0x1bc) [0x7f0d1906c47a]))) 0-vol-marker: failed to build parent loc of <gfid:2649
0608-2c8a-4bca-ad83-cc33d79df83e>/PARADOX
[2012-02-29 05:20:25.697356] W [marker-quota.c:1262:mq_get_parent_inode_local] (-->/usr/local/lib/glusterfs/3git/xlator/performance/io-thread
s.so(iot_inodelk_cbk+0x14d) [0x7f0d19495bab] (-->/usr/local/lib/libglusterfs.so.0(default_inodelk_cbk+0x14d) [0x7f0d1de2d6a0] (-->/usr/local/
lib/glusterfs/3git/xlator/features/marker.so(mq_inodelk_cbk+0x1bc) [0x7f0d1906c47a]))) 0-vol-marker: failed to build parent loc of <gfid:2649
0608-2c8a-4bca-ad83-cc33d79df83e>/PARADOX
[2012-02-29 05:20:25.698274] W [marker-quota.c:1262:mq_get_parent_inode_local] (-->/usr/local/lib/glusterfs/3git/xlator/performance/io-thread
s.so(iot_inodelk_cbk+0x14d) [0x7f0d19495bab] (-->/usr/local/lib/libglusterfs.so.0(default_inodelk_cbk+0x14d) [0x7f0d1de2d6a0] (-->/usr/local/
lib/glusterfs/3git/xlator/features/marker.so(mq_inodelk_cbk+0x1bc) [0x7f0d1906c47a]))) 0-vol-marker: failed to build parent loc of <gfid:2649
0608-2c8a-4bca-ad83-cc33d79df83e>/PARADOX
[2012-02-29 05:20:25.702436] W [inode.c:866:inode_lookup] (-->/usr/local/lib/glusterfs/3git/xlator/features/marker.so(marker_lookup_cbk+0x23d
) [0x7f0d19064a0a] (-->/usr/local/lib/glusterfs/3git/xlator/debug/io-stats.so(io_stats_lookup_cbk+0x262) [0x7f0d18e41c4a] (-->/usr/local/lib/
glusterfs/3git/xlator/protocol/server.so(server_lookup_cbk+0x5a2) [0x7f0d18c17944]))) 0-vol-server: inode not found
pending frames:

patchset: git://git.gluster.com/glusterfs.git
signal received: 6
time of crash: 2012-02-29 05:20:25
configuration details:
argp 1
backtrace 1
dlfcn 1
fdatasync 1
libpthread 1
llistxattr 1
setfsid 1
spinlock 1
epoll.h 1
xattr.h 1
st_atim.tv_nsec 1
package-string: glusterfs 3git
[2012-02-29 05:20:25.704257] W [marker-quota.c:1262:mq_get_parent_inode_local] (-->/usr/local/lib/glusterfs/3git/xlator/performance/io-threads.so(iot_inodelk_cbk+0x14d) [0x7f0d19495bab] (-->/usr/local/lib/libglusterfs.so.0(default_inodelk_cbk+0x14d) [0x7f0d1de2d6a0] (-->/usr/local/lib/glusterfs/3git/xlator/features/marker.so(mq_inodelk_cbk+0x1bc) [0x7f0d1906c47a]))) 0-vol-marker: failed to build parent loc of <gfid:26490608-2c8a-4bca-ad83-cc33d79df83e>/PARADOX
[2012-02-29 05:20:25.704510] W [inode.c:866:inode_lookup] (-->/usr/local/lib/glusterfs/3git/xlator/features/marker.so(marker_lookup_cbk+0x23d) [0x7f0d19064a0a] (-->/usr/local/lib/glusterfs/3git/xlator/debug/io-stats.so(io_stats_lookup_cbk+0x262) [0x7f0d18e41c4a] (-->/usr/local/lib/glusterfs/3git/xlator/protocol/server.so(server_lookup_cbk+0x5a2) [0x7f0d18c17944]))) 0-vol-server: inode not found
/lib64/libc.so.6[0x3831e32ac0]
/lib64/libc.so.6(gsignal+0x35)[0x3831e32a45]
/lib64/libc.so.6(abort+0x175)[0x3831e34225]
/lib64/libc.so.6(__assert_fail+0xf5)[0x3831e2b9d5]
/usr/local/lib/glusterfs/3git/xlator/features/marker.so(mq_fetch_child_size_and_contri+0x50f)[0x7f0d1906e228]
/usr/local/lib/libglusterfs.so.0(default_setxattr_cbk+0x14d)[0x7f0d1de2c9af]
/usr/local/lib/glusterfs/3git/xlator/performance/io-threads.so(iot_setxattr_cbk+0x14d)[0x7f0d19492b7d]
/usr/local/lib/libglusterfs.so.0(default_setxattr_cbk+0x14d)[0x7f0d1de2c9af]
/usr/local/lib/glusterfs/3git/xlator/features/access-control.so(posix_acl_setxattr_cbk+0x14d)[0x7f0d198c8162]
/usr/local/lib/glusterfs/3git/xlator/storage/posix.so(posix_setxattr+0x605)[0x7f0d19ae242c]
/usr/local/lib/glusterfs/3git/xlator/features/access-control.so(posix_acl_setxattr+0x287)[0x7f0d198c8406]
/usr/local/lib/libglusterfs.so.0(default_setxattr+0x23d)[0x7f0d1de34a81]
/usr/local/lib/glusterfs/3git/xlator/performance/io-threads.so(iot_setxattr_wrapper+0x23d)[0x7f0d19492dd7]
/usr/local/lib/libglusterfs.so.0(+0x40cec)[0x7f0d1de4dcec]
/usr/local/lib/libglusterfs.so.0(call_resume+0xd2)[0x7f0d1de552fc]
/usr/local/lib/glusterfs/3git/xlator/performance/io-threads.so(iot_worker+0x1f3)[0x7f0d194888cd]
/lib64/libpthread.so.0[0x38322077e1]
/lib64/libc.so.6(clone+0x6d)[0x3831ee68ed]
---------