Bug 783396

Description of problem:

The distributed replicate vol which is mounted via fuse with & without acl.
Ran sanity & iozone -a from both the mounts with quota & indexing enabled.

Caused crash.

Core was generated by `/usr/local/sbin/glusterfsd --xlator-option vol-server.listen-port=24009 -s loca'.
Program terminated with signal 11, Segmentation fault.
#0  0x0000003ae572686f in __strlen_sse42 () from /lib64/libc.so.6
Missing separate debuginfos, use: debuginfo-install glibc-2.12-1.25.el6.x86_64 libgcc-4.4.5-6.el6.x86_64
(gdb) bt
#0  0x0000003ae572686f in __strlen_sse42 () from /lib64/libc.so.6
#1  0x00007ff58b746d04 in gf_strdup (src=0x0) at mem-pool.h:127
#2  0x00007ff58b74cad2 in loc_copy (dst=0x7ff58a4e049c, src=0x21c94d0) at xlator.c:1732
#3  0x00007ff58b76d519 in fop_setxattr_stub (frame=0x7ff58a943048, fn=0x7ff5880fe497 <iot_setxattr_wrapper>, loc=0x21c94d0, 
    dict=0x7ff56c046510, flags=0) at call-stub.c:1246
#4  0x00007ff5880fe741 in iot_setxattr (frame=0x7ff58a943048, this=0x2133e40, loc=0x21c94d0, dict=0x7ff56c046510, flags=0)
    at io-threads.c:1533
#5  0x00007ff583de17fa in marker_start_setxattr (frame=0x7ff58a6f01fc, this=0x2135000) at marker.c:396
#6  0x00007ff583de18d5 in marker_create_frame (this=0x2135000, local=0x21c94c0) at marker.c:426
#7  0x00007ff583de1917 in marker_xtime_update_marks (this=0x2135000, local=0x21c94c0) at marker.c:438
#8  0x00007ff583de8c55 in marker_ftruncate_cbk (frame=0x7ff58a8876f4, cookie=0x7ff58a926ca4, this=0x2135000, op_ret=0, op_errno=0, 
    prebuf=0x7ff55fffe900, postbuf=0x7ff55fffe890) at marker.c:1561
#9  0x00007ff5880fc067 in iot_ftruncate_cbk (frame=0x7ff58a926ca4, cookie=0x7ff58a83f1fc, this=0x2133e40, op_ret=0, op_errno=0, 
    prebuf=0x7ff55fffe900, postbuf=0x7ff55fffe890) at io-threads.c:1223
#10 0x00007ff58830d6bd in pl_truncate_cbk (frame=0x7ff58a83f1fc, cookie=0x7ff58a943048, this=0x2132c60, op_ret=0, op_errno=0, 
    prebuf=0x7ff55fffe900, postbuf=0x7ff55fffe890) at posix.c:120
#11 0x00007ff58852756a in posix_acl_ftruncate_cbk (frame=0x7ff58a943048, cookie=0x7ff58a9a38d4, this=0x2131a00, op_ret=0, op_errno=0, 
    prebuf=0x7ff55fffe900, postbuf=0x7ff55fffe890) at posix-acl.c:976
#12 0x00007ff5887493b5 in posix_ftruncate (frame=0x7ff58a9a38d4, this=0x21305b0, fd=0x7ff5829ffdb8, offset=10240) at posix.c:4156
#13 0x00007ff5885277ee in posix_acl_ftruncate (frame=0x7ff58a943048, this=0x2131a00, fd=0x7ff5829ffdb8, offset=10240) at posix-acl.c:995
#14 0x00007ff58830df0c in truncate_stat_cbk (frame=0x7ff58a83f1fc, cookie=0x7ff58a91b944, this=0x2132c60, op_ret=0, op_errno=0, 
    buf=0x7ff55fffebd0) at posix.c:209
#15 0x00007ff58b751131 in default_fstat_cbk (frame=0x7ff58a91b944, cookie=0x7ff58a9bf6b4, this=0x2131a00, op_ret=0, op_errno=0, 
    buf=0x7ff55fffebd0) at defaults.c:240
#16 0x00007ff588749908 in posix_fstat (frame=0x7ff58a9bf6b4, this=0x21305b0, fd=0x7ff5829ffdb8) at posix.c:4209
#17 0x00007ff58b759d14 in default_fstat (frame=0x7ff58a91b944, this=0x2131a00, fd=0x7ff5829ffdb8) at defaults.c:860
#18 0x00007ff58830e8f0 in pl_ftruncate (frame=0x7ff58a83f1fc, this=0x2132c60, fd=0x7ff5829ffdb8, offset=10240) at posix.c:274
#19 0x00007ff5880fc2b4 in iot_ftruncate_wrapper (frame=0x7ff58a926ca4, this=0x2133e40, fd=0x7ff5829ffdb8, offset=10240) at io-threads.c:1233
#20 0x00007ff58b771235 in call_resume_wind (stub=0x7ff58a576e5c) at call-stub.c:2333
#21 0x00007ff58b778416 in call_resume (stub=0x7ff58a576e5c) at call-stub.c:3859
#22 0x00007ff5880f42b3 in iot_worker (data=0x213e480) at io-threads.c:129
#23 0x0000003ae5a077e1 in start_thread () from /lib64/libpthread.so.0
#24 0x0000003ae56e68ed in clone () from /lib64/libc.so.6
(gdb) f 8
#8  0x00007ff583de8c55 in marker_ftruncate_cbk (frame=0x7ff58a8876f4, cookie=0x7ff58a926ca4, this=0x2135000, op_ret=0, op_errno=0, 
    prebuf=0x7ff55fffe900, postbuf=0x7ff55fffe890) at marker.c:1561
1561                    marker_xtime_update_marks (this, local);
(gdb) p local
$1 = (marker_local_t *) 0x21c94c0
(gdb) p *local
$2 = {timebuf = {756619343, 3397389824}, pid = -1, loc = {path = 0x0, name = 0x0, inode = 0x0, parent = 0x0, 
    gfid = '\000' <repeats 15 times>, pargfid = '\000' <repeats 15 times>, ino = 0}, parent_loc = {path = 0x0, name = 0x0, inode = 0x0, 
    parent = 0x0, gfid = '\000' <repeats 15 times>, pargfid = '\000' <repeats 15 times>, ino = 0}, next_lock_on = 0x0, uid = 4294967295, 
  gid = 4294967295, ref = 2, ia_nlink = 0, lock = 1, mode = 0, err = 0, stub = 0x0, contribution = 0, oplocal = 0x0}
(gdb) f 7
#7  0x00007ff583de1917 in marker_xtime_update_marks (this=0x2135000, local=0x21c94c0) at marker.c:438
438             marker_create_frame (this, local);
(gdb) l
433     {
434             marker_gettimeofday (local);
435
436             marker_local_ref (local);
437
438             marker_create_frame (this, local);
439
440             return 0;
441     }

Volinfo:
Volume Name: vol
Type: Distributed-Replicate
Status: Started
Number of Bricks: 2 x 2 = 4
Transport-type: tcp
Bricks:
Brick1: 10.1.11.154:/export/brick
Brick2: 10.1.11.152:/export/brick
Brick3: 10.1.11.151:/export/brick
Brick4: 10.1.11.153:/export/brick
Options Reconfigured:
performance.cache-refresh-timeout: 5
geo-replication.indexing: on
features.quota: on
diagnostics.latency-measurement: on
diagnostics.count-fop-hits: on
features.limit-usage: /:50GB
performance.io-cache: on


Brick log:
[2012-01-19 08:22:50.964521] I [server3_1-fops.c:1146:server_ftruncate_cbk] 0-vol-server: 1848430: FTRUNCATE 2 (-750310584) ==> -1 (Cannot al
locate memory)
[2012-01-19 08:22:50.973422] W [inode.c:1044:inode_path] (-->/usr/local/lib/glusterfs/3.2.6qa1/xlator/protocol/server.so(server_resolve+0x9e)
 [0x7ff583999fb4] (-->/usr/local/lib/glusterfs/3.2.6qa1/xlator/protocol/server.so(server_resolve_inode+0x87) [0x7ff583999df3] (-->/usr/local/lib/glusterfs/3.2.6qa1/xlator/protocol/server.so(resolve_loc_touchup+0xd4) [0x7ff583998b0e]))) 0-/export/brick/inode: no dentry for non-root inode -750310584: abcdfc7d-974a-4d1a-ae26-5e862520ccc2
[2012-01-19 08:22:50.975583] W [inode.c:1044:inode_path] (-->/usr/local/lib/glusterfs/3.2.6qa1/xlator/protocol/server.so(server_resolve+0x9e) [0x7ff583999fb4] (-->/usr/local/lib/glusterfs/3.2.6qa1/xlator/protocol/server.so(server_resolve_inode+0x87) [0x7ff583999df3] (-->/usr/local/lib/glusterfs/3.2.6qa1/xlator/protocol/server.so(resolve_loc_touchup+0xd4) [0x7ff583998b0e]))) 0-/export/brick/inode: no dentry for non-root inode -750310584: abcdfc7d-974a-4d1a-ae26-5e862520ccc2
[2012-01-19 08:22:50.975664] E [posix.c:867:posix_setattr] 0-vol-posix: setattr (lstat) on /export/brick/run19387/test/file17017 failed: No such file or directory
[2012-01-19 08:22:50.975684] I [server3_1-fops.c:1526:server_setattr_cbk] 0-vol-server: 1848444: SETATTR /run19387/test/file17017 (-750310584) ==> -1 (No such file or directory)
[2012-01-19 08:22:53.944015] E [posix.c:3209:handle_pair] 0-vol-posix: setxattr on /export/brick/system_light/run19235/test/file12349 failed: No such file or directory
[2012-01-19 08:22:53.946638] W [inode.c:1044:inode_path] (-->/usr/local/lib/glusterfs/3.2.6qa1/xlator/debug/io-stats.so(io_stats_ftruncate+0x28d) [0x7ff583bd413f] (-->/usr/local/lib/glusterfs/3.2.6qa1/xlator/features/marker.so(marker_ftruncate+0x16b) [0x7ff583de8dd8] (-->/usr/local/lib/glusterfs/3.2.6qa1/xlator/features/marker.so(marker_inode_loc_fill+0x97) [0x7ff583de075d]))) 0-/export/brick/inode: no dentry for non-root inode -1921457037: bb97e1e5-1c5c-4f10-bb98-1728d1e0cd4b
[2012-01-19 08:22:53.950443] E [marker-quota.c:1950:mq_initiate_quota_txn] (-->/usr/local/lib/glusterfs/3.2.6qa1/xlator/features/locks.so(pl_truncate_cbk+0x18b) [0x7ff58830d6bd] (-->/usr/local/lib/glusterfs/3.2.6qa1/xlator/performance/io-threads.so(iot_ftruncate_cbk+0x160) [0x7ff5880fc067] (-->/usr/local/lib/glusterfs/3.2.6qa1/xlator/features/marker.so(marker_ftruncate_cbk+0x241) [0x7ff583de8c31]))) 0-marker: invalid argument: loc->inode
pending frames:

patchset: git://git.gluster.com/glusterfs.git
signal received: 11
time of crash: 2012-01-19 08:22:53
configuration details:
argp 1
backtrace 1
dlfcn 1
fdatasync 1
libpthread 1
llistxattr 1
setfsid 1
spinlock 1
epoll.h 1
xattr.h 1
st_atim.tv_nsec 1
package-string: glusterfs 3.2.6qa1
/lib64/libc.so.6[0x3ae5632ac0]
[2012-01-19 08:22:54.10464] W [inode.c:1044:inode_path] (-->/usr/local/lib/glusterfs/3.2.6qa1/xlator/protocol/server.so(server_resolve+0x9e) [0x7ff583999fb4] (-->/usr/local/lib/glusterfs/3.2.6qa1/xlator/protocol/server.so(server_resolve_inode+0x87) [0x7ff583999df3] (-->/usr/local/lib/glusterfs/3.2.6qa1/xlator/protocol/server.so(resolve_loc_touchup+0xd4) [0x7ff583998b0e]))) 0-/export/brick/inode: no dentry for non-root inode -1921457037: bb97e1e5-1c5c-4f10-bb98-1728d1e0cd4b
/lib64/libc.so.6[0x3ae572686f]
/usr/local/lib/libglusterfs.so.0(+0x19d04)[0x7ff58b746d04]
/usr/local/lib/libglusterfs.so.0(loc_copy+0x186)[0x7ff58b74cad2]
/usr/local/lib/libglusterfs.so.0(fop_setxattr_stub+0x196)[0x7ff58b76d519]
/usr/local/lib/glusterfs/3.2.6qa1/xlator/performance/io-threads.so(iot_setxattr+0x50)[0x7ff5880fe741]
/usr/local/lib/glusterfs/3.2.6qa1/xlator/features/marker.so(marker_start_setxattr+0x2fd)[0x7ff583de17fa]
/usr/local/lib/glusterfs/3.2.6qa1/xlator/features/marker.so(marker_create_frame+0x5f)[0x7ff583de18d5]
/usr/local/lib/glusterfs/3.2.6qa1/xlator/features/marker.so(marker_xtime_update_marks+0x3b)[0x7ff583de1917]
/usr/local/lib/glusterfs/3.2.6qa1/xlator/features/marker.so(marker_ftruncate_cbk+0x265)[0x7ff583de8c55]
/usr/local/lib/glusterfs/3.2.6qa1/xlator/performance/io-threads.so(iot_ftruncate_cbk+0x160)[0x7ff5880fc067]
/usr/local/lib/glusterfs/3.2.6qa1/xlator/features/locks.so(pl_truncate_cbk+0x18b)[0x7ff58830d6bd]
/usr/local/lib/glusterfs/3.2.6qa1/xlator/features/access-control.so(posix_acl_ftruncate_cbk+0x160)[0x7ff58852756a]
/usr/local/lib/glusterfs/3.2.6qa1/xlator/storage/posix.so(posix_ftruncate+0x6b6)[0x7ff5887493b5]
/usr/local/lib/glusterfs/3.2.6qa1/xlator/features/access-control.so(posix_acl_ftruncate+0x267)[0x7ff5885277ee]
/usr/local/lib/glusterfs/3.2.6qa1/xlator/features/locks.so(+0x6f0c)[0x7ff58830df0c]
/usr/local/lib/libglusterfs.so.0(default_fstat_cbk+0x158)[0x7ff58b751131]
/usr/local/lib/glusterfs/3.2.6qa1/xlator/storage/posix.so(posix_fstat+0x533)[0x7ff588749908]
/usr/local/lib/libglusterfs.so.0(default_fstat+0x227)[0x7ff58b759d14]
/usr/local/lib/glusterfs/3.2.6qa1/xlator/features/locks.so(pl_ftruncate+0x2e5)[0x7ff58830e8f0]
/usr/local/lib/glusterfs/3.2.6qa1/xlator/performance/io-threads.so(iot_ftruncate_wrapper+0x230)[0x7ff5880fc2b4]
/usr/local/lib/libglusterfs.so.0(+0x44235)[0x7ff58b771235]
/usr/local/lib/libglusterfs.so.0(call_resume+0xd2)[0x7ff58b778416]
/usr/local/lib/glusterfs/3.2.6qa1/xlator/performance/io-threads.so(iot_worker+0x1b2)[0x7ff5880f42b3]
/lib64/libpthread.so.0[0x3ae5a077e1]
/lib64/libc.so.6(clone+0x6d)[0x3ae56e68ed]
---------