Description of problem: Created a distributed replicate volume with quota & indexing enabled. From a fuse client, ran glusterfs untar & rm script. Brought a brick down, & executed `gluster volume heal` From another fuse client was modifying the script after renaming it. Crashed the 2nd replica pair with following backtrace: Core was generated by `/usr/local/sbin/glusterfsd -s localhost --volfile-id vol.dagobah.data-export3 -'. Program terminated with signal 6, Aborted. #0 0x00007f32d3e473a5 in __GI_raise (sig=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:64 64 ../nptl/sysdeps/unix/sysv/linux/raise.c: No such file or directory. in ../nptl/sysdeps/unix/sysv/linux/raise.c (gdb) bt #0 0x00007f32d3e473a5 in __GI_raise (sig=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:64 #1 0x00007f32d3e4ab0b in __GI_abort () at abort.c:92 #2 0x00007f32d3e3fd4d in __GI___assert_fail (assertion=0x7f32ce9bfb0f "!\"uuid null\"", file=<optimized out>, line=394, function=<optimized out>) at assert.c:81 #3 0x00007f32ce9a46d9 in marker_start_setxattr (frame=0x7f32d1e1a190, this=0x1231bd0) at ../../../../../xlators/features/marker/src/marker.c:394 #4 0x00007f32ce9a4a72 in marker_create_frame (this=0x1231bd0, local=0x7f32c87e5c50) at ../../../../../xlators/features/marker/src/marker.c:437 #5 0x00007f32ce9a4b8c in marker_xtime_update_marks (this=0x1231bd0, local=0x7f32c87e5c50) at ../../../../../xlators/features/marker/src/marker.c:455 #6 0x00007f32ce9a86be in marker_rename_done (frame=0x7f32d2e65778, cookie=0x0, this=0x1231bd0, op_ret=0, op_errno=0) at ../../../../../xlators/features/marker/src/marker.c:974 #7 0x00007f32ce9a888d in marker_rename_release_newp_lock (frame=0x7f32d2e65778, cookie=0x7f32d2e62dd0, this=0x1231bd0, op_ret=0, op_errno=0) at ../../../../../xlators/features/marker/src/marker.c:1007 #8 0x00007f32cebd876e in iot_inodelk_cbk (frame=0x7f32d2e62dd0, cookie=0x7f32d2e66cf8, this=0x1230920, op_ret=0, op_errno=0) at ../../../../../xlators/performance/io-threads/src/io-threads.c:1978 #9 0x00007f32cedf6d5c in pl_common_inodelk (frame=0x7f32d2e66cf8, this=0x122f710, volume=0x7f32c87e5aa0 "vol-marker", inode=0x7f32cd6ee04c, cmd=7, flock=0x7f32d1ce4a04, loc=0x7f32d1ce49bc, fd=0x0) at ../../../../../xlators/features/locks/src/inodelk.c:653 #10 0x00007f32cedf6dc9 in pl_inodelk (frame=0x7f32d2e66cf8, this=0x122f710, volume=0x7f32c87e5aa0 "vol-marker", loc=0x7f32d1ce49bc, cmd=7, flock=0x7f32d1ce4a04) at ../../../../../xlators/features/locks/src/inodelk.c:663 #11 0x00007f32cebd89ca in iot_inodelk_wrapper (frame=0x7f32d2e62dd0, this=0x1230920, volume=0x7f32c87e5aa0 "vol-marker", loc=0x7f32d1ce49bc, cmd=7, lock=0x7f32d1ce4a04) at ../../../../../xlators/performance/io-threads/src/io-threads.c:1987 #12 0x00007f32d48416a7 in call_resume_wind (stub=0x7f32d1ce497c) at ../../../libglusterfs/src/call-stub.c:2419 #13 0x00007f32d48488dd in call_resume (stub=0x7f32d1ce497c) at ../../../libglusterfs/src/call-stub.c:3938 #14 0x00007f32cebcb8ef in iot_worker (data=0x123b570) at ../../../../../xlators/performance/io-threads/src/io-threads.c:138 #15 0x00007f32d41b7efc in start_thread (arg=0x7f32cd444700) at pthread_create.c:304 #16 0x00007f32d3ef289d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112 #17 0x0000000000000000 in ?? () (gdb) f 3 #3 0x00007f32ce9a46d9 in marker_start_setxattr (frame=0x7f32d1e1a190, this=0x1231bd0) at ../../../../../xlators/features/marker/src/marker.c:394 394 GF_UUID_ASSERT (local->loc.gfid); (gdb) p local $1 = (marker_local_t *) 0x7f32c87e5c50 (gdb) p *local $2 = {timebuf = {876952143, 2250444032}, pid = 26987, loc = {path = 0x7f32c87e5a20 "/smoke1.sh~", name = 0x7f32c87e5a21 "smoke1.sh~", inode = 0x0, parent = 0x7f32cd6ee04c, gfid = '\000' <repeats 15 times>, pargfid = '\000' <repeats 15 times>, "\001"}, parent_loc = { path = 0x7f32c87e5e50 "/", name = 0x7f32c87e5e51 "", inode = 0x7f32cd6ee04c, parent = 0x0, gfid = '\000' <repeats 15 times>, pargfid = '\000' <repeats 15 times>}, next_lock_on = 0x0, uid = 0, gid = 0, ref = 2, ia_nlink = 0, lock = 1, mode = 0, err = 0, stub = 0x0, contribution = 0, oplocal = 0x7f32c87e5d50} (gdb) f 6 #6 0x00007ff76146e6be in marker_rename_done (frame=0x7ff76592d100, cookie=0x0, this=0x1890c00, op_ret=0, op_errno=0) at ../../../../../xlators/features/marker/src/marker.c:974 974 marker_xtime_update_marks (this, local); (gdb) l marker_rename_done 917 918 919 int32_t 920 marker_rename_done (call_frame_t *frame, void *cookie, xlator_t *this, 921 int32_t op_ret, int32_t op_errno) 922 { 923 marker_local_t *local = NULL, *oplocal = NULL; 924 loc_t newloc = {0, }; 925 marker_conf_t *priv = NULL; 926 (gdb) 927 local = frame->local; 928 oplocal = local->oplocal; 929 930 priv = this->private; 931 932 frame->local = NULL; 933 934 if (op_ret < 0) { 935 if (local->err == 0) { 936 local->err = op_errno; (gdb) 937 } 938 939 gf_log (this->name, GF_LOG_WARNING, 940 "inodelk (UNLOCK) failed on path:%s (gfid:%s) (%s)", 941 local->parent_loc.path, 942 uuid_utoa (local->parent_loc.inode->gfid), 943 strerror (op_errno)); 944 } 945 946 if (local->stub != NULL) { (gdb) 947 call_resume (local->stub); 948 local->stub = NULL; 949 } else if (local->err != 0) { 950 STACK_UNWIND_STRICT (rename, frame, -1, local->err, NULL, NULL, 951 NULL, NULL, NULL); 952 } 953 954 mq_reduce_parent_size (this, &oplocal->loc, oplocal->contribution); 955 956 if (local->loc.inode != NULL) { (gdb) 957 mq_reduce_parent_size (this, &local->loc, local->contribution); 958 } 959 960 newloc.inode = inode_ref (oplocal->loc.inode); 961 newloc.path = gf_strdup (local->loc.path); 962 newloc.name = strrchr (newloc.path, '/'); 963 if (newloc.name) 964 newloc.name++; 965 newloc.parent = inode_ref (local->loc.parent); 966 (gdb) p newloc.path $12 = 0x0 (gdb) p local->loc $13 = {path = 0x7ff754296130 "/smoke1.sh~", name = 0x7ff754296131 "smoke1.sh~", inode = 0x0, parent = 0x7ff7601b404c, gfid = '\000' <repeats 15 times>, pargfid = '\000' <repeats 15 times>, "\001"} (gdb) p newloc.inode $14 = (inode_t *) 0x0 (gdb) p oplocal->loc.inode $15 = (inode_t *) 0x7ff7601b404c (gdb) p *oplocal->loc.inode $16 = {table = 0x18a6d80, gfid = '\000' <repeats 15 times>, "\001", lock = 1, nlookup = 1, ref = 137571, ia_type = IA_IFDIR, fd_list = { next = 0x7ff75becc7f8, prev = 0x7ff75be8f114}, dentry_list = {next = 0x7ff7601b408c, prev = 0x7ff7601b408c}, hash = { next = 0x7ff7601b409c, prev = 0x7ff7601b409c}, list = {next = 0x18a6de0, prev = 0x7ff7601b41d4}, _ctx = 0x18bbfe0}
CHANGE: http://review.gluster.com/2750 (features/marker: copy the gfid to loc from oplocal->loc.inode) merged in master by Vijay Bellur (vijay)