Description of problem: (gdb) bt #0 0x00007f2758899c62 in list_del_init (old=0x7f273c020ed0) at ../../../../../libglusterfs/src/list.h:71 #1 0x00007f275889b1c8 in __delete_lock (pl_inode=0x7f273c00ac30, lock=0x7f273c020ed0) at ../../../../../xlators/features/locks/src/common.c:516 #2 0x00007f275889c196 in __insert_and_merge (pl_inode=0x7f273c00ac30, lock=0x7f273c020ed0) at ../../../../../xlators/features/locks/src/common.c:820 #3 0x00007f275889c094 in __insert_and_merge (pl_inode=0x7f273c00ac30, lock=0x7f273c0339e0) at ../../../../../xlators/features/locks/src/common.c:803 #4 0x00007f275889cd8a in pl_setlk (this=0x1f888a0, pl_inode=0x7f273c00ac30, lock=0x7f273c0339e0, can_block=0) at ../../../../../xlators/features/locks/src/common.c:1027 #5 0x00007f27588a314c in pl_lk (frame=0x7f275b80b7f8, this=0x1f888a0, fd=0x1fb9190, cmd=6, flock=0x7f275b4c26fc, xdata=0x0) at ../../../../../xlators/features/locks/src/posix.c:1229 #6 0x00007f27586871ab in iot_lk_wrapper (frame=0x7f275b80008c, this=0x1f89a20, fd=0x1fb9190, cmd=6, flock=0x7f275b4c26fc, xdata=0x0) at ../../../../../xlators/performance/io-threads/src/io-threads.c:1137 #7 0x00007f275c9e8fe0 in call_resume_wind (stub=0x7f275b4c26ac) at ../../../libglusterfs/src/call-stub.c:2640 #8 0x00007f275c9f067a in call_resume (stub=0x7f275b4c26ac) at ../../../libglusterfs/src/call-stub.c:4160 #9 0x00007f27586808cf in iot_worker (data=0x1fa0620) at ../../../../../xlators/performance/io-threads/src/io-threads.c:138 #10 0x0000003680c07b41 in start_thread () from /lib64/libpthread.so.0 #11 0x00000036808df49d in clone () from /lib64/libc.so.6 (gdb) info thr Id Target Id Frame 10 Thread 0x7f275b334700 (LWP 8731) 0x0000003680c0ec27 in do_sigwait () from /lib64/libpthread.so.0 9 Thread 0x7f2752fc2700 (LWP 8812) 0x0000003680c0b82f in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 8 Thread 0x7f275c54c720 (LWP 8730) 0x00000036808dfa73 in epoll_wait () from /lib64/libc.so.6 7 Thread 0x7f275359f700 (LWP 8738) 0x0000003680c0b82f in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 6 Thread 0x7f274bfff700 (LWP 8740) 0x0000003680c0b82f in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 5 Thread 0x7f27536a0700 (LWP 8737) 0x0000003680c0b4c5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 4 Thread 0x7f27596ef700 (LWP 8736) 0x0000003680c0e75d in nanosleep () from /lib64/libpthread.so.0 3 Thread 0x7f275a332700 (LWP 8733) 0x0000003680c0b4c5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 2 Thread 0x7f275ab33700 (LWP 8732) 0x0000003680c0b4c5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 * 1 Thread 0x7f2752ec1700 (LWP 8814) 0x00007f2758899c62 in list_del_init (old=0x7f273c020ed0) at ../../../../../libglusterfs/src/list.h:71 (gdb) up #1 0x00007f275889b1c8 in __delete_lock (pl_inode=0x7f273c00ac30, lock=0x7f273c020ed0) at ../../../../../xlators/features/locks/src/common.c:516 516 list_del_init (&lock->list); (gdb) p *lock $1 = {list = {next = 0x0, prev = 0x0}, fl_type = 1, fl_start = 0, fl_end = 9223372036854775807, blocked = 0, user_flock = { l_type = 0, l_whence = 0, l_start = 0, l_len = 0, l_pid = 0, l_owner = {len = 0, data = '\000' <repeats 1023 times>}}, this = 0x0, fd_num = 33264016, fd = 0x0, frame = 0x0, blkd_time = {tv_sec = 0, tv_usec = 0}, granted_time = {tv_sec = 0, tv_usec = 0}, transport = 0x1face60, owner = {len = 8, data = "\036\216C\\Vi\347\027", '\000' <repeats 1015 times>}, client_pid = 10265} (gdb) up #2 0x00007f275889c196 in __insert_and_merge (pl_inode=0x7f273c00ac30, lock=0x7f273c020ed0) at ../../../../../xlators/features/locks/src/common.c:820 820 __delete_lock (pl_inode, lock); (gdb) p *pl_inode $2 = {mutex = {__data = {__lock = 1, __count = 0, __owner = 8814, __nusers = 1, __kind = 0, __spins = 0, __list = { __prev = 0x0, __next = 0x0}}, __size = "\001\000\000\000\000\000\000\000n\"\000\000\001", '\000' <repeats 26 times>, __align = 1}, dom_list = { next = 0x7f27380497e0, prev = 0x7f273c00ad00}, ext_list = {next = 0x7f273c025e20, prev = 0x7f273c025e20}, rw_list = { next = 0x7f273c00ac78, prev = 0x7f273c00ac78}, reservelk_list = {next = 0x7f273c00ac88, prev = 0x7f273c00ac88}, blocked_reservelks = {next = 0x7f273c00ac98, prev = 0x7f273c00ac98}, blocked_calls = {next = 0x7f273c00aca8, prev = 0x7f273c00aca8}, mandatory = 0, refkeeper = 0x7f2753277640} (gdb) Version-Release number of selected component (if applicable): master How reproducible: happened 2/3 Additional info: Volume Name: a Type: Distributed-Replicate Volume ID: 9e1f5ab2-e363-4395-bb62-37b32db8c58f Status: Started Number of Bricks: 2 x 2 = 4 Transport-type: tcp Bricks: Brick1: supernova:/tmp/export/a1 Brick2: supernova:/tmp/export/a2 Brick3: supernova:/tmp/export/a3 Brick4: supernova:/tmp/export/a4 Options Reconfigured: geo-replication.indexing: on features.quota: on
CHANGE: http://review.gluster.com/3047 (features/locks: INIT the list before insert_merge) merged in master by Vijay Bellur (vijay)
Checked with glusterfs-3.3.0qa42. Ran ltp tests and they completed successfully without crashing glusterfs.