glusterfs server crashed by trying to access illegal memeory. Operation performed was

1) Untarring linux kernel tarball in a loop on the fuse mount point by enabling quota and gsync. The crash may be due to these features being enabled.


This is the backtrace.


Core was generated by `/usr/local/sbin/glusterfsd --xlator-option vol-server.listen-port=24011 -s loca'.
Program terminated with signal 11, Segmentation fault.
#0  pthread_spin_lock (lock=0x7f56880876a01c) at ../nptl/sysdeps/x86_64/../i386/pthread_spin_lock.c:35
35	../nptl/sysdeps/x86_64/../i386/pthread_spin_lock.c: No such file or directory.
	in ../nptl/sysdeps/x86_64/../i386/pthread_spin_lock.c
(gdb) bt
#0  pthread_spin_lock (lock=0x7f56880876a01c) at ../nptl/sysdeps/x86_64/../i386/pthread_spin_lock.c:35
#1  0x00007f5690e02812 in data_unref (this=0x7f56880876a000) at ../../../libglusterfs/src/dict.c:460
#2  0x00007f5690e025af in dict_destroy (this=0x7f5688049c50) at ../../../libglusterfs/src/dict.c:393
#3  0x00007f5690e026f9 in dict_unref (this=0x7f5688049c50) at ../../../libglusterfs/src/dict.c:430
#4  0x00007f5690e3a521 in call_stub_destroy_wind (stub=0x7f568f41754c) at ../../../libglusterfs/src/call-stub.c:3540
#5  0x00007f5690e3ab22 in call_stub_destroy (stub=0x7f568f41754c) at ../../../libglusterfs/src/call-stub.c:3832
#6  0x00007f5690e3ac56 in call_resume (stub=0x7f568f41754c) at ../../../libglusterfs/src/call-stub.c:3865
#7  0x00007f568d4a7792 in iot_worker (data=0x93fc70) at ../../../../../xlators/performance/io-threads/src/io-threads.c:129
#8  0x00007f569059d9ca in start_thread (arg=<value optimized out>) at pthread_create.c:300
#9  0x00007f56902fa6fd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#10 0x0000000000000000 in ?? ()
(gdb) bt
#0  pthread_spin_lock (lock=0x7f56880876a01c) at ../nptl/sysdeps/x86_64/../i386/pthread_spin_lock.c:35
#1  0x00007f5690e02812 in data_unref (this=0x7f56880876a000) at ../../../libglusterfs/src/dict.c:460
#2  0x00007f5690e025af in dict_destroy (this=0x7f5688049c50) at ../../../libglusterfs/src/dict.c:393
#3  0x00007f5690e026f9 in dict_unref (this=0x7f5688049c50) at ../../../libglusterfs/src/dict.c:430
#4  0x00007f5690e3a521 in call_stub_destroy_wind (stub=0x7f568f41754c) at ../../../libglusterfs/src/call-stub.c:3540
#5  0x00007f5690e3ab22 in call_stub_destroy (stub=0x7f568f41754c) at ../../../libglusterfs/src/call-stub.c:3832
#6  0x00007f5690e3ac56 in call_resume (stub=0x7f568f41754c) at ../../../libglusterfs/src/call-stub.c:3865
#7  0x00007f568d4a7792 in iot_worker (data=0x93fc70) at ../../../../../xlators/performance/io-threads/src/io-threads.c:129
#8  0x00007f569059d9ca in start_thread (arg=<value optimized out>) at pthread_create.c:300
#9  0x00007f56902fa6fd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#10 0x0000000000000000 in ?? ()
(gdb) f 1
#1  0x00007f5690e02812 in data_unref (this=0x7f56880876a000) at ../../../libglusterfs/src/dict.c:460
460	        LOCK (&this->lock);
(gdb) p this
$1 = (data_t *) 0x7f56880876a000
(gdb) p *this
Cannot access memory at address 0x7f56880876a000
(gdb) info thr
  7 Thread 31156  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:211
  6 Thread 31145  0x00007f56902be38d in nanosleep () at ../sysdeps/unix/syscall-template.S:82
  5 Thread 31143  0x00007f56902facf3 in epoll_wait () at ../sysdeps/unix/syscall-template.S:82
  4 Thread 32354  0x00007f56905a593d in read () at ../sysdeps/unix/syscall-template.S:82
  3 Thread 32361  0x00007f56905a3595 in ?? () from /lib/libpthread.so.0
  2 Thread 31144  do_sigwait (set=0x7f568f415e20, sig=0x7f568f415ea8)
    at ../nptl/sysdeps/unix/sysv/linux/../../../../../sysdeps/unix/sysv/linux/sigwait.c:65
* 1 Thread 31599  pthread_spin_lock (lock=0x7f56880876a01c) at ../nptl/sysdeps/x86_64/../i386/pthread_spin_lock.c:35
(gdb) t 3
[Switching to thread 3 (Thread 32361)]#0  0x00007f56905a3595 in ?? () from /lib/libpthread.so.0
(gdb) bt
#0  0x00007f56905a3595 in ?? () from /lib/libpthread.so.0
#1  0x00007f568d291880 in mq_set_ctx_updation_status (ctx=0x7f5688070860, status=_gf_false)
    at ../../../../../xlators/features/marker/src/marker-quota.c:63
#2  0x00007f568d297f52 in quota_fetch_child_size_and_contri (frame=0x7f568f4b90cc, cookie=0x7f568f73dc38, this=0x936860, op_ret=0, 
    op_errno=0) at ../../../../../xlators/features/marker/src/marker-quota.c:1607
#3  0x00007f568d4b0d9e in iot_setxattr_cbk (frame=0x7f568f73dc38, cookie=0x7f568f72e6dc, this=0x9356a0, op_ret=0, op_errno=0)
    at ../../../../../xlators/performance/io-threads/src/io-threads.c:1511
#4  0x00007f5690e1528b in default_setxattr_cbk (frame=0x7f568f72e6dc, cookie=0x7f568f7298b4, this=0x9344b0, op_ret=0, op_errno=0)
    at ../../../libglusterfs/src/defaults.c:273
#5  0x00007f568d8df36a in posix_acl_setxattr_cbk (frame=0x7f568f7298b4, cookie=0x7f568f734b70, this=0x933210, op_ret=0, op_errno=0)
    at ../../../../../xlators/system/posix-acl/src/posix-acl.c:1664
#6  0x00007f568daf96fe in posix_setxattr (frame=0x7f568f734b70, this=0x931e40, loc=0x7f568f41849c, dict=0x7f5688055de0, flags=0)
    at ../../../../../xlators/storage/posix/src/posix.c:3194
#7  0x00007f568d8df5e8 in posix_acl_setxattr (frame=0x7f568f7298b4, this=0x933210, loc=0x7f568f41849c, xattr=0x7f5688055de0, flags=0)
    at ../../../../../xlators/system/posix-acl/src/posix-acl.c:1683
#8  0x00007f5690e1c69f in default_setxattr (frame=0x7f568f72e6dc, this=0x9344b0, loc=0x7f568f41849c, dict=0x7f5688055de0, flags=0)
    at ../../../libglusterfs/src/defaults.c:828
#9  0x00007f568d4b0fd2 in iot_setxattr_wrapper (frame=0x7f568f73dc38, this=0x9356a0, loc=0x7f568f41849c, dict=0x7f5688055de0, flags=0)
    at ../../../../../xlators/performance/io-threads/src/io-threads.c:1520
#10 0x00007f5690e3407a in call_resume_wind (stub=0x7f568f418464) at ../../../libglusterfs/src/call-stub.c:2259
#11 0x00007f5690e3ac2b in call_resume (stub=0x7f568f418464) at ../../../libglusterfs/src/call-stub.c:3859
#12 0x00007f568d4a7792 in iot_worker (data=0x93fc70) at ../../../../../xlators/performance/io-threads/src/io-threads.c:129
#13 0x00007f569059d9ca in start_thread (arg=<value optimized out>) at pthread_create.c:300
#14 0x00007f56902fa6fd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#15 0x0000000000000000 in ?? ()
(gdb)  f 1
#1  0x00007f568d291880 in mq_set_ctx_updation_status (ctx=0x7f5688070860, status=_gf_false)
    at ../../../../../xlators/features/marker/src/marker-quota.c:63
63	        LOCK (&ctx->lock);
(gdb) p ctx
$2 = (quota_inode_ctx_t *) 0x7f5688070860
(gdb) p *ctx
$3 = {size = 0, dirty = 80 'P', updation_status = 32598, lock = 0, contribution_head = {next = 0x7f568800a370, prev = 0x7f5688032a60}}
(gdb) info thr
  7 Thread 31156  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:211
  6 Thread 31145  0x00007f56902be38d in nanosleep () at ../sysdeps/unix/syscall-template.S:82
  5 Thread 31143  0x00007f56902facf3 in epoll_wait () at ../sysdeps/unix/syscall-template.S:82
  4 Thread 32354  0x00007f56905a593d in read () at ../sysdeps/unix/syscall-template.S:82
* 3 Thread 32361  0x00007f56905a3595 in ?? () from /lib/libpthread.so.0
  2 Thread 31144  do_sigwait (set=0x7f568f415e20, sig=0x7f568f415ea8)
    at ../nptl/sysdeps/unix/sysv/linux/../../../../../sysdeps/unix/sysv/linux/sigwait.c:65
  1 Thread 31599  pthread_spin_lock (lock=0x7f56880876a01c) at ../nptl/sysdeps/x86_64/../i386/pthread_spin_lock.c:35
(gdb) t 4
[Switching to thread 4 (Thread 32354)]#0  0x00007f56905a593d in read () at ../sysdeps/unix/syscall-template.S:82
82	../sysdeps/unix/syscall-template.S: No such file or directory.
	in ../sysdeps/unix/syscall-template.S
(gdb) bt
#0  0x00007f56905a593d in read () at ../sysdeps/unix/syscall-template.S:82
#1  0x00007f568daea835 in _posix_xattr_get_set (xattr_req=0x7f56880398d0, key=0x7f568803a210 "glusterfs.content", data=0x7f568801e4a0, 
    xattrargs=0x7f5687be5a00) at ../../../../../xlators/storage/posix/src/posix.c:142
#2  0x00007f5690e0459e in dict_foreach (dict=0x7f56880398d0, fn=0x7f568daea69e <_posix_xattr_get_set>, data=0x7f5687be5a00)
    at ../../../libglusterfs/src/dict.c:1198
#3  0x00007f568daeb20c in posix_lookup_xattr_fill (this=0x931e40, 
    real_path=0x7f5687be5a60 "/d/glusterfs/export/export/linux-2.6.31.1/arch/microblaze/platform/generic/system.dts", loc=0x7f568f417808, 
    xattr_req=0x7f56880398d0, buf=0x7f5687be5bb0) at ../../../../../xlators/storage/posix/src/posix.c:322
#4  0x00007f568daebdbd in posix_lookup (frame=0x7f568f72ede8, this=0x931e40, loc=0x7f568f417808, xattr_req=0x7f56880398d0)
    at ../../../../../xlators/storage/posix/src/posix.c:587
#5  0x00007f568d8d8be6 in posix_acl_lookup (frame=0x7f568f725a90, this=0x933210, loc=0x7f568f417808, xattr=0x7f56880398d0)
    at ../../../../../xlators/system/posix-acl/src/posix-acl.c:753
#6  0x00007f568d6c7188 in pl_lookup (frame=0x7f568f739f5c, this=0x9344b0, loc=0x7f568f417808, xattr_req=0x7f56880398d0)
    at ../../../../../xlators/features/locks/src/posix.c:1491
#7  0x00007f568d4a7c5b in iot_lookup_wrapper (frame=0x7f568f727b3c, this=0x9356a0, loc=0x7f568f417808, xattr_req=0x7f56880398d0)
    at ../../../../../xlators/performance/io-threads/src/io-threads.c:209
#8  0x00007f5690e34494 in call_resume_wind (stub=0x7f568f4177d0) at ../../../libglusterfs/src/call-stub.c:2408
#9  0x00007f5690e3ac2b in call_resume (stub=0x7f568f4177d0) at ../../../libglusterfs/src/call-stub.c:3859
#10 0x00007f568d4a7792 in iot_worker (data=0x93fc70) at ../../../../../xlators/performance/io-threads/src/io-threads.c:129
#11 0x00007f569059d9ca in start_thread (arg=<value optimized out>) at pthread_create.c:300
#12 0x00007f56902fa6fd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#13 0x0000000000000000 in ?? ()
(gdb)