Bug 805469

Description of problem:
3 replica volume. had 4 fuse clients. Unmounted all the clients and gave volume stop to find that one of the glusterfs servers had crashed while destroying the connection object for one of the clients. This is the backtrace of the core.

GNU gdb (GDB) Red Hat Enterprise Linux (7.2-50.el6)
Copyright (C) 2010 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /usr/local/sbin/glusterfs...done.
[New Thread 21022]
[New Thread 19078]
[New Thread 19076]
[New Thread 19070]
[New Thread 19062]
[New Thread 19061]
[New Thread 19060]
[New Thread 19059]
Reading symbols from /usr/lib64/valgrind/vgpreload_core-amd64-linux.so...done.
Loaded symbols for /usr/lib64/valgrind/vgpreload_core-amd64-linux.so
Reading symbols from /usr/lib64/valgrind/vgpreload_memcheck-amd64-linux.so...done.
Loaded symbols for /usr/lib64/valgrind/vgpreload_memcheck-amd64-linux.so
Reading symbols from /usr/local/lib/libglusterfs.so.0...done.
Loaded symbols for /usr/local/lib/libglusterfs.so.0
Reading symbols from /usr/local/lib/libgfrpc.so.0...done.
Loaded symbols for /usr/local/lib/libgfrpc.so.0
Reading symbols from /usr/local/lib/libgfxdr.so.0...done.
Loaded symbols for /usr/local/lib/libgfxdr.so.0
Reading symbols from /lib64/libdl.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libdl.so.2
Reading symbols from /lib64/libpthread.so.0...(no debugging symbols found)...done.
[Thread debugging using libthread_db enabled]
Loaded symbols for /lib64/libpthread.so.0
Reading symbols from /lib64/libc.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib64/libc.so.6
Reading symbols from /lib64/ld-linux-x86-64.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/ld-linux-x86-64.so.2
Reading symbols from /usr/local/lib/glusterfs/3.3.0qa30/rpc-transport/socket.so...done.
Loaded symbols for /usr/local/lib/glusterfs/3.3.0qa30/rpc-transport/socket.so
Reading symbols from /lib64/libnss_files.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libnss_files.so.2
Reading symbols from /usr/local/lib/glusterfs/3.3.0qa30/xlator/storage/posix.so...done.
Loaded symbols for /usr/local/lib/glusterfs/3.3.0qa30/xlator/storage/posix.so
Reading symbols from /usr/local/lib/glusterfs/3.3.0qa30/xlator/features/access-control.so...done.
Loaded symbols for /usr/local/lib/glusterfs/3.3.0qa30/xlator/features/access-control.so
Reading symbols from /usr/local/lib/glusterfs/3.3.0qa30/xlator/features/locks.so...done.
Loaded symbols for /usr/local/lib/glusterfs/3.3.0qa30/xlator/features/locks.so
Reading symbols from /usr/local/lib/glusterfs/3.3.0qa30/xlator/performance/io-threads.so...done.
Loaded symbols for /usr/local/lib/glusterfs/3.3.0qa30/xlator/performance/io-threads.so
Reading symbols from /usr/local/lib/glusterfs/3.3.0qa30/xlator/features/index.so...done.
Loaded symbols for /usr/local/lib/glusterfs/3.3.0qa30/xlator/features/index.so
Reading symbols from /usr/local/lib/glusterfs/3.3.0qa30/xlator/features/marker.so...done.
Loaded symbols for /usr/local/lib/glusterfs/3.3.0qa30/xlator/features/marker.so
Reading symbols from /usr/local/lib/glusterfs/3.3.0qa30/xlator/debug/io-stats.so...done.
Loaded symbols for /usr/local/lib/glusterfs/3.3.0qa30/xlator/debug/io-stats.so
Reading symbols from /usr/local/lib/glusterfs/3.3.0qa30/xlator/protocol/server.so...done.
Loaded symbols for /usr/local/lib/glusterfs/3.3.0qa30/xlator/protocol/server.so
Reading symbols from /usr/local/lib/glusterfs/3.3.0qa30/auth/login.so...done.
Loaded symbols for /usr/local/lib/glusterfs/3.3.0qa30/auth/login.so
Reading symbols from /usr/local/lib/glusterfs/3.3.0qa30/auth/addr.so...done.
Loaded symbols for /usr/local/lib/glusterfs/3.3.0qa30/auth/addr.so
Reading symbols from /lib64/libgcc_s.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libgcc_s.so.1
Missing separate debuginfos, use: debuginfo-install glibc-2.12-1.47.el6_2.5.x86_64 libgcc-4.4.6-3.el6.x86_64 valgrind-3.6.0-4.el6.x86_64
(gdb) bt
#0  0x0000003cdc20b75b in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1  0x00000000099dd7d6 in iot_worker (data=0x56c1f00) at ../../../../../xlators/performance/io-threads/src/io-threads.c:110
#2  0x0000003cdc2077f1 in start_thread () from /lib64/libpthread.so.0
#3  0x0000003cdbae592d in clone () from /lib64/libc.so.6
(gdb) info thr
(gdb) t 2
[Switching to thread 2 (Thread 19078)]#0  0x0000003cdc20b75b in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
(gdb) t 3
[Switching to thread 3 (Thread 19076)]#0  0x0000003cdc20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
(gdb) t 4
[Switching to thread 4 (Thread 19070)]#0  0x0000003cdba32885 in raise () from /lib64/libc.so.6
(gdb) t 5
[Switching to thread 5 (Thread 19062)]#0  0x0000003cdc20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
(gdb) t 6
[Switching to thread 6 (Thread 19061)]#0  0x0000003cdc20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
(gdb) t 7
[Switching to thread 7 (Thread 19060)]#0  0x0000003cdc20f245 in sigwait () from /lib64/libpthread.so.0
(gdb) t 8
[Switching to thread 8 (Thread 19059)]#0  0x0000003cdbae5f23 in epoll_wait () from /lib64/libc.so.6
(gdb) f[Kt 9
(gdb) t 4
[Switching to thread 4 (Thread 19070)]#0  0x0000003cdba32885 in raise () from /lib64/libc.so.6
(gdb) bt
#0  0x0000003cdba32885 in raise () from /lib64/libc.so.6
#1  0x0000003cdba34065 in abort () from /lib64/libc.so.6
#2  0x0000003cdba2b9fe in __assert_fail_base () from /lib64/libc.so.6
#3  0x0000003cdba2bac0 in __assert_fail () from /lib64/libc.so.6
#4  0x0000000004c5a10e in __gf_free (free_ptr=0xcd8cda0) at ../../../libglusterfs/src/mem-pool.c:288
#5  0x000000000a252c68 in server_connection_destroy (this=0x65a71a0, conn=0xcd8ccc0)
    at ../../../../../xlators/protocol/server/src/server-helpers.c:627
#6  0x000000000a252d0b in server_conn_unref (conn=0xcd8ccc0) at ../../../../../xlators/protocol/server/src/server-helpers.c:652
#7  0x000000000a24ac8d in grace_time_handler (data=0xcd8ccc0) at ../../../../../xlators/protocol/server/src/server.c:63
#8  0x0000000004c3e4fc in gf_timer_proc (ctx=0x52e9040) at ../../../libglusterfs/src/timer.c:177
#9  0x0000003cdc2077f1 in start_thread () from /lib64/libpthread.so.0
#10 0x0000003cdbae592d in clone () from /lib64/libc.so.6
(gdb) f 4
#4  0x0000000004c5a10e in __gf_free (free_ptr=0xcd8cda0) at ../../../libglusterfs/src/mem-pool.c:288
288	                GF_ASSERT (0);
(gdb) l
283	        ptr = ptr - sizeof(xlator_t *);
284	        memcpy (&xl, ptr, sizeof(xlator_t *));
285	
286	        if (!xl) {
287	                //gf_free expects xl to be available
288	                GF_ASSERT (0);
289	        }
290	
291	        if (!xl->mem_acct.rec) {
292	                ptr = (char *)free_ptr - GF_MEM_HEADER_SIZE;
(gdb) p xl
$1 = (xlator_t *) 0x0
(gdb) p free_ptr
$2 = (void *) 0xcd8cda0
(gdb) p free_ptr(free_ptrcfree_ptrxfree_ptr[1Pfree_ptrhfree_ptrafree_ptrrfree_ptr free_ptr*free_ptr)free_ptr
$3 = 0xcd8cda0 "gqas007.sbu.lab.eng.bos.redhat.com-11320-2012/03/19-09:45:37:821860-mirror-client-0-97"
(gdb) up
#5  0x000000000a252c68 in server_connection_destroy (this=0x65a71a0, conn=0xcd8ccc0)
    at ../../../../../xlators/protocol/server/src/server-helpers.c:627
627	        GF_FREE (conn->id);
(gdb) p conn->id
$4 = 0xcd8cda0 "gqas007.sbu.lab.eng.bos.redhat.com-11320-2012/03/19-09:45:37:821860-mirror-client-0-97"
(gdb) p *conn
$5 = {list = {next = 0xcd8ccc0, prev = 0xcd8ccc0}, 
  id = 0xcd8cda0 "gqas007.sbu.lab.eng.bos.redhat.com-11320-2012/03/19-09:45:37:821860-mirror-client-0-97", ref = 0, bind_ref = 0, lock = {
    __data = {__lock = 0, __count = 0, __owner = 0, __nusers = 0, __kind = -1, __spins = 0, __list = {__prev = 0x0, __next = 0x0}}, 
    __size = '\000' <repeats 16 times>"\377, \377\377\377", '\000' <repeats 19 times>, __align = 0}, fdtable = 0x0, ltable = 0x0, 
  timer = 0x0, bound_xl = 0x65a5d30, this = 0x65a71a0, lk_version = 2}
(gdb) l
622	
623	        gf_log (this->name, GF_LOG_INFO, "destroyed connection of %s",
624	                conn->id);
625	
626	        pthread_mutex_destroy (&conn->lock);
627	        GF_FREE (conn->id);
628	        GF_FREE (conn);
629	        ret = 0;
630	out:
631	        return ret;
(gdb) down
#4  0x0000000004c5a10e in __gf_free (free_ptr=0xcd8cda0) at ../../../libglusterfs/src/mem-pool.c:288
288	                GF_ASSERT (0);

Version-Release number of selected component (if applicable):


How reproducible:


Steps to Reproduce:
1.
2.
3.
  
Actual results:

glusterfs server crashed while freeing the connection object.

Expected results:

glusterfs server should not crash.
Additional info: