Attachment 585337 Details for Bug 807996 – backtrace of libvirtd hang on one side

backtrace of libvirtd hang on one side

backtrace-hang-one-side (text/plain), 13.11 KB, created by weizhang on 2012-05-18 05:34:13 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: weizhang

Created: 2012-05-18 05:34:13 UTC

Size: 13.11 KB

patch

obsolete

>
>Thread 12 (Thread 0x7fed816f7700 (LWP 10425)):
>#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
>#1  0x000000302a658f56 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117
>#2  0x000000302a6594f3 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103
>#3  0x000000302a658d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#4  0x0000003cf5807851 in start_thread (arg=0x7fed816f7700) at pthread_create.c:301
>#5  0x0000003cf54e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>Thread 11 (Thread 0x7fed80cf6700 (LWP 10426)):
>#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
>#1  0x000000302a658f56 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117
>#2  0x000000302a6594f3 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103
>#3  0x000000302a658d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#4  0x0000003cf5807851 in start_thread (arg=0x7fed80cf6700) at pthread_create.c:301
>#5  0x0000003cf54e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>Thread 10 (Thread 0x7fed802f5700 (LWP 10427)):
>#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
>#1  0x000000302a658f56 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117
>#2  0x000000302a6594f3 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103
>#3  0x000000302a658d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#4  0x0000003cf5807851 in start_thread (arg=0x7fed802f5700) at pthread_create.c:301
>#5  0x0000003cf54e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>Thread 9 (Thread 0x7fed7f8f4700 (LWP 10428)):
>#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
>#1  0x000000302a658f56 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117
>#2  0x000000302a6594f3 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103
>#3  0x000000302a658d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#4  0x0000003cf5807851 in start_thread (arg=0x7fed7f8f4700) at pthread_create.c:301
>#5  0x0000003cf54e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>Thread 8 (Thread 0x7fed7eef3700 (LWP 10429)):
>#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:136
>#1  0x0000003cf5809388 in _L_lock_854 () from /lib64/libpthread.so.0
>#2  0x0000003cf5809257 in __pthread_mutex_lock (mutex=0x7fed5c05dc30) at pthread_mutex_lock.c:61
>#3  0x000000302a6f847a in remoteDriverLock (conn=<value optimized out>) at remote/remote_driver.c:100
>#4  remoteIsAlive (conn=<value optimized out>) at remote/remote_driver.c:4503
>#5  0x000000302a6c018c in virConnectIsAlive (conn=0x7fed5c0399f0) at libvirt.c:18423
>#6  0x00000000004939ad in qemuMigrationWaitForCompletion (driver=0x7fed7400d9a0, vm=0x7fed741d1390, asyncJob=QEMU_ASYNC_JOB_MIGRATION_OUT, dconn=0x7fed5c0399f0)
>    at qemu/qemu_migration.c:987
>#7  0x00000000004965fd in qemuMigrationRun (driver=0x7fed7400d9a0, vm=0x7fed741d1390, cookiein=<value optimized out>, cookieinlen=<value optimized out>, cookieout=0x7fed7eef28c0, 
>    cookieoutlen=0x7fed7eef28dc, flags=519, resource=0, spec=0x7fed7eef2780, dconn=0x7fed5c0399f0) at qemu/qemu_migration.c:1955
>#8  0x0000000000497300 in doTunnelMigrate (driver=0x7fed7400d9a0, vm=0x7fed741d1390, st=0x7fed5c119030, 
>    cookiein=0x7fed5c119680 "<qemu-migration>\n  <name>win2008-x86_64</name>\n  <uuid>655b27e9-86ba-9ebd-d988-feefecfcfaf9</uuid>\n  <hostname>intel-5310-32-2.englab.nay.redhat.com</hostname>\n  <hostuuid>44454c4c-5900-1051-8031-c2c0"..., cookieinlen=295, cookieout=0x7fed7eef28c0, cookieoutlen=0x7fed7eef28dc, flags=519, resource=0, dconn=0x7fed5c0399f0)
>    at qemu/qemu_migration.c:2145
>#9  0x000000000049a378 in doPeer2PeerMigrate3 (driver=0x7fed7400d9a0, conn=0x7fed68057070, vm=0x7fed741d1390, xmlin=<value optimized out>, 
>    dconnuri=0x7fed5c05dc00 "qemu+tls://10.66.83.193/system", uri=0x0, cookiein=0x0, cookieinlen=0, cookieout=0x7fed7eef2b30, cookieoutlen=0x7fed7eef2b3c, flags=519, dname=0x0, 
>    resource=0, v3proto=true) at qemu/qemu_migration.c:2393
>#10 doPeer2PeerMigrate (driver=0x7fed7400d9a0, conn=0x7fed68057070, vm=0x7fed741d1390, xmlin=<value optimized out>, dconnuri=0x7fed5c05dc00 "qemu+tls://10.66.83.193/system", 
>---Type <return> to continue, or q <return> to quit---
>    uri=0x0, cookiein=0x0, cookieinlen=0, cookieout=0x7fed7eef2b30, cookieoutlen=0x7fed7eef2b3c, flags=519, dname=0x0, resource=0, v3proto=true) at qemu/qemu_migration.c:2566
>#11 qemuMigrationPerformJob (driver=0x7fed7400d9a0, conn=0x7fed68057070, vm=0x7fed741d1390, xmlin=<value optimized out>, dconnuri=0x7fed5c05dc00 "qemu+tls://10.66.83.193/system", 
>    uri=0x0, cookiein=0x0, cookieinlen=0, cookieout=0x7fed7eef2b30, cookieoutlen=0x7fed7eef2b3c, flags=519, dname=0x0, resource=0, v3proto=true) at qemu/qemu_migration.c:2630
>#12 0x000000000044c253 in qemuDomainMigratePerform3 (dom=0x7fed5c05dbc0, xmlin=0x0, cookiein=<value optimized out>, cookieinlen=0, cookieout=0x7fed7eef2b30, 
>    cookieoutlen=0x7fed7eef2b3c, dconnuri=0x7fed5c05dc00 "qemu+tls://10.66.83.193/system", uri=0x0, flags=519, dname=0x0, resource=0) at qemu/qemu_driver.c:9070
>#13 0x000000302a6d8994 in virDomainMigratePerform3 (domain=0x7fed5c05dbc0, xmlin=0x0, cookiein=0x0, cookieinlen=<value optimized out>, cookieout=<value optimized out>, 
>    cookieoutlen=0x7fed7eef2b3c, dconnuri=0x7fed5c05dc00 "qemu+tls://10.66.83.193/system", uri=0x0, flags=519, dname=0x0, bandwidth=0) at libvirt.c:6265
>#14 0x0000000000427fd2 in remoteDispatchDomainMigratePerform3 (server=<value optimized out>, client=<value optimized out>, msg=<value optimized out>, rerr=0x7fed7eef2bc0, 
>    args=<value optimized out>, ret=0x7fed5c199b50) at remote.c:3289
>#15 remoteDispatchDomainMigratePerform3Helper (server=<value optimized out>, client=<value optimized out>, msg=<value optimized out>, rerr=0x7fed7eef2bc0, 
>    args=<value optimized out>, ret=0x7fed5c199b50) at remote_dispatch.h:3293
>#16 0x000000302a714bf5 in virNetServerProgramDispatchCall (prog=0x2436f90, server=0x242bd00, client=0x25a1850, msg=0x2477290) at rpc/virnetserverprogram.c:416
>#17 virNetServerProgramDispatch (prog=0x2436f90, server=0x242bd00, client=0x25a1850, msg=0x2477290) at rpc/virnetserverprogram.c:289
>#18 0x000000302a715e81 in virNetServerHandleJob (jobOpaque=<value optimized out>, opaque=0x242bd00) at rpc/virnetserver.c:164
>#19 0x000000302a65945c in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:144
>#20 0x000000302a658d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#21 0x0000003cf5807851 in start_thread (arg=0x7fed7eef3700) at pthread_create.c:301
>#22 0x0000003cf54e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>Thread 7 (Thread 0x7fed7e4f2700 (LWP 10430)):
>#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
>#1  0x000000302a658f56 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117
>#2  0x000000302a6594f3 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103
>#3  0x000000302a658d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#4  0x0000003cf5807851 in start_thread (arg=0x7fed7e4f2700) at pthread_create.c:301
>#5  0x0000003cf54e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>Thread 6 (Thread 0x7fed7daf1700 (LWP 10431)):
>#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
>#1  0x000000302a658f56 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117
>#2  0x000000302a6594f3 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103
>#3  0x000000302a658d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#4  0x0000003cf5807851 in start_thread (arg=0x7fed7daf1700) at pthread_create.c:301
>#5  0x0000003cf54e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>Thread 5 (Thread 0x7fed7d0f0700 (LWP 10432)):
>#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
>#1  0x000000302a658f56 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117
>#2  0x000000302a6594f3 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103
>#3  0x000000302a658d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#4  0x0000003cf5807851 in start_thread (arg=0x7fed7d0f0700) at pthread_create.c:301
>#5  0x0000003cf54e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>Thread 4 (Thread 0x7fed7c6ef700 (LWP 10433)):
>#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
>#1  0x000000302a658f56 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117
>#2  0x000000302a6594f3 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103
>#3  0x000000302a658d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#4  0x0000003cf5807851 in start_thread (arg=0x7fed7c6ef700) at pthread_create.c:301
>#5  0x0000003cf54e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>Thread 3 (Thread 0x7fed7bcee700 (LWP 10434)):
>---Type <return> to continue, or q <return> to quit---
>#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
>#1  0x000000302a658f56 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117
>#2  0x000000302a6594f3 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103
>#3  0x000000302a658d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#4  0x0000003cf5807851 in start_thread (arg=0x7fed7bcee700) at pthread_create.c:301
>#5  0x0000003cf54e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>Thread 2 (Thread 0x7fed7b2e6700 (LWP 15534)):
>#0  0x0000003cf54ddfc3 in __poll (fds=<value optimized out>, nfds=<value optimized out>, timeout=<value optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
>#1  0x000000302a711047 in virNetClientIOEventLoop (client=0x7fed5c19a0b0, thiscall=0x7fed48050cf0) at rpc/virnetclient.c:1352
>#2  0x000000302a7119b1 in virNetClientIO (client=0x7fed5c19a0b0, msg=<value optimized out>, expectReply=false, nonBlock=false) at rpc/virnetclient.c:1607
>#3  virNetClientSendInternal (client=0x7fed5c19a0b0, msg=<value optimized out>, expectReply=false, nonBlock=false) at rpc/virnetclient.c:1715
>#4  0x000000302a711f44 in virNetClientSendNoReply (client=0x7fed5c19a0b0, msg=0x7fed48010940) at rpc/virnetclient.c:1769
>#5  0x000000302a71298d in virNetClientStreamSendPacket (st=0x7fed5c1195a0, client=0x7fed5c19a0b0, status=2, data=0x7fed480008c0 "T$0\271@", nbytes=65536)
>    at rpc/virnetclientstream.c:357
>#6  0x000000302a6f810b in remoteStreamSend (st=<value optimized out>, data=0x7fed480008c0 "T$0\271@", nbytes=65536) at remote/remote_driver.c:3747
>#7  0x000000302a6c237c in virStreamSend (stream=0x7fed5c119030, data=0x7fed480008c0 "T$0\271@", nbytes=65536) at libvirt.c:14865
>#8  0x000000000049470a in qemuMigrationIOFunc (arg=0x7fed5c118fb0) at qemu/qemu_migration.c:1664
>#9  0x000000302a658d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#10 0x0000003cf5807851 in start_thread (arg=0x7fed7b2e6700) at pthread_create.c:301
>#11 0x0000003cf54e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>Thread 1 (Thread 0x7fed87589860 (LWP 10424)):
>#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
>#1  0x000000302a658f56 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117
>#2  0x000000302a711b3b in virNetClientIO (client=0x7fed5c19a0b0, msg=0x25a1a20, expectReply=false, nonBlock=true) at rpc/virnetclient.c:1549
>#3  virNetClientSendInternal (client=0x7fed5c19a0b0, msg=0x25a1a20, expectReply=false, nonBlock=true) at rpc/virnetclient.c:1715
>#4  0x000000302a711ea7 in virNetClientSendNonBlock (client=<value optimized out>, msg=0x25a1a20) at rpc/virnetclient.c:1791
>#5  0x000000302a711edd in virNetClientKeepAliveSendCB (opaque=<value optimized out>, msg=0x25a1a20) at rpc/virnetclient.c:263
>#6  0x000000302a7213c6 in virKeepAliveSend (ka=0x7fed5c0594d0, msg=0x25a1a20) at rpc/virkeepalive.c:127
>#7  0x000000302a721535 in virKeepAliveTimer (timer=<value optimized out>, opaque=0x7fed5c0594d0) at rpc/virkeepalive.c:195
>#8  0x000000302a648398 in virEventPollDispatchTimeouts () at util/event_poll.c:440
>#9  virEventPollRunOnce () at util/event_poll.c:633
>#10 0x000000302a6473b7 in virEventRunDefaultImpl () at util/event.c:247
>#11 0x000000302a71544d in virNetServerRun (srv=0x242bd00) at rpc/virnetserver.c:736
>#12 0x0000000000422421 in main (argc=<value optimized out>, argv=<value optimized out>) at libvirtd.c:1615
>

Actions: View

Attachments on bug 807996: 573615 | 573621 | 573622 | 573890 | 573891 | 582896 | 582897 | 585337 | 585338