Attachment 585107 Details for Bug 799478 – the backtrace for libvirtd hang

the backtrace for libvirtd hang

backtrace-hang (text/plain), 12.64 KB, created by weizhang on 2012-05-17 06:07:00 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: weizhang

Created: 2012-05-17 06:07:00 UTC

Size: 12.64 KB

patch

obsolete

>Thread 13 (Thread 0x7f810bfff700 (LWP 12959)):
>#0  0x0000003cf580e8ad in connect () at ../sysdeps/unix/syscall-template.S:82
>#1  0x00000000004a13c4 in qemuMonitorOpenUnix (vm=<value optimized out>, config=<value optimized out>, json=<value optimized out>, cb=<value optimized out>)
>    at qemu/qemu_monitor.c:253
>#2  qemuMonitorOpen (vm=<value optimized out>, config=<value optimized out>, json=<value optimized out>, cb=<value optimized out>) at qemu/qemu_monitor.c:692
>#3  0x000000000048b18c in qemuConnectMonitor (driver=0x7f810c00a260, vm=0x7f810c151830) at qemu/qemu_process.c:1099
>#4  0x000000000048fed2 in qemuProcessReconnect (opaque=<value optimized out>) at qemu/qemu_process.c:2953
>#5  0x000000302a658d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#6  0x0000003cf5807851 in start_thread (arg=0x7f810bfff700) at pthread_create.c:301
>#7  0x0000003cf54e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>Thread 11 (Thread 0x7f811165e700 (LWP 12898)):
>#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
>#1  0x000000302a658f56 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117
>#2  0x000000302a6594f3 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103
>#3  0x000000302a658d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#4  0x0000003cf5807851 in start_thread (arg=0x7f811165e700) at pthread_create.c:301
>#5  0x0000003cf54e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>Thread 10 (Thread 0x7f811205f700 (LWP 12897)):
>#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
>#1  0x000000302a658f56 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117
>#2  0x000000302a6594f3 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103
>#3  0x000000302a658d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#4  0x0000003cf5807851 in start_thread (arg=0x7f811205f700) at pthread_create.c:301
>#5  0x0000003cf54e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>Thread 9 (Thread 0x7f8112a60700 (LWP 12896)):
>#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
>#1  0x000000302a658f56 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117
>#2  0x000000302a6594f3 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103
>#3  0x000000302a658d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#4  0x0000003cf5807851 in start_thread (arg=0x7f8112a60700) at pthread_create.c:301
>#5  0x0000003cf54e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>Thread 8 (Thread 0x7f8113461700 (LWP 12895)):
>#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
>#1  0x000000302a658f56 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117
>#2  0x000000302a6594f3 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103
>#3  0x000000302a658d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#4  0x0000003cf5807851 in start_thread (arg=0x7f8113461700) at pthread_create.c:301
>#5  0x0000003cf54e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>Thread 7 (Thread 0x7f8113e62700 (LWP 12894)):
>#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
>#1  0x000000302a658f56 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117
>#2  0x000000302a6594f3 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103
>#3  0x000000302a658d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#4  0x0000003cf5807851 in start_thread (arg=0x7f8113e62700) at pthread_create.c:301
>#5  0x0000003cf54e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>---Type <return> to continue, or q <return> to quit--- 
>Thread 6 (Thread 0x7f8114863700 (LWP 12893)):
>#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
>#1  0x000000302a658f56 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117
>#2  0x000000302a6594f3 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103
>#3  0x000000302a658d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#4  0x0000003cf5807851 in start_thread (arg=0x7f8114863700) at pthread_create.c:301
>#5  0x0000003cf54e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>Thread 5 (Thread 0x7f8115264700 (LWP 12892)):
>#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
>#1  0x000000302a658f56 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117
>#2  0x000000302a6594f3 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103
>#3  0x000000302a658d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#4  0x0000003cf5807851 in start_thread (arg=0x7f8115264700) at pthread_create.c:301
>#5  0x0000003cf54e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>Thread 4 (Thread 0x7f8115c65700 (LWP 12891)):
>#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
>#1  0x000000302a658f56 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117
>#2  0x000000302a6594f3 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103
>#3  0x000000302a658d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#4  0x0000003cf5807851 in start_thread (arg=0x7f8115c65700) at pthread_create.c:301
>#5  0x0000003cf54e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>Thread 3 (Thread 0x7f8116666700 (LWP 12890)):
>#0  0x0000003cf580ed8d in open64 () at ../sysdeps/unix/syscall-template.S:82
>#1  0x000000302a65e70d in open (path=0x7f80f410a950 "/var/run/libvirt/qemu/kvm-rhel6u3-x86_64.xml", mode=384, rewrite=0x302a6603f0 <virXMLRewriteFile>, opaque=0x7f8116665670)
>    at /usr/include/bits/fcntl2.h:54
>#2  virFileRewrite (path=0x7f80f410a950 "/var/run/libvirt/qemu/kvm-rhel6u3-x86_64.xml", mode=384, rewrite=0x302a6603f0 <virXMLRewriteFile>, opaque=0x7f8116665670)
>    at util/virfile.c:379
>#3  0x000000302a6603e6 in virXMLSaveFile (path=<value optimized out>, warnName=<value optimized out>, warnCommand=<value optimized out>, xml=<value optimized out>) at util/xml.c:847
>#4  0x000000302a677067 in virDomainSaveXML (configDir=0x7f810c08d020 "/var/run/libvirt/qemu", def=0x7f810c017d40, 
>    xml=0x7f80f410d520 "<domstatus state='running' reason='booted' pid='12295'>\n  <monitor path='/var/lib/libvirt/qemu/kvm-rhel6u3-x86_64.monitor' json='1' type='unix'/>\n  <vcpus>\n    <vcpu pid='12319'/>\n  </vcpus>\n  <qemuCa"...) at conf/domain_conf.c:12874
>#5  0x000000302a692169 in virDomainSaveStatus (caps=0x7f810c0cc810, statusDir=0x7f810c08d020 "/var/run/libvirt/qemu", obj=<value optimized out>) at conf/domain_conf.c:12915
>#6  0x000000000047e4b8 in qemuDomainObjSaveJob (driver=<value optimized out>, obj=<value optimized out>) at qemu/qemu_domain.c:645
>#7  0x0000000000497f9a in qemuMigrationConfirm (driver=0x7f810c00a260, conn=0x7f8100000bd0, vm=0x7f810c0cb8f0, cookiein=0x0, cookieinlen=0, flags=<value optimized out>, retcode=1)
>    at qemu/qemu_migration.c:3106
>#8  0x0000000000499add in doPeer2PeerMigrate3 (driver=0x7f810c00a260, conn=0x7f8100000bd0, vm=0x7f810c0cb8f0, xmlin=<value optimized out>, 
>    dconnuri=0x7f80f40008c0 "qemu+tls://10.66.83.193/system", uri=<value optimized out>, cookiein=0x0, cookieinlen=0, cookieout=0x7f8116665b30, cookieoutlen=0x7f8116665b3c, 
>    flags=3, dname=0x7f810c018060 "kvm-rhel6u3-x86_64", resource=0, v3proto=true) at qemu/qemu_migration.c:2462
>#9  doPeer2PeerMigrate (driver=0x7f810c00a260, conn=0x7f8100000bd0, vm=0x7f810c0cb8f0, xmlin=<value optimized out>, dconnuri=0x7f80f40008c0 "qemu+tls://10.66.83.193/system", 
>    uri=<value optimized out>, cookiein=0x0, cookieinlen=0, cookieout=0x7f8116665b30, cookieoutlen=0x7f8116665b3c, flags=3, dname=0x7f810c018060 "kvm-rhel6u3-x86_64", resource=0, 
>    v3proto=true) at qemu/qemu_migration.c:2566
>#10 qemuMigrationPerformJob (driver=0x7f810c00a260, conn=0x7f8100000bd0, vm=0x7f810c0cb8f0, xmlin=<value optimized out>, dconnuri=0x7f80f40008c0 "qemu+tls://10.66.83.193/system", 
>    uri=<value optimized out>, cookiein=0x0, cookieinlen=0, cookieout=0x7f8116665b30, cookieoutlen=0x7f8116665b3c, flags=3, dname=0x7f810c018060 "kvm-rhel6u3-x86_64", resource=0, 
>    v3proto=true) at qemu/qemu_migration.c:2630
>#11 0x000000000044c253 in qemuDomainMigratePerform3 (dom=0x7f80f4000b40, xmlin=0x0, cookiein=<value optimized out>, cookieinlen=0, cookieout=0x7f8116665b30, 
>    cookieoutlen=0x7f8116665b3c, dconnuri=0x7f80f40008c0 "qemu+tls://10.66.83.193/system", uri=0x0, flags=3, dname=0x0, resource=0) at qemu/qemu_driver.c:9070
>#12 0x000000302a6d8994 in virDomainMigratePerform3 (domain=0x7f80f4000b40, xmlin=0x0, cookiein=0x0, cookieinlen=<value optimized out>, cookieout=<value optimized out>, 
>    cookieoutlen=0x7f8116665b3c, dconnuri=0x7f80f40008c0 "qemu+tls://10.66.83.193/system", uri=0x0, flags=3, dname=0x0, bandwidth=0) at libvirt.c:6265
>#13 0x0000000000427fd2 in remoteDispatchDomainMigratePerform3 (server=<value optimized out>, client=<value optimized out>, msg=<value optimized out>, rerr=0x7f8116665bc0, 
>---Type <return> to continue, or q <return> to quit---
>    args=<value optimized out>, ret=0x7f80f4000ad0) at remote.c:3289
>#14 remoteDispatchDomainMigratePerform3Helper (server=<value optimized out>, client=<value optimized out>, msg=<value optimized out>, rerr=0x7f8116665bc0, 
>    args=<value optimized out>, ret=0x7f80f4000ad0) at remote_dispatch.h:3293
>#15 0x000000302a714bf5 in virNetServerProgramDispatchCall (prog=0x1c2ef60, server=0x1c23cd0, client=0x1c2a510, msg=0x1c2f1e0) at rpc/virnetserverprogram.c:416
>#16 virNetServerProgramDispatch (prog=0x1c2ef60, server=0x1c23cd0, client=0x1c2a510, msg=0x1c2f1e0) at rpc/virnetserverprogram.c:289
>#17 0x000000302a715e81 in virNetServerHandleJob (jobOpaque=<value optimized out>, opaque=0x1c23cd0) at rpc/virnetserver.c:164
>#18 0x000000302a65945c in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:144
>#19 0x000000302a658d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#20 0x0000003cf5807851 in start_thread (arg=0x7f8116666700) at pthread_create.c:301
>#21 0x0000003cf54e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>Thread 2 (Thread 0x7f8117067700 (LWP 12889)):
>#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
>#1  0x000000302a658f56 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117
>#2  0x000000302a6594f3 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103
>#3  0x000000302a658d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#4  0x0000003cf5807851 in start_thread (arg=0x7f8117067700) at pthread_create.c:301
>#5  0x0000003cf54e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>Thread 1 (Thread 0x7f811cef9860 (LWP 12886)):
>#0  0x000000302a711933 in virNetClientCallQueue (client=0x7f80f400db10, msg=0x7f80f418d600, expectReply=false, nonBlock=true) at rpc/virnetclient.c:132
>#1  virNetClientIO (client=0x7f80f400db10, msg=0x7f80f418d600, expectReply=false, nonBlock=true) at rpc/virnetclient.c:1533
>#2  virNetClientSendInternal (client=0x7f80f400db10, msg=0x7f80f418d600, expectReply=false, nonBlock=true) at rpc/virnetclient.c:1715
>#3  0x000000302a711ea7 in virNetClientSendNonBlock (client=<value optimized out>, msg=0x7f80f418d600) at rpc/virnetclient.c:1791
>#4  0x000000302a711edd in virNetClientKeepAliveSendCB (opaque=<value optimized out>, msg=0x7f80f418d600) at rpc/virnetclient.c:263
>#5  0x000000302a7213c6 in virKeepAliveSend (ka=0x7f80f4004860, msg=0x7f80f418d600) at rpc/virkeepalive.c:127
>#6  0x000000302a72172d in virKeepAliveResponseTimer (timer=<value optimized out>, opaque=0x7f80f4004860) at rpc/virkeepalive.c:218
>#7  0x000000302a648398 in virEventPollDispatchTimeouts () at util/event_poll.c:440
>#8  virEventPollRunOnce () at util/event_poll.c:633
>#9  0x000000302a6473b7 in virEventRunDefaultImpl () at util/event.c:247
>#10 0x000000302a71544d in virNetServerRun (srv=0x1c23cd0) at rpc/virnetserver.c:736
>#11 0x0000000000422421 in main (argc=<value optimized out>, argv=<value optimized out>) at libvirtd.c:1615
>

Actions: View

Attachments on bug 799478: 585107