Attachment 585338 Details for Bug 807996 – backtrace of libvirtd hang on other side

backtrace of libvirtd hang on other side

backtrace-hang-other-side (text/plain), 14.86 KB, created by weizhang on 2012-05-18 05:35:01 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: weizhang

Created: 2012-05-18 05:35:01 UTC

Size: 14.86 KB

patch

obsolete

>
>Thread 12 (Thread 0x7f2a34ee8700 (LWP 1339)):
>#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
>#1  0x0000003006a58f56 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117
>#2  0x0000003006a594f3 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103
>#3  0x0000003006a58d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#4  0x00000035fb807851 in start_thread (arg=0x7f2a34ee8700) at pthread_create.c:301
>#5  0x00000035fb4e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>Thread 11 (Thread 0x7f2a344e7700 (LWP 1340)):
>#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
>#1  0x0000003006a58f56 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117
>#2  0x0000003006a594f3 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103
>#3  0x0000003006a58d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#4  0x00000035fb807851 in start_thread (arg=0x7f2a344e7700) at pthread_create.c:301
>#5  0x00000035fb4e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>Thread 10 (Thread 0x7f2a33ae6700 (LWP 1341)):
>#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:136
>#1  0x00000035fb809388 in _L_lock_854 () from /lib64/libpthread.so.0
>#2  0x00000035fb809257 in __pthread_mutex_lock (mutex=0x7f2a2800a380) at pthread_mutex_lock.c:61
>#3  0x0000000000452c22 in qemuDomainGetJobInfo (dom=<value optimized out>, info=0x7f2a33ae5ae0) at qemu/qemu_driver.c:9371
>#4  0x0000003006ad4107 in virDomainGetJobInfo (domain=0x7f2a14051090, info=0x7f2a33ae5ae0) at libvirt.c:16418
>#5  0x000000000042c855 in remoteDispatchDomainGetJobInfo (server=<value optimized out>, client=0x1a8f7a0, msg=<value optimized out>, rerr=0x7f2a33ae5bc0, 
>    args=<value optimized out>, ret=0x7f2a1409ac90) at remote_dispatch.h:1722
>#6  remoteDispatchDomainGetJobInfoHelper (server=<value optimized out>, client=0x1a8f7a0, msg=<value optimized out>, rerr=0x7f2a33ae5bc0, args=<value optimized out>, 
>    ret=0x7f2a1409ac90) at remote_dispatch.h:1698
>#7  0x0000003006b14bf5 in virNetServerProgramDispatchCall (prog=0x1946f60, server=0x193bcd0, client=0x1a8f7a0, msg=0x1a0d610) at rpc/virnetserverprogram.c:416
>#8  virNetServerProgramDispatch (prog=0x1946f60, server=0x193bcd0, client=0x1a8f7a0, msg=0x1a0d610) at rpc/virnetserverprogram.c:289
>#9  0x0000003006b15e81 in virNetServerHandleJob (jobOpaque=<value optimized out>, opaque=0x193bcd0) at rpc/virnetserver.c:164
>#10 0x0000003006a5945c in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:144
>#11 0x0000003006a58d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#12 0x00000035fb807851 in start_thread (arg=0x7f2a33ae6700) at pthread_create.c:301
>#13 0x00000035fb4e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>Thread 9 (Thread 0x7f2a330e5700 (LWP 1342)):
>#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
>#1  0x0000003006a58f56 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117
>#2  0x0000003006a594f3 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103
>#3  0x0000003006a58d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#4  0x00000035fb807851 in start_thread (arg=0x7f2a330e5700) at pthread_create.c:301
>#5  0x00000035fb4e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>Thread 8 (Thread 0x7f2a326e4700 (LWP 1343)):
>#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:136
>#1  0x00000035fb809388 in _L_lock_854 () from /lib64/libpthread.so.0
>#2  0x00000035fb809257 in __pthread_mutex_lock (mutex=0x7f2a040924a0) at pthread_mutex_lock.c:61
>#3  0x0000003006af847a in remoteDriverLock (conn=<value optimized out>) at remote/remote_driver.c:100
>#4  remoteIsAlive (conn=<value optimized out>) at remote/remote_driver.c:4503
>#5  0x0000003006ac018c in virConnectIsAlive (conn=0x7f2a0409b5b0) at libvirt.c:18423
>#6  0x00000000004939ad in qemuMigrationWaitForCompletion (driver=0x7f2a2800a380, vm=0x7f2a241bd650, asyncJob=QEMU_ASYNC_JOB_MIGRATION_OUT, dconn=0x7f2a0409b5b0)
>    at qemu/qemu_migration.c:987
>---Type <return> to continue, or q <return> to quit---
>#7  0x00000000004965fd in qemuMigrationRun (driver=0x7f2a2800a380, vm=0x7f2a241bd650, cookiein=<value optimized out>, cookieinlen=<value optimized out>, cookieout=0x7f2a326e38c0, 
>    cookieoutlen=0x7f2a326e38dc, flags=519, resource=0, spec=0x7f2a326e3780, dconn=0x7f2a0409b5b0) at qemu/qemu_migration.c:1955
>#8  0x0000000000497300 in doTunnelMigrate (driver=0x7f2a2800a380, vm=0x7f2a241bd650, st=0x7f2a041d3d60, 
>    cookiein=0x7f2a041d1480 "<qemu-migration>\n  <name>kvm-rhel5u8-i386</name>\n  <uuid>3ca3f683-8ef8-d100-26f6-ff0ff1e2e721</uuid>\n  <hostname>intel-5310-32-1.englab.nay.redhat.com</hostname>\n  <hostuuid>44454c4c-5900-1051-8031-c3"..., cookieinlen=297, cookieout=0x7f2a326e38c0, cookieoutlen=0x7f2a326e38dc, flags=519, resource=0, dconn=0x7f2a0409b5b0)
>    at qemu/qemu_migration.c:2145
>#9  0x000000000049a378 in doPeer2PeerMigrate3 (driver=0x7f2a2800a380, conn=0x7f2a100c0ed0, vm=0x7f2a241bd650, xmlin=<value optimized out>, 
>    dconnuri=0x7f2a0407d420 "qemu+tls://10.66.83.191/system", uri=0x0, cookiein=0x0, cookieinlen=0, cookieout=0x7f2a326e3b30, cookieoutlen=0x7f2a326e3b3c, flags=519, dname=0x0, 
>    resource=0, v3proto=true) at qemu/qemu_migration.c:2393
>#10 doPeer2PeerMigrate (driver=0x7f2a2800a380, conn=0x7f2a100c0ed0, vm=0x7f2a241bd650, xmlin=<value optimized out>, dconnuri=0x7f2a0407d420 "qemu+tls://10.66.83.191/system", 
>    uri=0x0, cookiein=0x0, cookieinlen=0, cookieout=0x7f2a326e3b30, cookieoutlen=0x7f2a326e3b3c, flags=519, dname=0x0, resource=0, v3proto=true) at qemu/qemu_migration.c:2566
>#11 qemuMigrationPerformJob (driver=0x7f2a2800a380, conn=0x7f2a100c0ed0, vm=0x7f2a241bd650, xmlin=<value optimized out>, dconnuri=0x7f2a0407d420 "qemu+tls://10.66.83.191/system", 
>    uri=0x0, cookiein=0x0, cookieinlen=0, cookieout=0x7f2a326e3b30, cookieoutlen=0x7f2a326e3b3c, flags=519, dname=0x0, resource=0, v3proto=true) at qemu/qemu_migration.c:2630
>#12 0x000000000044c253 in qemuDomainMigratePerform3 (dom=0x7f2a0407d3e0, xmlin=0x0, cookiein=<value optimized out>, cookieinlen=0, cookieout=0x7f2a326e3b30, 
>    cookieoutlen=0x7f2a326e3b3c, dconnuri=0x7f2a0407d420 "qemu+tls://10.66.83.191/system", uri=0x0, flags=519, dname=0x0, resource=0) at qemu/qemu_driver.c:9070
>#13 0x0000003006ad8994 in virDomainMigratePerform3 (domain=0x7f2a0407d3e0, xmlin=0x0, cookiein=0x0, cookieinlen=<value optimized out>, cookieout=<value optimized out>, 
>    cookieoutlen=0x7f2a326e3b3c, dconnuri=0x7f2a0407d420 "qemu+tls://10.66.83.191/system", uri=0x0, flags=519, dname=0x0, bandwidth=0) at libvirt.c:6265
>#14 0x0000000000427fd2 in remoteDispatchDomainMigratePerform3 (server=<value optimized out>, client=<value optimized out>, msg=<value optimized out>, rerr=0x7f2a326e3bc0, 
>    args=<value optimized out>, ret=0x7f2a04000ce0) at remote.c:3289
>#15 remoteDispatchDomainMigratePerform3Helper (server=<value optimized out>, client=<value optimized out>, msg=<value optimized out>, rerr=0x7f2a326e3bc0, 
>    args=<value optimized out>, ret=0x7f2a04000ce0) at remote_dispatch.h:3293
>#16 0x0000003006b14bf5 in virNetServerProgramDispatchCall (prog=0x1946f60, server=0x193bcd0, client=0x1a8f7a0, msg=0x1a4d680) at rpc/virnetserverprogram.c:416
>#17 virNetServerProgramDispatch (prog=0x1946f60, server=0x193bcd0, client=0x1a8f7a0, msg=0x1a4d680) at rpc/virnetserverprogram.c:289
>#18 0x0000003006b15e81 in virNetServerHandleJob (jobOpaque=<value optimized out>, opaque=0x193bcd0) at rpc/virnetserver.c:164
>#19 0x0000003006a5945c in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:144
>#20 0x0000003006a58d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#21 0x00000035fb807851 in start_thread (arg=0x7f2a326e4700) at pthread_create.c:301
>#22 0x00000035fb4e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>Thread 7 (Thread 0x7f2a31ce3700 (LWP 1344)):
>#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
>#1  0x0000003006a58f56 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117
>#2  0x0000003006a594f3 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103
>#3  0x0000003006a58d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#4  0x00000035fb807851 in start_thread (arg=0x7f2a31ce3700) at pthread_create.c:301
>#5  0x00000035fb4e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>Thread 6 (Thread 0x7f2a312e2700 (LWP 1345)):
>#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
>#1  0x0000003006a58f56 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117
>#2  0x0000003006a594f3 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103
>#3  0x0000003006a58d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#4  0x00000035fb807851 in start_thread (arg=0x7f2a312e2700) at pthread_create.c:301
>#5  0x00000035fb4e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>Thread 5 (Thread 0x7f2a308e1700 (LWP 1346)):
>#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
>#1  0x0000003006a58f56 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117
>#2  0x0000003006a594f3 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103
>#3  0x0000003006a58d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#4  0x00000035fb807851 in start_thread (arg=0x7f2a308e1700) at pthread_create.c:301
>#5  0x00000035fb4e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>---Type <return> to continue, or q <return> to quit---
>
>Thread 4 (Thread 0x7f2a2fee0700 (LWP 1347)):
>#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
>#1  0x0000003006a58f56 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117
>#2  0x0000003006a594f3 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103
>#3  0x0000003006a58d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#4  0x00000035fb807851 in start_thread (arg=0x7f2a2fee0700) at pthread_create.c:301
>#5  0x00000035fb4e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>Thread 3 (Thread 0x7f2a2f4df700 (LWP 1348)):
>#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
>#1  0x0000003006a58f56 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117
>#2  0x0000003006a594f3 in virThreadPoolWorker (opaque=<value optimized out>) at util/threadpool.c:103
>#3  0x0000003006a58d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#4  0x00000035fb807851 in start_thread (arg=0x7f2a2f4df700) at pthread_create.c:301
>#5  0x00000035fb4e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>Thread 2 (Thread 0x7f2a2ead7700 (LWP 4191)):
>#0  0x00000035fb4ddfc3 in __poll (fds=<value optimized out>, nfds=<value optimized out>, timeout=<value optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
>#1  0x0000003006b11047 in virNetClientIOEventLoop (client=0x7f2a0410bf00, thiscall=0x7f2a00050ac0) at rpc/virnetclient.c:1352
>#2  0x0000003006b119b1 in virNetClientIO (client=0x7f2a0410bf00, msg=<value optimized out>, expectReply=false, nonBlock=false) at rpc/virnetclient.c:1607
>#3  virNetClientSendInternal (client=0x7f2a0410bf00, msg=<value optimized out>, expectReply=false, nonBlock=false) at rpc/virnetclient.c:1715
>#4  0x0000003006b11f44 in virNetClientSendNoReply (client=0x7f2a0410bf00, msg=0x7f2a00010940) at rpc/virnetclient.c:1769
>#5  0x0000003006b1298d in virNetClientStreamSendPacket (st=0x7f2a041d12c0, client=0x7f2a0410bf00, status=2, 
>    data=0x7f2a000008c0 "\tipv6_exec_ip -6 route del $networkipv6 via $gatewayipv6 dev $device\n\t\t\tlocal result=$?\n\t\tfi\n\tfi\n\n\tif [ $result -eq 2 ]; then\n\t\t# Netlink: \"No such process\"\n\t\ttrue\n\telif [ $result -ne 0 ]; then\n\t\tretu"..., nbytes=65536) at rpc/virnetclientstream.c:357
>#6  0x0000003006af810b in remoteStreamSend (st=<value optimized out>, 
>    data=0x7f2a000008c0 "\tipv6_exec_ip -6 route del $networkipv6 via $gatewayipv6 dev $device\n\t\t\tlocal result=$?\n\t\tfi\n\tfi\n\n\tif [ $result -eq 2 ]; then\n\t\t# Netlink: \"No such process\"\n\t\ttrue\n\telif [ $result -ne 0 ]; then\n\t\tretu"..., nbytes=65536) at remote/remote_driver.c:3747
>#7  0x0000003006ac237c in virStreamSend (stream=0x7f2a041d3d60, 
>    data=0x7f2a000008c0 "\tipv6_exec_ip -6 route del $networkipv6 via $gatewayipv6 dev $device\n\t\t\tlocal result=$?\n\t\tfi\n\tfi\n\n\tif [ $result -eq 2 ]; then\n\t\t# Netlink: \"No such process\"\n\t\ttrue\n\telif [ $result -ne 0 ]; then\n\t\tretu"..., nbytes=65536) at libvirt.c:14865
>#8  0x000000000049470a in qemuMigrationIOFunc (arg=0x7f2a041d13d0) at qemu/qemu_migration.c:1664
>#9  0x0000003006a58d79 in virThreadHelper (data=<value optimized out>) at util/threads-pthread.c:161
>#10 0x00000035fb807851 in start_thread (arg=0x7f2a2ead7700) at pthread_create.c:301
>#11 0x00000035fb4e767d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
>
>Thread 1 (Thread 0x7f2a3ad7a860 (LWP 1338)):
>#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
>#1  0x0000003006a58f56 in virCondWait (c=<value optimized out>, m=<value optimized out>) at util/threads-pthread.c:117
>#2  0x0000003006b11b3b in virNetClientIO (client=0x7f2a0410bf00, msg=0x1bb2570, expectReply=false, nonBlock=true) at rpc/virnetclient.c:1549
>#3  virNetClientSendInternal (client=0x7f2a0410bf00, msg=0x1bb2570, expectReply=false, nonBlock=true) at rpc/virnetclient.c:1715
>#4  0x0000003006b11ea7 in virNetClientSendNonBlock (client=<value optimized out>, msg=0x1bb2570) at rpc/virnetclient.c:1791
>#5  0x0000003006b11edd in virNetClientKeepAliveSendCB (opaque=<value optimized out>, msg=0x1bb2570) at rpc/virnetclient.c:263
>#6  0x0000003006b213c6 in virKeepAliveSend (ka=0x7f2a040cd0f0, msg=0x1bb2570) at rpc/virkeepalive.c:127
>#7  0x0000003006b21535 in virKeepAliveTimer (timer=<value optimized out>, opaque=0x7f2a040cd0f0) at rpc/virkeepalive.c:195
>#8  0x0000003006a48398 in virEventPollDispatchTimeouts () at util/event_poll.c:440
>#9  virEventPollRunOnce () at util/event_poll.c:633
>#10 0x0000003006a473b7 in virEventRunDefaultImpl () at util/event.c:247
>#11 0x0000003006b1544d in virNetServerRun (srv=0x193bcd0) at rpc/virnetserver.c:736
>#12 0x0000000000422421 in main (argc=<value optimized out>, argv=<value optimized out>) at libvirtd.c:1615
>

Actions: View

Attachments on bug 807996: 573615 | 573621 | 573622 | 573890 | 573891 | 582896 | 582897 | 585337 | 585338