Description of problem: qemu and guest hang when hotunplug a device with block commit running on it (data-plane enable) Version-Release number of selected component (if applicable): kernel version: 3.10.0-957.1.3.el7.x86_64 qemu-kvm-rhev version: qemu-kvm-rhev-2.12.0-18.el7_6.3.x86_64 How reproducible: 100% Steps to Reproduce: 1.Start guest with dataplane enable on data disk. /usr/libexec/qemu-kvm \ -name 'avocado-vt-vm1' \ -machine pc \ -nodefaults \ -device VGA,bus=pci.0,addr=0x2 \ -chardev socket,id=qmp_id_qmpmonitor1,path=/var/tmp/monitor-qmpmonitor1-20190111-062058-PzYcVzPD,server,nowait \ -mon chardev=qmp_id_qmpmonitor1,mode=control \ -chardev socket,id=qmp_id_catch_monitor,path=/var/tmp/monitor-catch_monitor-20190111-062058-PzYcVzPD,server,nowait \ -mon chardev=qmp_id_catch_monitor,mode=control \ -device pvpanic,ioport=0x505,id=idKlOyb8 \ -chardev socket,id=serial_id_serial0,path=/var/tmp/serial-serial0-20190111-062058-PzYcVzPD,server,nowait \ -device isa-serial,chardev=serial_id_serial0 \ -chardev socket,id=seabioslog_id_20190111-062058-PzYcVzPD,path=/var/tmp/seabios-20190111-062058-PzYcVzPD,server,nowait \ -device isa-debugcon,chardev=seabioslog_id_20190111-062058-PzYcVzPD,iobase=0x402 \ -device nec-usb-xhci,id=usb1,bus=pci.0,addr=0x3 \ -object iothread,id=iothread0 \ -blockdev node-name=file_node,driver=file,filename=/home/kvm_autotest_root/images/rhel76-64-virtio.qcow2,auto-read-only=on \ -blockdev node-name=drive_image1,file=file_node,driver=qcow2,auto-read-only=on \ -device virtio-blk-pci,id=image1,drive=drive_image1,bootindex=0,bus=pci.0,addr=0x4 \ -blockdev node-name=file_data1,driver=file,filename=/home/data1.qcow2,auto-read-only=on \ -blockdev node-name=drive_data1,file=file_data1,driver=qcow2,auto-read-only=on \ -device virtio-blk-pci,id=data1,drive=drive_data1,bus=pci.0 \ -blockdev node-name=file_data2,driver=file,filename=/home/data2.qcow2,auto-read-only=on \ -blockdev node-name=drive_data2,file=file_data2,driver=qcow2,auto-read-only=on \ -device virtio-blk-pci,id=data2,drive=drive_data2,bus=pci.0,iothread=iothread0 \ -device virtio-net-pci,mac=9a:bd:be:bf:c0:c1,id=id0tbMEZ,vectors=4,netdev=idSMuUwi,bus=pci.0 \ -netdev tap,id=idSMuUwi,vhost=on \ -m 4096 \ -smp 16,maxcpus=16,cores=8,threads=1,sockets=2 \ -cpu 'IvyBridge',+kvm_pv_unhalt \ -device usb-tablet,id=usb-tablet1,bus=usb1.0,port=1 \ -vnc :0 \ -rtc base=utc,clock=host,driftfix=slew \ -boot menu=off,strict=off,order=cdn,once=c \ -enable-kvm \ -monitor stdio \ 2. Create snapshot chain: base->sn4->sn6 {'execute':'blockdev-create','arguments':{'options': {'driver':'file','filename':'/root/sn$i','size':21474836480},'job-id':'job1'}} {'execute':'blockdev-add','arguments':{'driver':'file','node-name':'drive_sn$i','filename':'/root/sn$i'}} {'execute':'blockdev-create','arguments':{'options': {'driver': 'qcow2','file':'drive_sn$i','size':21474836480},'job-id':'job2'}} {'execute':'blockdev-add','arguments':{'driver':'qcow2','node-name':'sn$i','file':'drive_sn$i'}} {'execute':'job-dismiss','arguments':{'id':'job1'}} {'execute':'job-dismiss','arguments':{'id':'job2'}} {"execute":"blockdev-snapshot","arguments":{"node":"drive_data2","overlay":"sn5"}} {"execute":"blockdev-snapshot","arguments":{"node":"sn5","overlay":"sn6"}} 3. In guest, create a file on the test disk and do sync (guest)# dd if=/dev/urandom of=a bs=1M count=800 (guest)# sync 4. Do block commit, and during block_commit unplug the data disk. {'execute': 'block-commit', 'arguments': { 'device': 'sn6', 'base-node':'drive_data2', 'top-node':'sn6','speed':1,'job-id':'j3'}} {"execute": "device_del", "arguments": {"id":"data2"}} {"return": {}} Actual results: After step4, qemu and guest hang after device_del operation. Expected results: Device can be deleted successfully, and after that block_commit can executed successfully. And after block commit finished, drives can be unplugged. Additional info: 1.pstack info as bellow: # pstack 53275 Thread 21 (Thread 0x7fc8cf73a700 (LWP 53276)): #0 0x00007fc8dfbe21c9 in syscall () at /lib64/libc.so.6 #1 0x00005567639d5410 in qemu_event_wait (val=<optimized out>, f=<optimized out>) at /usr/src/debug/qemu-2.12.0/include/qemu/futex.h:29 #2 0x00005567639d5410 in qemu_event_wait (ev=ev@entry=0x55676465cbe8 <rcu_call_ready_event>) at util/qemu-thread-posix.c:445 #3 0x00005567639e593e in call_rcu_thread (opaque=<optimized out>) at util/rcu.c:261 #4 0x00007fc8dfebedd5 in start_thread () at /lib64/libpthread.so.0 #5 0x00007fc8dfbe7ead in clone () at /lib64/libc.so.6 Thread 20 (Thread 0x7fc8cef39700 (LWP 53277)): #0 0x00007fc8dfec54ed in __lll_lock_wait () at /lib64/libpthread.so.0 #1 0x00007fc8dfec0de6 in _L_lock_941 () at /lib64/libpthread.so.0 #2 0x00007fc8dfec0cdf in pthread_mutex_lock () at /lib64/libpthread.so.0 #3 0x00005567639d4d39 in qemu_mutex_lock_impl (mutex=mutex@entry=0x556765ae06a0, file=file@entry=0x556763b687df "util/async.c", line=line@entry=502) at util/qemu-thread-posix.c:67 #4 0x00005567639cfdfc in co_schedule_bh_cb (ctx=0x556765ae0640) at util/async.c:502 #5 0x00005567639cfdfc in co_schedule_bh_cb (opaque=0x556765ae0640) at util/async.c:390 #6 0x00005567639cff71 in aio_bh_poll (bh=0x556765ac7380) at util/async.c:90 #7 0x00005567639cff71 in aio_bh_poll (ctx=ctx@entry=0x556765ae0640) at util/async.c:118 #8 0x00005567639d31fb in aio_poll (ctx=0x556765ae0640, blocking=blocking@entry=true) at util/aio-posix.c:704 #9 0x00005567637a2d5e in iothread_run (opaque=0x556765acca80) at iothread.c:64 #10 0x00007fc8dfebedd5 in start_thread () at /lib64/libpthread.so.0 #11 0x00007fc8dfbe7ead in clone () at /lib64/libc.so.6 Thread 19 (Thread 0x7fc8cdd35700 (LWP 53295)): #0 0x00007fc8dfbdd2cf in ppoll () at /lib64/libc.so.6 #1 0x00005567639d140b in qemu_poll_ns (__ss=0x0, __timeout=0x0, __nfds=<optimized out>, __fds=<optimized out>) at /usr/include/bits/poll2.h:77 #2 0x00005567639d140b in qemu_poll_ns (fds=<optimized out>, nfds=<optimized out>, timeout=<optimized out>) at util/qemu-timer.c:322 #3 0x00005567639d3187 in aio_poll (ctx=0x556765ae0b40, blocking=blocking@entry=true) at util/aio-posix.c:645 #4 0x00005567637a2d5e in iothread_run (opaque=0x556765accc40) at iothread.c:64 #5 0x00007fc8dfebedd5 in start_thread () at /lib64/libpthread.so.0 #6 0x00007fc8dfbe7ead in clone () at /lib64/libc.so.6 Thread 18 (Thread 0x7fc8cd534700 (LWP 53296)): #0 0x00007fc8dfec54ed in __lll_lock_wait () at /lib64/libpthread.so.0 #1 0x00007fc8dfec0dcb in _L_lock_883 () at /lib64/libpthread.so.0 #2 0x00007fc8dfec0c98 in pthread_mutex_lock () at /lib64/libpthread.so.0 #3 0x00005567639d4d39 in qemu_mutex_lock_impl (mutex=mutex@entry=0x5567642230e0 <qemu_global_mutex>, file=file@entry=0x556763a69308 "/builddir/build/BUILD/qemu-2.12.0/cpus.c", line=line@entry=1765) at util/qemu-thread-posix.c:67 #4 0x00005567636b8298 in qemu_mutex_lock_iothread () at /usr/src/debug/qemu-2.12.0/cpus.c:1765 #5 0x00005567636dabfc in kvm_cpu_exec (cpu=cpu@entry=0x556765f42000) at /usr/src/debug/qemu-2.12.0/accel/kvm/kvm-all.c:2047 #6 0x00005567636b8446 in qemu_kvm_cpu_thread_fn (arg=0x556765f42000) at /usr/src/debug/qemu-2.12.0/cpus.c:1215 #7 0x00007fc8dfebedd5 in start_thread () at /lib64/libpthread.so.0 #8 0x00007fc8dfbe7ead in clone () at /lib64/libc.so.6 Thread 17 (Thread 0x7fc8ccd33700 (LWP 53297)): #0 0x00007fc8dfec54ed in __lll_lock_wait () at /lib64/libpthread.so.0 #1 0x00007fc8dfec0dcb in _L_lock_883 () at /lib64/libpthread.so.0 #2 0x00007fc8dfec0c98 in pthread_mutex_lock () at /lib64/libpthread.so.0 #3 0x00005567639d4d39 in qemu_mutex_lock_impl (mutex=mutex@entry=0x5567642230e0 <qemu_global_mutex>, file=file@entry=0x556763a69308 "/builddir/build/BUILD/qemu-2.12.0/cpus.c", line=line@entry=1765) at util/qemu-thread-posix.c:67 #4 0x00005567636b8298 in qemu_mutex_lock_iothread () at /usr/src/debug/qemu-2.12.0/cpus.c:1765 #5 0x00005567636dabfc in kvm_cpu_exec (cpu=cpu@entry=0x556765fa8000) at /usr/src/debug/qemu-2.12.0/accel/kvm/kvm-all.c:2047 #6 0x00005567636b8446 in qemu_kvm_cpu_thread_fn (arg=0x556765fa8000) at /usr/src/debug/qemu-2.12.0/cpus.c:1215 #7 0x00007fc8dfebedd5 in start_thread () at /lib64/libpthread.so.0 #8 0x00007fc8dfbe7ead in clone () at /lib64/libc.so.6 Thread 16 (Thread 0x7fc8cc532700 (LWP 53298)): #0 0x00007fc8dfec54ed in __lll_lock_wait () at /lib64/libpthread.so.0 #1 0x00007fc8dfec0dcb in _L_lock_883 () at /lib64/libpthread.so.0 #2 0x00007fc8dfec0c98 in pthread_mutex_lock () at /lib64/libpthread.so.0 #3 0x00005567639d4d39 in qemu_mutex_lock_impl (mutex=mutex@entry=0x5567642230e0 <qemu_global_mutex>, file=file@entry=0x556763a69308 "/builddir/build/BUILD/qemu-2.12.0/cpus.c", line=line@entry=1765) at util/qemu-thread-posix.c:67 #4 0x00005567636b8298 in qemu_mutex_lock_iothread () at /usr/src/debug/qemu-2.12.0/cpus.c:1765 #5 0x00005567636dabfc in kvm_cpu_exec (cpu=cpu@entry=0x556765fc8000) at /usr/src/debug/qemu-2.12.0/accel/kvm/kvm-all.c:2047 #6 0x00005567636b8446 in qemu_kvm_cpu_thread_fn (arg=0x556765fc8000) at /usr/src/debug/qemu-2.12.0/cpus.c:1215 #7 0x00007fc8dfebedd5 in start_thread () at /lib64/libpthread.so.0 #8 0x00007fc8dfbe7ead in clone () at /lib64/libc.so.6 Thread 15 (Thread 0x7fc8cbd31700 (LWP 53299)): #0 0x00007fc8dfec54ed in __lll_lock_wait () at /lib64/libpthread.so.0 #1 0x00007fc8dfec0dcb in _L_lock_883 () at /lib64/libpthread.so.0 #2 0x00007fc8dfec0c98 in pthread_mutex_lock () at /lib64/libpthread.so.0 #3 0x00005567639d4d39 in qemu_mutex_lock_impl (mutex=mutex@entry=0x5567642230e0 <qemu_global_mutex>, file=file@entry=0x556763a69308 "/builddir/build/BUILD/qemu-2.12.0/cpus.c", line=line@entry=1765) at util/qemu-thread-posix.c:67 #4 0x00005567636b8298 in qemu_mutex_lock_iothread () at /usr/src/debug/qemu-2.12.0/cpus.c:1765 #5 0x00005567636dabfc in kvm_cpu_exec (cpu=cpu@entry=0x556765fea000) at /usr/src/debug/qemu-2.12.0/accel/kvm/kvm-all.c:2047 #6 0x00005567636b8446 in qemu_kvm_cpu_thread_fn (arg=0x556765fea000) at /usr/src/debug/qemu-2.12.0/cpus.c:1215 #7 0x00007fc8dfebedd5 in start_thread () at /lib64/libpthread.so.0 #8 0x00007fc8dfbe7ead in clone () at /lib64/libc.so.6 Thread 14 (Thread 0x7fc8cb530700 (LWP 53300)): #0 0x00007fc8dfec54ed in __lll_lock_wait () at /lib64/libpthread.so.0 #1 0x00007fc8dfec0dcb in _L_lock_883 () at /lib64/libpthread.so.0 #2 0x00007fc8dfec0c98 in pthread_mutex_lock () at /lib64/libpthread.so.0 #3 0x00005567639d4d39 in qemu_mutex_lock_impl (mutex=mutex@entry=0x5567642230e0 <qemu_global_mutex>, file=file@entry=0x556763a69308 "/builddir/build/BUILD/qemu-2.12.0/cpus.c", line=line@entry=1765) at util/qemu-thread-posix.c:67 #4 0x00005567636b8298 in qemu_mutex_lock_iothread () at /usr/src/debug/qemu-2.12.0/cpus.c:1765 #5 0x00005567636dabfc in kvm_cpu_exec (cpu=cpu@entry=0x55676600e000) at /usr/src/debug/qemu-2.12.0/accel/kvm/kvm-all.c:2047 #6 0x00005567636b8446 in qemu_kvm_cpu_thread_fn (arg=0x55676600e000) at /usr/src/debug/qemu-2.12.0/cpus.c:1215 #7 0x00007fc8dfebedd5 in start_thread () at /lib64/libpthread.so.0 #8 0x00007fc8dfbe7ead in clone () at /lib64/libc.so.6 Thread 13 (Thread 0x7fc8cad2f700 (LWP 53301)): #0 0x00007fc8dfec54ed in __lll_lock_wait () at /lib64/libpthread.so.0 #1 0x00007fc8dfec0dcb in _L_lock_883 () at /lib64/libpthread.so.0 #2 0x00007fc8dfec0c98 in pthread_mutex_lock () at /lib64/libpthread.so.0 #3 0x00005567639d4d39 in qemu_mutex_lock_impl (mutex=mutex@entry=0x5567642230e0 <qemu_global_mutex>, file=file@entry=0x556763a69308 "/builddir/build/BUILD/qemu-2.12.0/cpus.c", line=line@entry=1765) at util/qemu-thread-posix.c:67 #4 0x00005567636b8298 in qemu_mutex_lock_iothread () at /usr/src/debug/qemu-2.12.0/cpus.c:1765 #5 0x00005567636dabfc in kvm_cpu_exec (cpu=cpu@entry=0x55676602e000) at /usr/src/debug/qemu-2.12.0/accel/kvm/kvm-all.c:2047 #6 0x00005567636b8446 in qemu_kvm_cpu_thread_fn (arg=0x55676602e000) at /usr/src/debug/qemu-2.12.0/cpus.c:1215 #7 0x00007fc8dfebedd5 in start_thread () at /lib64/libpthread.so.0 #8 0x00007fc8dfbe7ead in clone () at /lib64/libc.so.6 Thread 12 (Thread 0x7fc8ca52e700 (LWP 53302)): #0 0x00007fc8dfec54ed in __lll_lock_wait () at /lib64/libpthread.so.0 #1 0x00007fc8dfec0dcb in _L_lock_883 () at /lib64/libpthread.so.0 #2 0x00007fc8dfec0c98 in pthread_mutex_lock () at /lib64/libpthread.so.0 #3 0x00005567639d4d39 in qemu_mutex_lock_impl (mutex=mutex@entry=0x5567642230e0 <qemu_global_mutex>, file=file@entry=0x556763a69308 "/builddir/build/BUILD/qemu-2.12.0/cpus.c", line=line@entry=1765) at util/qemu-thread-posix.c:67 #4 0x00005567636b8298 in qemu_mutex_lock_iothread () at /usr/src/debug/qemu-2.12.0/cpus.c:1765 #5 0x00005567636dabfc in kvm_cpu_exec (cpu=cpu@entry=0x55676604e000) at /usr/src/debug/qemu-2.12.0/accel/kvm/kvm-all.c:2047 #6 0x00005567636b8446 in qemu_kvm_cpu_thread_fn (arg=0x55676604e000) at /usr/src/debug/qemu-2.12.0/cpus.c:1215 #7 0x00007fc8dfebedd5 in start_thread () at /lib64/libpthread.so.0 #8 0x00007fc8dfbe7ead in clone () at /lib64/libc.so.6 Thread 11 (Thread 0x7fc8c9d2d700 (LWP 53303)): #0 0x00007fc8dfec54ed in __lll_lock_wait () at /lib64/libpthread.so.0 #1 0x00007fc8dfec0dcb in _L_lock_883 () at /lib64/libpthread.so.0 #2 0x00007fc8dfec0c98 in pthread_mutex_lock () at /lib64/libpthread.so.0 #3 0x00005567639d4d39 in qemu_mutex_lock_impl (mutex=mutex@entry=0x5567642230e0 <qemu_global_mutex>, file=file@entry=0x556763a69308 "/builddir/build/BUILD/qemu-2.12.0/cpus.c", line=line@entry=1765) at util/qemu-thread-posix.c:67 #4 0x00005567636b8298 in qemu_mutex_lock_iothread () at /usr/src/debug/qemu-2.12.0/cpus.c:1765 #5 0x00005567636dabfc in kvm_cpu_exec (cpu=cpu@entry=0x556766070000) at /usr/src/debug/qemu-2.12.0/accel/kvm/kvm-all.c:2047 #6 0x00005567636b8446 in qemu_kvm_cpu_thread_fn (arg=0x556766070000) at /usr/src/debug/qemu-2.12.0/cpus.c:1215 #7 0x00007fc8dfebedd5 in start_thread () at /lib64/libpthread.so.0 #8 0x00007fc8dfbe7ead in clone () at /lib64/libc.so.6 Thread 10 (Thread 0x7fc8c952c700 (LWP 53304)): #0 0x00007fc8dfec54ed in __lll_lock_wait () at /lib64/libpthread.so.0 #1 0x00007fc8dfec0dcb in _L_lock_883 () at /lib64/libpthread.so.0 #2 0x00007fc8dfec0c98 in pthread_mutex_lock () at /lib64/libpthread.so.0 #3 0x00005567639d4d39 in qemu_mutex_lock_impl (mutex=mutex@entry=0x5567642230e0 <qemu_global_mutex>, file=file@entry=0x556763a69308 "/builddir/build/BUILD/qemu-2.12.0/cpus.c", line=line@entry=1765) at util/qemu-thread-posix.c:67 #4 0x00005567636b8298 in qemu_mutex_lock_iothread () at /usr/src/debug/qemu-2.12.0/cpus.c:1765 #5 0x00005567636dabfc in kvm_cpu_exec (cpu=cpu@entry=0x556766096000) at /usr/src/debug/qemu-2.12.0/accel/kvm/kvm-all.c:2047 #6 0x00005567636b8446 in qemu_kvm_cpu_thread_fn (arg=0x556766096000) at /usr/src/debug/qemu-2.12.0/cpus.c:1215 #7 0x00007fc8dfebedd5 in start_thread () at /lib64/libpthread.so.0 #8 0x00007fc8dfbe7ead in clone () at /lib64/libc.so.6 Thread 9 (Thread 0x7fc8c8d2b700 (LWP 53305)): #0 0x00007fc8dfec54ed in __lll_lock_wait () at /lib64/libpthread.so.0 #1 0x00007fc8dfec0dcb in _L_lock_883 () at /lib64/libpthread.so.0 #2 0x00007fc8dfec0c98 in pthread_mutex_lock () at /lib64/libpthread.so.0 #3 0x00005567639d4d39 in qemu_mutex_lock_impl (mutex=mutex@entry=0x5567642230e0 <qemu_global_mutex>, file=file@entry=0x556763a69308 "/builddir/build/BUILD/qemu-2.12.0/cpus.c", line=line@entry=1765) at util/qemu-thread-posix.c:67 #4 0x00005567636b8298 in qemu_mutex_lock_iothread () at /usr/src/debug/qemu-2.12.0/cpus.c:1765 #5 0x00005567636dabfc in kvm_cpu_exec (cpu=cpu@entry=0x5567660b4000) at /usr/src/debug/qemu-2.12.0/accel/kvm/kvm-all.c:2047 #6 0x00005567636b8446 in qemu_kvm_cpu_thread_fn (arg=0x5567660b4000) at /usr/src/debug/qemu-2.12.0/cpus.c:1215 #7 0x00007fc8dfebedd5 in start_thread () at /lib64/libpthread.so.0 #8 0x00007fc8dfbe7ead in clone () at /lib64/libc.so.6 Thread 8 (Thread 0x7fc8c852a700 (LWP 53306)): #0 0x00007fc8dfec54ed in __lll_lock_wait () at /lib64/libpthread.so.0 #1 0x00007fc8dfec0dcb in _L_lock_883 () at /lib64/libpthread.so.0 #2 0x00007fc8dfec0c98 in pthread_mutex_lock () at /lib64/libpthread.so.0 #3 0x00005567639d4d39 in qemu_mutex_lock_impl (mutex=mutex@entry=0x5567642230e0 <qemu_global_mutex>, file=file@entry=0x556763a69308 "/builddir/build/BUILD/qemu-2.12.0/cpus.c", line=line@entry=1765) at util/qemu-thread-posix.c:67 #4 0x00005567636b8298 in qemu_mutex_lock_iothread () at /usr/src/debug/qemu-2.12.0/cpus.c:1765 #5 0x00005567636dabfc in kvm_cpu_exec (cpu=cpu@entry=0x5567660d2000) at /usr/src/debug/qemu-2.12.0/accel/kvm/kvm-all.c:2047 #6 0x00005567636b8446 in qemu_kvm_cpu_thread_fn (arg=0x5567660d2000) at /usr/src/debug/qemu-2.12.0/cpus.c:1215 #7 0x00007fc8dfebedd5 in start_thread () at /lib64/libpthread.so.0 #8 0x00007fc8dfbe7ead in clone () at /lib64/libc.so.6 Thread 7 (Thread 0x7fc8c7d29700 (LWP 53307)): #0 0x00007fc8dfec54ed in __lll_lock_wait () at /lib64/libpthread.so.0 #1 0x00007fc8dfec0dcb in _L_lock_883 () at /lib64/libpthread.so.0 #2 0x00007fc8dfec0c98 in pthread_mutex_lock () at /lib64/libpthread.so.0 #3 0x00005567639d4d39 in qemu_mutex_lock_impl (mutex=mutex@entry=0x5567642230e0 <qemu_global_mutex>, file=file@entry=0x556763a69308 "/builddir/build/BUILD/qemu-2.12.0/cpus.c", line=line@entry=1765) at util/qemu-thread-posix.c:67 #4 0x00005567636b8298 in qemu_mutex_lock_iothread () at /usr/src/debug/qemu-2.12.0/cpus.c:1765 #5 0x00005567636dabfc in kvm_cpu_exec (cpu=cpu@entry=0x5567660f0000) at /usr/src/debug/qemu-2.12.0/accel/kvm/kvm-all.c:2047 #6 0x00005567636b8446 in qemu_kvm_cpu_thread_fn (arg=0x5567660f0000) at /usr/src/debug/qemu-2.12.0/cpus.c:1215 #7 0x00007fc8dfebedd5 in start_thread () at /lib64/libpthread.so.0 #8 0x00007fc8dfbe7ead in clone () at /lib64/libc.so.6 Thread 6 (Thread 0x7fc8c7528700 (LWP 53308)): #0 0x00007fc8dfec54ed in __lll_lock_wait () at /lib64/libpthread.so.0 #1 0x00007fc8dfec0dcb in _L_lock_883 () at /lib64/libpthread.so.0 #2 0x00007fc8dfec0c98 in pthread_mutex_lock () at /lib64/libpthread.so.0 #3 0x00005567639d4d39 in qemu_mutex_lock_impl (mutex=mutex@entry=0x5567642230e0 <qemu_global_mutex>, file=file@entry=0x556763a69308 "/builddir/build/BUILD/qemu-2.12.0/cpus.c", line=line@entry=1765) at util/qemu-thread-posix.c:67 #4 0x00005567636b8298 in qemu_mutex_lock_iothread () at /usr/src/debug/qemu-2.12.0/cpus.c:1765 #5 0x00005567636dabfc in kvm_cpu_exec (cpu=cpu@entry=0x556766112000) at /usr/src/debug/qemu-2.12.0/accel/kvm/kvm-all.c:2047 #6 0x00005567636b8446 in qemu_kvm_cpu_thread_fn (arg=0x556766112000) at /usr/src/debug/qemu-2.12.0/cpus.c:1215 #7 0x00007fc8dfebedd5 in start_thread () at /lib64/libpthread.so.0 #8 0x00007fc8dfbe7ead in clone () at /lib64/libc.so.6 Thread 5 (Thread 0x7fc8c6d27700 (LWP 53309)): #0 0x00007fc8dfec54ed in __lll_lock_wait () at /lib64/libpthread.so.0 #1 0x00007fc8dfec0dcb in _L_lock_883 () at /lib64/libpthread.so.0 #2 0x00007fc8dfec0c98 in pthread_mutex_lock () at /lib64/libpthread.so.0 #3 0x00005567639d4d39 in qemu_mutex_lock_impl (mutex=mutex@entry=0x5567642230e0 <qemu_global_mutex>, file=file@entry=0x556763a69308 "/builddir/build/BUILD/qemu-2.12.0/cpus.c", line=line@entry=1765) at util/qemu-thread-posix.c:67 #4 0x00005567636b8298 in qemu_mutex_lock_iothread () at /usr/src/debug/qemu-2.12.0/cpus.c:1765 #5 0x00005567636dabfc in kvm_cpu_exec (cpu=cpu@entry=0x556766138000) at /usr/src/debug/qemu-2.12.0/accel/kvm/kvm-all.c:2047 #6 0x00005567636b8446 in qemu_kvm_cpu_thread_fn (arg=0x556766138000) at /usr/src/debug/qemu-2.12.0/cpus.c:1215 #7 0x00007fc8dfebedd5 in start_thread () at /lib64/libpthread.so.0 #8 0x00007fc8dfbe7ead in clone () at /lib64/libc.so.6 Thread 4 (Thread 0x7fc8c6526700 (LWP 53310)): #0 0x00007fc8dfec54ed in __lll_lock_wait () at /lib64/libpthread.so.0 #1 0x00007fc8dfec0dcb in _L_lock_883 () at /lib64/libpthread.so.0 #2 0x00007fc8dfec0c98 in pthread_mutex_lock () at /lib64/libpthread.so.0 #3 0x00005567639d4d39 in qemu_mutex_lock_impl (mutex=mutex@entry=0x5567642230e0 <qemu_global_mutex>, file=file@entry=0x556763a69308 "/builddir/build/BUILD/qemu-2.12.0/cpus.c", line=line@entry=1765) at util/qemu-thread-posix.c:67 #4 0x00005567636b8298 in qemu_mutex_lock_iothread () at /usr/src/debug/qemu-2.12.0/cpus.c:1765 #5 0x00005567636dabfc in kvm_cpu_exec (cpu=cpu@entry=0x55676615a000) at /usr/src/debug/qemu-2.12.0/accel/kvm/kvm-all.c:2047 #6 0x00005567636b8446 in qemu_kvm_cpu_thread_fn (arg=0x55676615a000) at /usr/src/debug/qemu-2.12.0/cpus.c:1215 #7 0x00007fc8dfebedd5 in start_thread () at /lib64/libpthread.so.0 #8 0x00007fc8dfbe7ead in clone () at /lib64/libc.so.6 Thread 3 (Thread 0x7fc8c5d25700 (LWP 53311)): #0 0x00005567638fff40 in child_job_drained_poll (c=0x556765dde5a0) at blockjob.c:159 #1 0x000055676394afb8 in bdrv_drain_poll (c=<optimized out>) at block/io.c:75 #2 0x000055676394afb8 in bdrv_drain_poll (bs=0x556765c82800, ignore_bds_parents=false, ignore=0x0) at block/io.c:90 #3 0x000055676394afb8 in bdrv_drain_poll (bs=0x556765c82800, recursive=<optimized out>, ignore_parent=0x0, ignore_bds_parents=false) at block/io.c:244 #4 0x000055676394b1e0 in bdrv_do_drained_begin (ignore_parent=0x0, recursive=false, bs=0x556765c82800) at block/io.c:267 #5 0x000055676394b1e0 in bdrv_do_drained_begin (bs=0x556765c82800, recursive=<optimized out>, parent=0x0, ignore_bds_parents=<optimized out>, poll=<optimized out>) at block/io.c:397 #6 0x000055676393dd78 in blk_drain (blk=0x556765b258c0) at block/block-backend.c:1607 #7 0x000055676394655d in mirror_drain (job=<optimized out>) at block/mirror.c:1002 #8 0x00005567638fffa8 in block_job_detach_aio_context (opaque=0x556765b25b80) at blockjob.c:139 #9 0x00005567638fc341 in bdrv_detach_aio_context (bs=bs@entry=0x556766644800) at block.c:4932 #10 0x00005567638fc4e6 in bdrv_set_aio_context (bs=0x556766644800, new_context=0x556765ae0500) at block.c:4993 #11 0x000055676393e2b8 in blk_set_aio_context (blk=<optimized out>, new_context=<optimized out>) at block/block-backend.c:1933 #12 0x00005567636eddd8 in virtio_blk_data_plane_stop (vdev=<optimized out>) at /usr/src/debug/qemu-2.12.0/hw/block/dataplane/virtio-blk.c:286 #13 0x00005567638939d5 in virtio_bus_stop_ioeventfd (bus=bus@entry=0x556767df00f8) at hw/virtio/virtio-bus.c:246 #14 0x0000556763890d41 in virtio_pci_common_write (proxy=0x556767de8000) at hw/virtio/virtio-pci.c:294 #15 0x0000556763890d41 in virtio_pci_common_write (opaque=0x556767de8000, addr=<optimized out>, val=0, size=<optimized out>) at hw/virtio/virtio-pci.c:1283 #16 0x00005567636ca593 in memory_region_write_accessor (mr=<optimized out>, addr=<optimized out>, value=<optimized out>, size=<optimized out>, shift=<optimized out>, mask=<optimized out>, attrs=...) at /usr/src/debug/qemu-2.12.0/memory.c:530 #17 0x00005567636c82a9 in access_with_adjusted_size (addr=addr@entry=20, value=value@entry=0x7fc8c5d22308, size=size@entry=1, access_size_min=<optimized out>, access_size_max=<optimized out>, access_fn=access_fn@entry=0x5567636ca550 <memory_region_write_accessor>, mr=mr@entry=0x556767de89d0, attrs=attrs@entry=...) at /usr/src/debug/qemu-2.12.0/memory.c:597 #18 0x00005567636cc345 in memory_region_dispatch_write (mr=<optimized out>, addr=20, data=0, size=<optimized out>, attrs=...) at /usr/src/debug/qemu-2.12.0/memory.c:1474 #19 0x000055676367c28b in flatview_write (fv=0x5567672978c0, addr=<optimized out>, attrs=..., buf=<optimized out>, len=<optimized out>) at /usr/src/debug/qemu-2.12.0/exec.c:3101 #20 0x000055676367fd2f in address_space_write (as=<optimized out>, addr=<optimized out>, attrs=..., buf=<optimized out>, len=<optimized out>) at /usr/src/debug/qemu-2.12.0/exec.c:3261 #21 0x000055676367fdd5 in address_space_rw (as=<optimized out>, addr=<optimized out>, attrs=..., attrs@entry=..., buf=buf@entry=0x7fc8e698a028 "", len=<optimized out>, is_write=<optimized out>) at /usr/src/debug/qemu-2.12.0/exec.c:3272 #22 0x00005567636dac58 in kvm_cpu_exec (cpu=cpu@entry=0x55676617a000) at /usr/src/debug/qemu-2.12.0/accel/kvm/kvm-all.c:1992 #23 0x00005567636b8446 in qemu_kvm_cpu_thread_fn (arg=0x55676617a000) at /usr/src/debug/qemu-2.12.0/cpus.c:1215 #24 0x00007fc8dfebedd5 in start_thread () at /lib64/libpthread.so.0 #25 0x00007fc8dfbe7ead in clone () at /lib64/libc.so.6 Thread 2 (Thread 0x7fc7c39ff700 (LWP 53313)): #0 0x00007fc8dfec2965 in pthread_cond_wait@@GLIBC_2.3.2 () at /lib64/libpthread.so.0 #1 0x00005567639d4fe9 in qemu_cond_wait_impl (cond=cond@entry=0x556765931710, mutex=mutex@entry=0x556765931748, file=file@entry=0x556763b3ed07 "ui/vnc-jobs.c", line=line@entry=212) at util/qemu-thread-posix.c:164 #2 0x00005567638efb1f in vnc_worker_thread_loop (queue=queue@entry=0x556765931710) at ui/vnc-jobs.c:212 #3 0x00005567638f00e8 in vnc_worker_thread (arg=0x556765931710) at ui/vnc-jobs.c:319 #4 0x00007fc8dfebedd5 in start_thread () at /lib64/libpthread.so.0 #5 0x00007fc8dfbe7ead in clone () at /lib64/libc.so.6 Thread 1 (Thread 0x7fc8e6abd1c0 (LWP 53275)): #0 0x00007fc8dfec54ed in __lll_lock_wait () at /lib64/libpthread.so.0 #1 0x00007fc8dfec0dcb in _L_lock_883 () at /lib64/libpthread.so.0 #2 0x00007fc8dfec0c98 in pthread_mutex_lock () at /lib64/libpthread.so.0 #3 0x00005567639d4d39 in qemu_mutex_lock_impl (mutex=mutex@entry=0x5567642230e0 <qemu_global_mutex>, file=file@entry=0x556763a69308 "/builddir/build/BUILD/qemu-2.12.0/cpus.c", line=line@entry=1765) at util/qemu-thread-posix.c:67 #4 0x00005567636b8298 in qemu_mutex_lock_iothread () at /usr/src/debug/qemu-2.12.0/cpus.c:1765 #5 0x00005567639d22da in main_loop_wait (timeout=<optimized out>) at util/main-loop.c:236 #6 0x00005567639d22da in main_loop_wait (nonblocking=nonblocking@entry=0) at util/main-loop.c:497 #7 0x0000556763674a87 in main () at vl.c:1963 #8 0x0000556763674a87 in main (argc=<optimized out>, argv=<optimized out>, envp=<optimized out>) at vl.c:4768 2.Don't hit this issue with dataplane disabled. 3.As it's a negative test, set its priorty and serverity to medium.
Test on qemu-kvm-3.1.0-6.module+el8+2711+98525d2b.x86_64 + kernel:4.18.0-62.el8.x86_64,also hit this issue: # pstack 24234 Thread 24 (Thread 0x7f4750dc5700 (LWP 24306)): #0 0x00007f4760b657da in pthread_cond_timedwait@@GLIBC_2.3.2 () at /lib64/libpthread.so.0 #1 0x00007f4748079790 in iot_worker () at /usr/lib64/glusterfs/3.12.2/xlator/performance/io-threads.so #2 0x00007f4760b5f2de in start_thread () at /lib64/libpthread.so.0 #3 0x00007f476088fa63 in clone () at /lib64/libc.so.6 Thread 23 (Thread 0x7f4750e06700 (LWP 24305)): #0 0x00007f4760b657da in pthread_cond_timedwait@@GLIBC_2.3.2 () at /lib64/libpthread.so.0 #1 0x00007f4748079790 in iot_worker () at /usr/lib64/glusterfs/3.12.2/xlator/performance/io-threads.so #2 0x00007f4760b5f2de in start_thread () at /lib64/libpthread.so.0 #3 0x00007f476088fa63 in clone () at /lib64/libc.so.6 Thread 22 (Thread 0x7f4750e47700 (LWP 24304)): #0 0x00007f4760b657da in pthread_cond_timedwait@@GLIBC_2.3.2 () at /lib64/libpthread.so.0 #1 0x00007f4748079790 in iot_worker () at /usr/lib64/glusterfs/3.12.2/xlator/performance/io-threads.so #2 0x00007f4760b5f2de in start_thread () at /lib64/libpthread.so.0 #3 0x00007f476088fa63 in clone () at /lib64/libc.so.6 Thread 21 (Thread 0x7f4750e88700 (LWP 24303)): #0 0x00007f4760b657da in pthread_cond_timedwait@@GLIBC_2.3.2 () at /lib64/libpthread.so.0 #1 0x00007f4748079790 in iot_worker () at /usr/lib64/glusterfs/3.12.2/xlator/performance/io-threads.so #2 0x00007f4760b5f2de in start_thread () at /lib64/libpthread.so.0 #3 0x00007f476088fa63 in clone () at /lib64/libc.so.6 Thread 20 (Thread 0x7f4727edd700 (LWP 24302)): #0 0x00007f4760b657da in pthread_cond_timedwait@@GLIBC_2.3.2 () at /lib64/libpthread.so.0 #1 0x00007f4748079790 in iot_worker () at /usr/lib64/glusterfs/3.12.2/xlator/performance/io-threads.so #2 0x00007f4760b5f2de in start_thread () at /lib64/libpthread.so.0 #3 0x00007f476088fa63 in clone () at /lib64/libc.so.6 Thread 19 (Thread 0x7f474022e700 (LWP 24301)): #0 0x00007f4760b657da in pthread_cond_timedwait@@GLIBC_2.3.2 () at /lib64/libpthread.so.0 #1 0x00007f4748079790 in iot_worker () at /usr/lib64/glusterfs/3.12.2/xlator/performance/io-threads.so #2 0x00007f4760b5f2de in start_thread () at /lib64/libpthread.so.0 #3 0x00007f476088fa63 in clone () at /lib64/libc.so.6 Thread 18 (Thread 0x7f47265ff700 (LWP 24267)): #0 0x00007f4760b6548c in pthread_cond_wait@@GLIBC_2.3.2 () at /lib64/libpthread.so.0 #1 0x0000562def8dff9d in qemu_cond_wait_impl (cond=<optimized out>, mutex=0x562df318a728, file=0x562defa43ee7 "ui/vnc-jobs.c", line=214) at util/qemu-thread-posix.c:161 #2 0x0000562def813d11 in vnc_worker_thread_loop (queue=queue@entry=0x562df318a6f0) at ui/vnc-jobs.c:214 #3 0x0000562def8142d0 in vnc_worker_thread (arg=0x562df318a6f0) at ui/vnc-jobs.c:324 #4 0x0000562def8dfbe4 in qemu_thread_start (args=0x562df3151a40) at util/qemu-thread-posix.c:498 #5 0x00007f4760b5f2de in start_thread () at /lib64/libpthread.so.0 #6 0x00007f476088fa63 in clone () at /lib64/libc.so.6 Thread 17 (Thread 0x7f4740ab1700 (LWP 24261)): #0 0x00007f4760b6889d in __lll_lock_wait () at /lib64/libpthread.so.0 #1 0x00007f4760b61b59 in pthread_mutex_lock () at /lib64/libpthread.so.0 #2 0x0000562def8dfccd in qemu_mutex_lock_impl (mutex=0x562df010f500 <qemu_global_mutex>, file=0x562def967c38 "/builddir/build/BUILD/qemu-3.1.0/exec.c", line=3197) at util/qemu-thread-posix.c:66 #3 0x0000562def5d5fee in qemu_mutex_lock_iothread_impl (file=<optimized out>, line=<optimized out>) at /usr/src/debug/qemu-kvm-3.1.0-6.module+el8+2711+98525d2b.x86_64/cpus.c:1849 #4 0x0000562def594049 in prepare_mmio_access (mr=<optimized out>, mr=<optimized out>) at /usr/src/debug/qemu-kvm-3.1.0-6.module+el8+2711+98525d2b.x86_64/exec.c:3197 #5 0x0000562def598f0f in flatview_read_continue (fv=0x7f472c0806f0, addr=1017, attrs=..., buf=<optimized out>, len=1, addr1=<optimized out>, l=<optimized out>, mr=0x562df32b6580) at /usr/src/debug/qemu-kvm-3.1.0-6.module+el8+2711+98525d2b.x86_64/exec.c:3292 #6 0x0000562def599136 in flatview_read (fv=0x7f472c0806f0, addr=1017, attrs=..., buf=0x7f47657a6000 "", len=1) at /usr/src/debug/qemu-kvm-3.1.0-6.module+el8+2711+98525d2b.x86_64/exec.c:3332 #7 0x0000562def599263 in address_space_read_full (as=<optimized out>, addr=<optimized out>, attrs=..., buf=<optimized out>, len=<optimized out>) at /usr/src/debug/qemu-kvm-3.1.0-6.module+el8+2711+98525d2b.x86_64/exec.c:3345 #8 0x0000562def5fcba0 in kvm_handle_io (count=1, size=1, direction=<optimized out>, data=<optimized out>, attrs=..., port=1017) at /usr/src/debug/qemu-kvm-3.1.0-6.module+el8+2711+98525d2b.x86_64/accel/kvm/kvm-all.c:1787 #9 0x0000562def5fcba0 in kvm_cpu_exec (cpu=<optimized out>) at /usr/src/debug/qemu-kvm-3.1.0-6.module+el8+2711+98525d2b.x86_64/accel/kvm/kvm-all.c:2033 #10 0x0000562def5d61b6 in qemu_kvm_cpu_thread_fn (arg=0x562df22a5c00) at /usr/src/debug/qemu-kvm-3.1.0-6.module+el8+2711+98525d2b.x86_64/cpus.c:1281 #11 0x0000562def8dfbe4 in qemu_thread_start (args=0x562df22c8970) at util/qemu-thread-posix.c:498 #12 0x00007f4760b5f2de in start_thread () at /lib64/libpthread.so.0 #13 0x00007f476088fa63 in clone () at /lib64/libc.so.6 Thread 16 (Thread 0x7f47412b2700 (LWP 24260)): #0 0x00007f476088645b in ioctl () at /lib64/libc.so.6 #1 0x0000562def5fc899 in kvm_vcpu_ioctl (cpu=cpu@entry=0x562df2283060, type=type@entry=44672) at /usr/src/debug/qemu-kvm-3.1.0-6.module+el8+2711+98525d2b.x86_64/accel/kvm/kvm-all.c:2156 #2 0x0000562def5fc95a in kvm_cpu_exec (cpu=<optimized out>) at /usr/src/debug/qemu-kvm-3.1.0-6.module+el8+2711+98525d2b.x86_64/accel/kvm/kvm-all.c:1993 #3 0x0000562def5d61b6 in qemu_kvm_cpu_thread_fn (arg=0x562df2283060) at /usr/src/debug/qemu-kvm-3.1.0-6.module+el8+2711+98525d2b.x86_64/cpus.c:1281 #4 0x0000562def8dfbe4 in qemu_thread_start (args=0x562df22a53c0) at util/qemu-thread-posix.c:498 #5 0x00007f4760b5f2de in start_thread () at /lib64/libpthread.so.0 #6 0x00007f476088fa63 in clone () at /lib64/libc.so.6 Thread 15 (Thread 0x7f4741ab3700 (LWP 24259)): #0 0x00007f4760b63355 in __pthread_mutex_unlock_usercnt () at /lib64/libpthread.so.0 #1 0x0000562def8dfd94 in qemu_mutex_unlock_impl (mutex=<optimized out>, file=<optimized out>, line=<optimized out>) at util/qemu-thread-posix.c:94 #2 0x0000562def82685f in job_enter (job=0x562df2522ad0) at job.c:440 #3 0x0000562def82685f in job_drain (job=0x562df2522ad0) at job.c:530 #4 0x0000562def824af8 in block_job_detach_aio_context (opaque=0x562df2522ad0) at blockjob.c:139 #5 0x0000562def820be3 in bdrv_detach_aio_context (bs=bs@entry=0x562df31493e0) at block.c:5068 #6 0x0000562def820d9e in bdrv_set_aio_context (bs=0x562df31493e0, new_context=0x562df20cbe10) at block.c:5129 #7 0x0000562def8530bc in blk_set_aio_context (blk=<optimized out>, new_context=<optimized out>) at block/block-backend.c:1901 #8 0x0000562def6186a2 in virtio_blk_data_plane_stop (vdev=<optimized out>) at /usr/src/debug/qemu-kvm-3.1.0-6.module+el8+2711+98525d2b.x86_64/hw/block/dataplane/virtio-blk.c:285 #9 0x0000562def7b173f in virtio_bus_stop_ioeventfd (bus=bus@entry=0x562df3672ad8) at hw/virtio/virtio-bus.c:246 #10 0x0000562def7b1e9f in virtio_bus_stop_ioeventfd (bus=bus@entry=0x562df3672ad8) at hw/virtio/virtio-bus.c:238 #11 0x0000562def7b4d98 in virtio_pci_stop_ioeventfd (proxy=0x562df366a9e0) at hw/virtio/virtio-pci.c:1284 #12 0x0000562def7b4d98 in virtio_pci_common_write (opaque=0x562df366a9e0, addr=<optimized out>, val=<optimized out>, size=<optimized out>) at hw/virtio/virtio-pci.c:1284 #13 0x0000562def5e8f83 in memory_region_write_accessor (mr=<optimized out>, addr=<optimized out>, value=<optimized out>, size=<optimized out>, shift=<optimized out>, mask=<optimized out>, attrs=...) at /usr/src/debug/qemu-kvm-3.1.0-6.module+el8+2711+98525d2b.x86_64/memory.c:504 #14 0x0000562def5e7136 in access_with_adjusted_size (addr=addr@entry=20, value=value@entry=0x7f4741ab2548, size=size@entry=1, access_size_min=<optimized out>, access_size_max=<optimized out>, access_fn=access_fn@entry=0x562def5e8f30 <memory_region_write_accessor>, mr=0x562df366b3b0, attrs=...) at /usr/src/debug/qemu-kvm-3.1.0-6.module+el8+2711+98525d2b.x86_64/memory.c:570 #15 0x0000562def5eaf00 in memory_region_dispatch_write (mr=0x562df366b3b0, addr=20, data=<optimized out>, size=1, attrs=...) at /usr/src/debug/qemu-kvm-3.1.0-6.module+el8+2711+98525d2b.x86_64/memory.c:1452 #16 0x0000562def594ff3 in flatview_write_continue (fv=0x7f472c061a40, addr=4261445652, attrs=..., buf=0x7f47657ab028 "", len=1, addr1=<optimized out>, l=<optimized out>, mr=0x562df366b3b0) at /usr/src/debug/qemu-kvm-3.1.0-6.module+el8+2711+98525d2b.x86_64/exec.c:3233 #17 0x0000562def595219 in flatview_write (fv=0x7f472c061a40, addr=4261445652, attrs=..., buf=0x7f47657ab028 "", len=1) at /usr/src/debug/qemu-kvm-3.1.0-6.module+el8+2711+98525d2b.x86_64/exec.c:3272 #18 0x0000562def599313 in address_space_write (as=<optimized out>, addr=<optimized out>, attrs=..., buf=<optimized out>, len=<optimized out>) at /usr/src/debug/qemu-kvm-3.1.0-6.module+el8+2711+98525d2b.x86_64/exec.c:3362 #19 0x0000562def5fcb18 in kvm_cpu_exec (cpu=<optimized out>) at /usr/src/debug/qemu-kvm-3.1.0-6.module+el8+2711+98525d2b.x86_64/accel/kvm/kvm-all.c:2043 #20 0x0000562def5d61b6 in qemu_kvm_cpu_thread_fn (arg=0x562df225f790) at /usr/src/debug/qemu-kvm-3.1.0-6.module+el8+2711+98525d2b.x86_64/cpus.c:1281 #21 0x0000562def8dfbe4 in qemu_thread_start (args=0x562df2282820) at util/qemu-thread-posix.c:498 #22 0x00007f4760b5f2de in start_thread () at /lib64/libpthread.so.0 #23 0x00007f476088fa63 in clone () at /lib64/libc.so.6 Thread 14 (Thread 0x7f47422b4700 (LWP 24258)): #0 0x00007f476088645b in ioctl () at /lib64/libc.so.6 #1 0x0000562def5fc899 in kvm_vcpu_ioctl (cpu=cpu@entry=0x562df2213fc0, type=type@entry=44672) at /usr/src/debug/qemu-kvm-3.1.0-6.module+el8+2711+98525d2b.x86_64/accel/kvm/kvm-all.c:2156 #2 0x0000562def5fc95a in kvm_cpu_exec (cpu=<optimized out>) at /usr/src/debug/qemu-kvm-3.1.0-6.module+el8+2711+98525d2b.x86_64/accel/kvm/kvm-all.c:1993 #3 0x0000562def5d61b6 in qemu_kvm_cpu_thread_fn (arg=0x562df2213fc0) at /usr/src/debug/qemu-kvm-3.1.0-6.module+el8+2711+98525d2b.x86_64/cpus.c:1281 #4 0x0000562def8dfbe4 in qemu_thread_start (args=0x562df2234ea0) at util/qemu-thread-posix.c:498 #5 0x00007f4760b5f2de in start_thread () at /lib64/libpthread.so.0 #6 0x00007f476088fa63 in clone () at /lib64/libc.so.6 Thread 13 (Thread 0x7f47432bc700 (LWP 24257)): #0 0x00007f476088fd97 in epoll_wait () at /lib64/libc.so.6 #1 0x00007f4753d86d48 in event_dispatch_epoll_worker () at /lib64/libglusterfs.so.0 #2 0x00007f4760b5f2de in start_thread () at /lib64/libpthread.so.0 #3 0x00007f476088fa63 in clone () at /lib64/libc.so.6 Thread 12 (Thread 0x7f47580ea700 (LWP 24256)): #0 0x00007f4760b657da in pthread_cond_timedwait@@GLIBC_2.3.2 () at /lib64/libpthread.so.0 #1 0x00007f4748079790 in iot_worker () at /usr/lib64/glusterfs/3.12.2/xlator/performance/io-threads.so #2 0x00007f4760b5f2de in start_thread () at /lib64/libpthread.so.0 #3 0x00007f476088fa63 in clone () at /lib64/libc.so.6 Thread 11 (Thread 0x7f4749ffb700 (LWP 24255)): #0 0x00007f476088fd97 in epoll_wait () at /lib64/libc.so.6 #1 0x00007f4753d86d48 in event_dispatch_epoll_worker () at /lib64/libglusterfs.so.0 #2 0x00007f4760b5f2de in start_thread () at /lib64/libpthread.so.0 #3 0x00007f476088fa63 in clone () at /lib64/libc.so.6 Thread 10 (Thread 0x7f474a7fc700 (LWP 24254)): #0 0x00007f4760b6081d in __pthread_timedjoin_ex () at /lib64/libpthread.so.0 #1 0x00007f4753d874d7 in event_dispatch_epoll () at /lib64/libglusterfs.so.0 #2 0x00007f47581f34e8 in glfs_poller () at /lib64/libgfapi.so.0 #3 0x00007f4760b5f2de in start_thread () at /lib64/libpthread.so.0 #4 0x00007f476088fa63 in clone () at /lib64/libc.so.6 Thread 9 (Thread 0x7f474affd700 (LWP 24253)): #0 0x00007f4760b69210 in nanosleep () at /lib64/libpthread.so.0 #1 0x00007f4753d3ca86 in gf_timer_proc () at /lib64/libglusterfs.so.0 #2 0x00007f4760b5f2de in start_thread () at /lib64/libpthread.so.0 #3 0x00007f476088fa63 in clone () at /lib64/libc.so.6 Thread 8 (Thread 0x7f474b7fe700 (LWP 24252)): #0 0x00007f4760b657da in pthread_cond_timedwait@@GLIBC_2.3.2 () at /lib64/libpthread.so.0 #1 0x00007f4753d678d1 in syncenv_task () at /lib64/libglusterfs.so.0 #2 0x00007f4753d68558 in syncenv_processor () at /lib64/libglusterfs.so.0 #3 0x00007f4760b5f2de in start_thread () at /lib64/libpthread.so.0 #4 0x00007f476088fa63 in clone () at /lib64/libc.so.6 Thread 7 (Thread 0x7f474bfff700 (LWP 24251)): #0 0x00007f4760b657da in pthread_cond_timedwait@@GLIBC_2.3.2 () at /lib64/libpthread.so.0 #1 0x00007f4753d678d1 in syncenv_task () at /lib64/libglusterfs.so.0 #2 0x00007f4753d68558 in syncenv_processor () at /lib64/libglusterfs.so.0 #3 0x00007f4760b5f2de in start_thread () at /lib64/libpthread.so.0 #4 0x00007f476088fa63 in clone () at /lib64/libc.so.6 Thread 6 (Thread 0x7f4751a89700 (LWP 24250)): #0 0x00007f4760b657da in pthread_cond_timedwait@@GLIBC_2.3.2 () at /lib64/libpthread.so.0 #1 0x00007f4753d678d1 in syncenv_task () at /lib64/libglusterfs.so.0 #2 0x00007f4753d68558 in syncenv_processor () at /lib64/libglusterfs.so.0 #3 0x00007f4760b5f2de in start_thread () at /lib64/libpthread.so.0 #4 0x00007f476088fa63 in clone () at /lib64/libc.so.6 Thread 5 (Thread 0x7f475228a700 (LWP 24249)): #0 0x00007f4760b657da in pthread_cond_timedwait@@GLIBC_2.3.2 () at /lib64/libpthread.so.0 #1 0x00007f4753d678d1 in syncenv_task () at /lib64/libglusterfs.so.0 #2 0x00007f4753d68558 in syncenv_processor () at /lib64/libglusterfs.so.0 #3 0x00007f4760b5f2de in start_thread () at /lib64/libpthread.so.0 #4 0x00007f476088fa63 in clone () at /lib64/libc.so.6 Thread 4 (Thread 0x7f475348b700 (LWP 24248)): #0 0x00007f476085bbe8 in nanosleep () at /lib64/libc.so.6 #1 0x00007f476085baee in sleep () at /lib64/libc.so.6 #2 0x00007f4753d55936 in pool_sweeper () at /lib64/libglusterfs.so.0 #3 0x00007f4760b5f2de in start_thread () at /lib64/libpthread.so.0 #4 0x00007f476088fa63 in clone () at /lib64/libc.so.6 Thread 3 (Thread 0x7f4759027700 (LWP 24236)): #0 0x00007f4760b6889d in __lll_lock_wait () at /lib64/libpthread.so.0 #1 0x00007f4760b61c24 in pthread_mutex_lock () at /lib64/libpthread.so.0 #2 0x0000562def8dfccd in qemu_mutex_lock_impl (mutex=0x562df20d9310, file=0x562defa67c7f "util/async.c", line=511) at util/qemu-thread-posix.c:66 #3 0x0000562def8daf18 in co_schedule_bh_cb (opaque=0x562df20d92b0) at util/async.c:399 #4 0x0000562def8da896 in aio_bh_call (bh=0x562df20ce890) at util/async.c:118 #5 0x0000562def8da896 in aio_bh_poll (ctx=ctx@entry=0x562df20d92b0) at util/async.c:118 #6 0x0000562def8ddf64 in aio_poll (ctx=0x562df20d92b0, blocking=blocking@entry=true) at util/aio-posix.c:704 #7 0x0000562def6ca6c6 in iothread_run (opaque=0x562df20c0000) at iothread.c:64 #8 0x0000562def8dfbe4 in qemu_thread_start (args=0x562df20d9610) at util/qemu-thread-posix.c:498 #9 0x00007f4760b5f2de in start_thread () at /lib64/libpthread.so.0 #10 0x00007f476088fa63 in clone () at /lib64/libc.so.6 Thread 2 (Thread 0x7f4759828700 (LWP 24235)): #0 0x00007f476088a2bd in syscall () at /lib64/libc.so.6 #1 0x0000562def8e040f in qemu_futex_wait (val=<optimized out>, f=<optimized out>) at util/qemu-thread-posix.c:438 #2 0x0000562def8e040f in qemu_event_wait (ev=ev@entry=0x562df01448c8 <rcu_call_ready_event>) at util/qemu-thread-posix.c:442 #3 0x0000562def8f1b42 in call_rcu_thread (opaque=<optimized out>) at util/rcu.c:261 #4 0x0000562def8dfbe4 in qemu_thread_start (args=0x562df2007260) at util/qemu-thread-posix.c:498 #5 0x00007f4760b5f2de in start_thread () at /lib64/libpthread.so.0 #6 0x00007f476088fa63 in clone () at /lib64/libc.so.6 Thread 1 (Thread 0x7f476577eec0 (LWP 24234)): #0 0x00007f4760b6889d in __lll_lock_wait () at /lib64/libpthread.so.0 #1 0x00007f4760b61b59 in pthread_mutex_lock () at /lib64/libpthread.so.0 #2 0x0000562def8dfccd in qemu_mutex_lock_impl (mutex=0x562df010f500 <qemu_global_mutex>, file=0x562defa67f94 "util/main-loop.c", line=236) at util/qemu-thread-posix.c:66 #3 0x0000562def5d5fee in qemu_mutex_lock_iothread_impl (file=file@entry=0x562defa67f94 "util/main-loop.c", line=line@entry=236) at /usr/src/debug/qemu-kvm-3.1.0-6.module+el8+2711+98525d2b.x86_64/cpus.c:1849 #4 0x0000562def8dcdcd in os_host_main_loop_wait (timeout=<optimized out>) at util/main-loop.c:236 #5 0x0000562def8dcdcd in main_loop_wait (nonblocking=<optimized out>) at util/main-loop.c:497 #6 0x0000562def6cffe9 in main_loop () at vl.c:1910 #7 0x0000562def590544 in main (argc=<optimized out>, argv=<optimized out>, envp=<optimized out>) at vl.c:4681
Note: only virtio_blk hit this issue, virtio_scsi works fine.
We have a upstream patch tackling this issue: - https://lists.gnu.org/archive/html/qemu-devel/2019-03/msg02676.html
Test with qemu-kvm-rhev-2.12.0-29.el7.x86_64 on RHEL7.7 with -drive/device, also hit this issue
Need QA_ACK+
Verified on qemu-kvm-4.0.0-3.module+el8.1.0+3265+26c4ed71.x86_64, the bug has been fixed, set its status to verified. Test steps: 1.Start guest with qemu cmds: /usr/libexec/qemu-kvm \ -name 'avocado-vt-vm1' \ -machine q35 \ -nodefaults \ -device VGA,bus=pcie.0,addr=0x1 \ -chardev socket,id=qmp_id_qmpmonitor1,path=/var/tmp/monitor-qmpmonitor1-20190602-221944-MrlxVzia,server,nowait \ -mon chardev=qmp_id_qmpmonitor1,mode=control \ -chardev socket,id=qmp_id_catch_monitor,path=/var/tmp/monitor-catch_monitor-20190602-221944-MrlxVzia,server,nowait \ -mon chardev=qmp_id_catch_monitor,mode=control \ -device pvpanic,ioport=0x505,id=idn20piu \ -chardev socket,id=serial_id_serial0,path=/var/tmp/serial-serial0-20190602-221944-MrlxVzia,server,nowait \ -device isa-serial,chardev=serial_id_serial0 \ -chardev socket,id=seabioslog_id_20190602-221944-MrlxVzia,path=/var/tmp/seabios-20190602-221944-MrlxVzia,server,nowait \ -device isa-debugcon,chardev=seabioslog_id_20190602-221944-MrlxVzia,iobase=0x402 \ -device pcie-root-port,id=pcie.0-root-port-2,slot=2,chassis=2,addr=0x2,bus=pcie.0 \ -device qemu-xhci,id=usb1,bus=pcie.0-root-port-2,addr=0x0 \ -device pcie-root-port,id=pcie.0-root-port-3,slot=3,chassis=3,addr=0x3,bus=pcie.0 \ -device virtio-scsi-pci,id=virtio_scsi_pci0,bus=pcie.0-root-port-3,addr=0x0 \ -blockdev driver=file,filename=/home/kvm_autotest_root/images/rhel810-64-virtio-scsi.qcow2,node-name=file_node \ -blockdev driver=qcow2,node-name=drive_image1,file=file_node \ -device scsi-hd,id=image1,drive=drive_image1 \ -device pcie-root-port,id=pcie.0-root-port-5,slot=5,chassis=5,addr=0x5,bus=pcie.0 \ -device virtio-scsi-pci,id=virtio_scsi_pci1,bus=pcie.0-root-port-5,addr=0x0 \ -blockdev driver=file,filename=/home/data,node-name=file_data \ -blockdev driver=qcow2,node-name=drive_data1,file=file_data \ -device scsi-hd,id=data1,drive=drive_data1 \ -device pcie-root-port,id=pcie.0-root-port-4,slot=4,chassis=4,addr=0x4,bus=pcie.0 \ -device virtio-net-pci,mac=9a:33:34:35:36:37,id=idj01pFr,vectors=4,netdev=idMgbx8B,bus=pcie.0-root-port-4,addr=0x0 \ -netdev tap,id=idMgbx8B,vhost=on \ -m 7168 \ -smp 4,maxcpus=4,cores=2,threads=1,sockets=2 \ -cpu 'Skylake-Client',+kvm_pv_unhalt \ -device usb-tablet,id=usb-tablet1,bus=usb1.0,port=1 \ -vnc :0 \ -rtc base=utc,clock=host,driftfix=slew \ -boot order=cdn,once=c,menu=off,strict=off \ -enable-kvm \ -monitor stdio \ -qmp tcp:0:3000,server,nowait \ 2.Create a snapshot. {'execute':'blockdev-create','arguments':{'options': {'driver':'file','filename':'/root/sn$i','size':21474836480},'job-id':'job1'}} {'execute':'blockdev-add','arguments':{'driver':'file','node-name':'drive_sn$i','filename':'/root/sn$i'}} {'execute':'blockdev-create','arguments':{'options': {'driver': 'qcow2','file':'drive_sn$i','size':21474836480},'job-id':'job2'}} {'execute':'blockdev-add','arguments':{'driver':'qcow2','node-name':'sn$i','file':'drive_sn$i'}} {'execute':'job-dismiss','arguments':{'id':'job1'}} {'execute':'job-dismiss','arguments':{'id':'job2'}} {"execute":"blockdev-snapshot","arguments":{"node":"drive_data1","overlay":"sn1"}} 3.DD on the data disk (guest)#mkfs.ext4 /dev/sdb #mount /dev/sdb /mnt #cd /mnt #dd if=/dev/urandom of=1 bs=1M count=900 #sync 4.Do block commit from snapshot to base on data disk and during block commit, unplug data disk {'execute': 'block-commit', 'arguments': { 'device': 'sn1', 'base-node':'drive_data1', 'top-node':'sn1','speed':1,'job-id':'j1'}} {"execute": "device_del", "arguments": {"id":"data1"}} {"return": {}} After step4, no coredump exist and guest works well.
Since the problem described in this bug report should be resolved in a recent advisory, it has been closed with a resolution of ERRATA. For information on the advisory, and where to find the updated files, follow the link below. If the solution does not work for you, open a new bug report. https://access.redhat.com/errata/RHBA-2019:3723