Bug 1782175 - Qemu core dump when add persistent bitmap(data plane enable)
Summary: Qemu core dump when add persistent bitmap(data plane enable)
Keywords:
Status: VERIFIED
Alias: None
Product: Red Hat Enterprise Linux Advanced Virtualization
Classification: Red Hat
Component: qemu-kvm
Version: 8.2
Hardware: Unspecified
OS: Unspecified
urgent
unspecified
Target Milestone: rc
: ---
Assignee: Sergio Lopez
QA Contact: aihua liang
URL:
Whiteboard:
Depends On:
Blocks:
TreeView+ depends on / blocked
 
Reported: 2019-12-11 10:42 UTC by aihua liang
Modified: 2020-02-19 01:55 UTC (History)
10 users (show)

Fixed In Version: qemu-kvm-4.2.0-10.module+el8.2.0+5740+c3dff59e
Doc Type: If docs needed, set a value
Doc Text:
Clone Of:
Environment:
Last Closed:
Type: Bug
Target Upstream Version:


Attachments (Terms of Use)

Description aihua liang 2019-12-11 10:42:55 UTC
Description of problem:
 Qemu core dump when add persistent bitmap(data plane enable)

Version-Release number of selected component (if applicable):
  kernel version:4.18.0-160.el8.x86_64
  qemu-kvm version:qemu-kvm-4.2.0-2.module+el8.2.0+5135+ed3b2489.x86_64

How reproducible:
100%

Steps to Reproduce:
1.Start guest with qemu cmds:
   /usr/libexec/qemu-kvm \
    -name 'avocado-vt-vm1'  \
    -sandbox on  \
    -machine q35  \
    -nodefaults \
    -device VGA,bus=pcie.0,addr=0x1 \
    -m 7168  \
    -smp 4,maxcpus=4,cores=2,threads=1,dies=1,sockets=2  \
    -cpu 'Skylake-Client',+kvm_pv_unhalt  \
    -chardev socket,id=qmp_id_qmpmonitor1,path=/var/tmp/monitor-qmpmonitor1-20191210-025743-Q0JzJpKT,server,nowait \
    -mon chardev=qmp_id_qmpmonitor1,mode=control  \
    -chardev socket,id=qmp_id_catch_monitor,path=/var/tmp/monitor-catch_monitor-20191210-025743-Q0JzJpKT,server,nowait \
    -mon chardev=qmp_id_catch_monitor,mode=control \
    -device pvpanic,ioport=0x505,id=idDU2Q2E \
    -chardev socket,server,path=/var/tmp/serial-serial0-20191210-025743-Q0JzJpKT,id=chardev_serial0,nowait \
    -device isa-serial,id=serial0,chardev=chardev_serial0  \
    -chardev socket,id=seabioslog_id_20191210-025743-Q0JzJpKT,path=/var/tmp/seabios-20191210-025743-Q0JzJpKT,server,nowait \
    -device isa-debugcon,chardev=seabioslog_id_20191210-025743-Q0JzJpKT,iobase=0x402 \
    -device pcie-root-port,id=pcie.0-root-port-2,slot=2,chassis=2,addr=0x2,bus=pcie.0,multifunction=on \
    -device qemu-xhci,id=usb1,bus=pcie.0-root-port-2,addr=0x0 \
    -object iothread,id=iothread0 \
    -object iothread,id=iothread1 \
    -device pcie-root-port,id=pcie.0-root-port-3,slot=3,chassis=3,addr=0x2.0x1,bus=pcie.0 \
    -device virtio-scsi-pci,id=virtio_scsi_pci0,bus=pcie.0-root-port-3,addr=0x0,iothread=iothread1 \
    -blockdev node-name=file_image1,driver=file,aio=threads,filename=/home/kvm_autotest_root/images/rhel820-64-virtio-scsi.qcow2,cache.direct=on,cache.no-flush=off \
    -blockdev node-name=drive_image1,driver=qcow2,cache.direct=on,cache.no-flush=off,file=file_image1 \
    -device scsi-hd,id=image1,drive=drive_image1,write-cache=on \
    -device pcie-root-port,id=pcie.0-root-port-4,slot=4,chassis=4,addr=0x2.0x2,bus=pcie.0 \
    -blockdev node-name=file_data1,driver=file,aio=threads,filename=/home/data.qcow2,cache.direct=on,cache.no-flush=off \
    -blockdev node-name=drive_data1,driver=qcow2,cache.direct=on,cache.no-flush=off,file=file_data1 \
    -device virtio-blk-pci,id=data1,drive=drive_data1,write-cache=on,bus=pcie.0-root-port-4,iothread=iothread1 \
    -device pcie-root-port,id=pcie.0-root-port-5,slot=5,chassis=5,addr=0x2.0x3,bus=pcie.0 \
    -device virtio-net-pci,mac=9a:9b:1d:13:61:86,id=idKg9AzR,netdev=idxDM2m8,bus=pcie.0-root-port-5,addr=0x0  \
    -netdev tap,id=idxDM2m8,vhost=on \
    -device usb-tablet,id=usb-tablet1,bus=usb1.0,port=1  \
    -vnc :0  \
    -rtc base=utc,clock=host,driftfix=slew  \
    -boot menu=off,order=cdn,once=c,strict=off \
    -enable-kvm \
    -monitor stdio \
    -device pcie-root-port,id=pcie_extra_root_port_0,slot=6,chassis=6,addr=0x2.0x4,bus=pcie.0 \
    -qmp tcp:0:3000,server,nowait \

2. Add persistent bitmap to drive_data1
    { "execute": "block-dirty-bitmap-add", "arguments": {"node": "drive_data1", "name": "bitmap1","persistent":true}}
Ncat: Connection reset by peer.


Actual results:
 After step2, qemu core dump with info:
   (qemu) qemu: qemu_mutex_unlock_impl: Operation not permitted
test.txt: line 43: 25192 Aborted                 (core dumped) /usr/libexec/qemu-kvm -name 'avocado-vt-vm1' -sandbox on -machine q35 ..

 gdb info:
  (gdb) bt
#0  0x00007f0ef146370f in __GI_raise (sig=sig@entry=6)
    at ../sysdeps/unix/sysv/linux/raise.c:50
#1  0x00007f0ef144db25 in __GI_abort () at abort.c:79
#2  0x0000565022294dce in error_exit
    (err=<optimized out>, msg=msg@entry=0x56502243a730 <__func__.16350> "qemu_mutex_unlock_impl") at util/qemu-thread-posix.c:36
#3  0x00005650222950ba in qemu_mutex_unlock_impl
    (mutex=mutex@entry=0x5650244b0240, file=file@entry=0x565022439adf "util/async.c", line=line@entry=526) at util/qemu-thread-posix.c:108
#4  0x0000565022290029 in aio_context_release
    (ctx=ctx@entry=0x5650244b01e0) at util/async.c:526
#5  0x000056502221cd08 in bdrv_can_store_new_dirty_bitmap
    (bs=bs@entry=0x5650244dc820, name=name@entry=0x56502481d360 "bitmap1", granularity=granularity@entry=65536, errp=errp@entry=0x7fff22831718)
    at block/dirty-bitmap.c:542
#6  0x000056502206ae53 in qmp_block_dirty_bitmap_add
    (errp=0x7fff22831718, disabled=false, has_disabled=<optimized out>, persistent=<optimized out>, has_persistent=true, granularity=65536, has_gr--Type <RET> for more, q to quit, c to continue without paging--
anularity=<optimized out>, name=0x56502481d360 "bitmap1", node=<optimized out>) at blockdev.c:2894
#7  0x000056502206ae53 in qmp_block_dirty_bitmap_add
    (node=<optimized out>, name=0x56502481d360 "bitmap1", has_granularity=<optimized out>, granularity=<optimized out>, has_persistent=true, persistent=<optimized out>, has_disabled=false, disabled=false, errp=0x7fff22831718) at blockdev.c:2856
#8  0x00005650221847a3 in qmp_marshal_block_dirty_bitmap_add
    (args=<optimized out>, ret=<optimized out>, errp=0x7fff22831798)
    at qapi/qapi-commands-block-core.c:651
#9  0x0000565022247e6c in do_qmp_dispatch
    (errp=0x7fff22831790, allow_oob=<optimized out>, request=<optimized out>, cmds=0x565022b32d60 <qmp_commands>) at qapi/qmp-dispatch.c:132
#10 0x0000565022247e6c in qmp_dispatch
    (cmds=0x565022b32d60 <qmp_commands>, request=<optimized out>, allow_oob=<optimized out>) at qapi/qmp-dispatch.c:175
#11 0x0000565022166061 in monitor_qmp_dispatch
    (mon=0x56502450faa0, req=<optimized out>) at monitor/qmp.c:145
--Type <RET> for more, q to quit, c to continue without paging--
#12 0x00005650221666fa in monitor_qmp_bh_dispatcher
    (data=<optimized out>) at monitor/qmp.c:234
#13 0x000056502228f866 in aio_bh_call (bh=0x56502440eae0)
    at util/async.c:117
#14 0x000056502228f866 in aio_bh_poll (ctx=ctx@entry=0x56502440d7a0)
    at util/async.c:117
#15 0x0000565022292c54 in aio_dispatch (ctx=0x56502440d7a0)
    at util/aio-posix.c:459
#16 0x000056502228f742 in aio_ctx_dispatch
    (source=<optimized out>, callback=<optimized out>, user_data=<optimized out>) at util/async.c:260
#17 0x00007f0ef5ce667d in g_main_dispatch (context=0x56502449aa40)
    at gmain.c:3176
#18 0x00007f0ef5ce667d in g_main_context_dispatch
    (context=context@entry=0x56502449aa40) at gmain.c:3829
#19 0x0000565022291d08 in glib_pollfds_poll () at util/main-loop.c:219
#20 0x0000565022291d08 in os_host_main_loop_wait
    (timeout=<optimized out>) at util/main-loop.c:242
--Type <RET> for more, q to quit, c to continue without paging--
#21 0x0000565022291d08 in main_loop_wait (nonblocking=<optimized out>)
    at util/main-loop.c:518
#22 0x00005650220743c1 in main_loop () at vl.c:1828
#23 0x0000565021f20a72 in main
    (argc=<optimized out>, argv=<optimized out>, envp=<optimized out>)
    at vl.c:4504   

Expected results:
 Add persistent bitmap can be executed successfully.

Additional info:
 Test with qemu-kvm-4.2.0-1.module+el8.2.0+4793+b09dd2fb.x86_64, don't hit this issue, detail see bz#1778646.
 Will attach coredump file later.

Comment 2 aihua liang 2019-12-11 11:03:16 UTC
Don't hit this issue when disable data plane.

Comment 6 Nir Soffer 2020-01-07 15:01:03 UTC
Sergio, how can we consume the related patches mentioned in comment 4 with qemu
master or on Fedora?

We hit the same issue with qemu-4.2.0-2 on Fedora 30.

Comment 7 Sergio Lopez 2020-01-08 10:08:19 UTC
Hi Nir, I guess we should get the patch series upstreamed first.

Comment 8 Ademar Reis 2020-02-05 23:10:43 UTC
QEMU has been recently split into sub-components and as a one-time operation to avoid breakage of tools, we are setting the QEMU sub-component of this BZ to "General". Please review and change the sub-component if necessary the next time you review this BZ. Thanks

Comment 14 aihua liang 2020-02-19 01:55:37 UTC
Verified on qemu-kvm-4.2.0-10.module+el8.2.0+5740+c3dff59e, the problem has been resolved, set bug's status to "VERIFIED".

Test Steps:
 1.Start guest with qemu cmds:
   /usr/libexec/qemu-kvm \
    -name 'avocado-vt-vm1'  \
    -sandbox on  \
    -machine q35  \
    -nodefaults \
    -device VGA,bus=pcie.0,addr=0x1 \
    -m 7168  \
    -smp 4,maxcpus=4,cores=2,threads=1,dies=1,sockets=2  \
    -cpu 'Skylake-Client',+kvm_pv_unhalt  \
    -chardev socket,id=qmp_id_qmpmonitor1,path=/var/tmp/monitor-qmpmonitor1-20191210-025743-Q0JzJpKT,server,nowait \
    -mon chardev=qmp_id_qmpmonitor1,mode=control  \
    -chardev socket,id=qmp_id_catch_monitor,path=/var/tmp/monitor-catch_monitor-20191210-025743-Q0JzJpKT,server,nowait \
    -mon chardev=qmp_id_catch_monitor,mode=control \
    -device pvpanic,ioport=0x505,id=idDU2Q2E \
    -chardev socket,server,path=/var/tmp/serial-serial0-20191210-025743-Q0JzJpKT,id=chardev_serial0,nowait \
    -device isa-serial,id=serial0,chardev=chardev_serial0  \
    -chardev socket,id=seabioslog_id_20191210-025743-Q0JzJpKT,path=/var/tmp/seabios-20191210-025743-Q0JzJpKT,server,nowait \
    -device isa-debugcon,chardev=seabioslog_id_20191210-025743-Q0JzJpKT,iobase=0x402 \
    -device pcie-root-port,id=pcie.0-root-port-2,slot=2,chassis=2,addr=0x2,bus=pcie.0,multifunction=on \
    -device qemu-xhci,id=usb1,bus=pcie.0-root-port-2,addr=0x0 \
    -object iothread,id=iothread0 \
    -object iothread,id=iothread1 \
    -device pcie-root-port,id=pcie.0-root-port-3,slot=3,chassis=3,addr=0x2.0x1,bus=pcie.0 \
    -device virtio-scsi-pci,id=virtio_scsi_pci0,bus=pcie.0-root-port-3,addr=0x0,iothread=iothread1 \
    -blockdev node-name=file_image1,driver=file,aio=threads,filename=/home/kvm_autotest_root/images/rhel820-64-virtio-scsi.qcow2,cache.direct=on,cache.no-flush=off \
    -blockdev node-name=drive_image1,driver=qcow2,cache.direct=on,cache.no-flush=off,file=file_image1 \
    -device scsi-hd,id=image1,drive=drive_image1,write-cache=on \
    -device pcie-root-port,id=pcie.0-root-port-4,slot=4,chassis=4,addr=0x2.0x2,bus=pcie.0 \
    -blockdev node-name=file_data1,driver=file,aio=threads,filename=/home/data.qcow2,cache.direct=on,cache.no-flush=off \
    -blockdev node-name=drive_data1,driver=qcow2,cache.direct=on,cache.no-flush=off,file=file_data1 \
    -device virtio-blk-pci,id=data1,drive=drive_data1,write-cache=on,bus=pcie.0-root-port-4,iothread=iothread1 \
    -device pcie-root-port,id=pcie.0-root-port-5,slot=5,chassis=5,addr=0x2.0x3,bus=pcie.0 \
    -device virtio-net-pci,mac=9a:9b:1d:13:61:86,id=idKg9AzR,netdev=idxDM2m8,bus=pcie.0-root-port-5,addr=0x0  \
    -netdev tap,id=idxDM2m8,vhost=on \
    -device usb-tablet,id=usb-tablet1,bus=usb1.0,port=1  \
    -vnc :0  \
    -rtc base=utc,clock=host,driftfix=slew  \
    -boot menu=off,order=cdn,once=c,strict=off \
    -enable-kvm \
    -monitor stdio \
    -device pcie-root-port,id=pcie_extra_root_port_0,slot=6,chassis=6,addr=0x2.0x4,bus=pcie.0 \
    -qmp tcp:0:3000,server,nowait \

2. Add persistent bitmap to drive_data1
    { "execute": "block-dirty-bitmap-add", "arguments": {"node": "drive_data1", "name": "bitmap1","persistent":true}}

3. Check bitmap info:
    {"execute":"query-block"}
{"return": [{"io-status": "ok", "device": "", "locked": false, "removable": false, "inserted": {"iops_rd": 0, "detect_zeroes": "off", "image": {"virtual-size": 21474836480, "filename": "/home/kvm_autotest_root/images/rhel820-64-virtio-scsi.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": 6019284992, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false, "refcount-bits": 16, "corrupt": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "node-name": "drive_image1", "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "write_threshold": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "cache": {"no-flush": false, "direct": true, "writeback": true}, "file": "/home/kvm_autotest_root/images/rhel820-64-virtio-scsi.qcow2", "encryption_key_missing": false}, "qdev": "image1", "type": "unknown"}, {"io-status": "ok", "device": "", "locked": false, "removable": false, "inserted": {"iops_rd": 0, "detect_zeroes": "off", "image": {"virtual-size": 2147483648, "filename": "/home/data.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": 1050943488, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false, "refcount-bits": 16, "corrupt": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "node-name": "drive_data1", "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "write_threshold": 0, "dirty-bitmaps": [{"name": "bitmap1", "recording": true, "persistent": true, "busy": false, "status": "active", "granularity": 65536, "count": 0}], "encrypted": false, "bps": 0, "bps_rd": 0, "cache": {"no-flush": false, "direct": true, "writeback": true}, "file": "/home/data.qcow2", "encryption_key_missing": false}, "qdev": "/machine/peripheral/data1/virtio-backend", "dirty-bitmaps": [{"name": "bitmap1", "recording": true, "persistent": true, "busy": false, "status": "active", "granularity": 65536, "count": 0}], "type": "unknown"}]}

4. Quit vm, check bitmap info
# qemu-img info /home/data.qcow2 
image: /home/data.qcow2
file format: qcow2
virtual size: 2 GiB (2147483648 bytes)
disk size: 0.979 GiB
cluster_size: 65536
Format specific information:
    compat: 1.1
    lazy refcounts: false
    bitmaps:
        [0]:
            flags:
                [0]: auto
            name: bitmap1
            granularity: 65536
    refcount bits: 16
    corrupt: false


Note You need to log in before you can comment on or make changes to this bug.