Bug 2227118
| Summary: | [vdpa-blk] Can not share disk with driver virtio-blk-vhost-vdpa when migration or boot VMs on one host | ||
|---|---|---|---|
| Product: | Red Hat Enterprise Linux 9 | Reporter: | qing.wang <qinwang> |
| Component: | qemu-kvm | Assignee: | Stefano Garzarella <sgarzare> |
| qemu-kvm sub component: | virtio-blk,scsi | QA Contact: | qing.wang <qinwang> |
| Status: | CLOSED CURRENTRELEASE | Docs Contact: | |
| Severity: | medium | ||
| Priority: | medium | CC: | aliang, chayang, coli, jinzhao, juzhang, kwolf, lijin, qizhu, sgarzare, stefanha, virt-maint, xuwei, zhenyzha |
| Version: | 9.3 | Keywords: | TestOnly, Triaged |
| Target Milestone: | rc | ||
| Target Release: | --- | ||
| Hardware: | Unspecified | ||
| OS: | Unspecified | ||
| Whiteboard: | |||
| Fixed In Version: | Doc Type: | If docs needed, set a value | |
| Doc Text: | Story Points: | --- | |
| Clone Of: | Environment: | ||
| Last Closed: | 2023-08-15 03:33:52 UTC | Type: | --- |
| Regression: | --- | Mount Type: | --- |
| Documentation: | --- | CRM: | |
| Verified Versions: | Category: | --- | |
| oVirt Team: | --- | RHEL 7.3 requirements from Atomic Host: | |
| Cloudforms Team: | --- | Target Upstream Version: | |
| Embargoed: | |||
|
Description
qing.wang
2023-07-28 03:27:58 UTC
BOOT VMs on one host case: VM1: usr/libexec/qemu-kvm \ -name vm1 \ -machine q35,memory-backend=mem \ -object memory-backend-memfd,id=mem,size=6G,share=on \ -m 6G \ -smp 2 \ -cpu host,+kvm_pv_unhalt \ -device ich9-usb-ehci1,id=usb1 \ -device usb-tablet,id=usb-tablet1,bus=usb1.0,port=1 \ \ \ -device pcie-root-port,id=pcie-root-port-0,multifunction=on,bus=pcie.0,addr=0x3,chassis=1 \ -device pcie-root-port,id=pcie-root-port-1,port=0x1,addr=0x3.0x1,bus=pcie.0,chassis=2 \ -device pcie-root-port,id=pcie-root-port-2,port=0x2,addr=0x3.0x2,bus=pcie.0,chassis=3 \ -device pcie-root-port,id=pcie-root-port-3,port=0x3,addr=0x3.0x3,bus=pcie.0,chassis=4 \ -device pcie-root-port,id=pcie-root-port-4,port=0x4,addr=0x3.0x4,bus=pcie.0,chassis=5 \ -device pcie-root-port,id=pcie-root-port-5,port=0x5,addr=0x3.0x5,bus=pcie.0,chassis=6 \ -device pcie-root-port,id=pcie-root-port-6,port=0x6,addr=0x3.0x6,bus=pcie.0,chassis=7 \ -device pcie-root-port,id=pcie-root-port-7,port=0x7,addr=0x3.0x7,bus=pcie.0,chassis=8 \ -device pcie-root-port,id=pcie_extra_root_port_0,bus=pcie.0,addr=0x4 \ -object iothread,id=iothread0 \ -device virtio-scsi-pci,id=scsi0,bus=pcie-root-port-0,iothread=iothread0 \ -blockdev driver=qcow2,file.driver=file,cache.direct=off,cache.no-flush=on,file.filename=/home/kvm_autotest_root/images/rhel930-64-virtio-scsi-1.qcow2,node-name=drive_image1,file.aio=threads \ -device scsi-hd,id=os,drive=drive_image1,share-rw=on,bus=scsi0.0,bootindex=0,serial=OS_DISK \ \ -blockdev node-name=prot_stg0,driver=virtio-blk-vhost-vdpa,path=/dev/vhost-vdpa-0,cache.direct=on,discard=unmap,detect-zeroes=on \ -blockdev node-name=fmt_stg0,driver=raw,file=prot_stg0 \ -device virtio-blk-pci,iothread=iothread0,share-rw=on,serial=data0,bus=pcie-root-port-4,addr=0,id=stg0,drive=fmt_stg0,bootindex=1 \ \ -blockdev node-name=prot_stg1,driver=virtio-blk-vhost-vdpa,path=/dev/vhost-vdpa-1,cache.direct=on,auto-read-only=on,read-only=off,force-share=off \ -blockdev node-name=fmt_stg1,driver=raw,file=prot_stg1 \ -device scsi-hd,id=stg1,drive=fmt_stg1,share-rw=on,serial=data1,bootindex=2 \ -monitor stdio \ \ -device virtio-net-pci,mac=9a:b5:b6:b1:b2:b7,id=nic1,netdev=nicpci,bus=pcie-root-port-7 \ -netdev tap,id=nicpci \ -boot menu=on,reboot-timeout=1000,strict=off \ \ -chardev socket,id=socket-serial,path=/var/tmp/socket-serial,logfile=/var/tmp/file-serial.log,mux=on,server=on,wait=off \ -serial chardev:socket-serial \ -chardev file,path=/var/tmp/file-bios.log,id=file-bios \ -device isa-debugcon,chardev=file-bios,iobase=0x402 \ \ -chardev socket,id=socket-qmp,path=/var/tmp/socket-qmp,logfile=/var/tmp/file-qmp.log,mux=on,server=on,wait=off \ -mon chardev=socket-qmp,mode=control \ -chardev socket,id=socket-hmp,path=/var/tmp/socket-hmp,logfile=/var/tmp/file-hmp.log,mux=on,server=on,wait=off \ -mon chardev=socket-hmp,mode=readline \ -vnc :5 -qmp tcp:0:5955,server=on,wait=off VM2: /usr/libexec/qemu-kvm \ -name vm2 \ -machine q35,memory-backend=mem \ -object memory-backend-memfd,id=mem,size=6G,share=on \ -m 6G \ -smp 2 \ -cpu host,+kvm_pv_unhalt \ -device ich9-usb-ehci1,id=usb1 \ -device usb-tablet,id=usb-tablet1,bus=usb1.0,port=1 \ \ \ -device pcie-root-port,id=pcie-root-port-0,multifunction=on,bus=pcie.0,addr=0x3,chassis=1 \ -device pcie-root-port,id=pcie-root-port-1,port=0x1,addr=0x3.0x1,bus=pcie.0,chassis=2 \ -device pcie-root-port,id=pcie-root-port-2,port=0x2,addr=0x3.0x2,bus=pcie.0,chassis=3 \ -device pcie-root-port,id=pcie-root-port-3,port=0x3,addr=0x3.0x3,bus=pcie.0,chassis=4 \ -device pcie-root-port,id=pcie-root-port-4,port=0x4,addr=0x3.0x4,bus=pcie.0,chassis=5 \ -device pcie-root-port,id=pcie-root-port-5,port=0x5,addr=0x3.0x5,bus=pcie.0,chassis=6 \ -device pcie-root-port,id=pcie-root-port-6,port=0x6,addr=0x3.0x6,bus=pcie.0,chassis=7 \ -device pcie-root-port,id=pcie-root-port-7,port=0x7,addr=0x3.0x7,bus=pcie.0,chassis=8 \ -device pcie-root-port,id=pcie_extra_root_port_0,bus=pcie.0,addr=0x4 \ -object iothread,id=iothread0 \ -device virtio-scsi-pci,id=scsi0,bus=pcie-root-port-0,iothread=iothread0 \ -blockdev driver=qcow2,file.driver=file,cache.direct=off,cache.no-flush=on,file.filename=/home/kvm_autotest_root/images/rhel930-64-virtio-scsi-2.qcow2,node-name=drive_image1,file.aio=threads \ -device scsi-hd,id=os,drive=drive_image1,share-rw=on,bus=scsi0.0,bootindex=0,serial=OS_DISK \ \ -blockdev node-name=prot_stg0,driver=virtio-blk-vhost-vdpa,path=/dev/vhost-vdpa-0,cache.direct=on,discard=unmap,detect-zeroes=on \ -blockdev node-name=fmt_stg0,driver=raw,file=prot_stg0 \ -device virtio-blk-pci,iothread=iothread0,share-rw=on,serial=data0,bus=pcie-root-port-4,addr=0,id=stg0,drive=fmt_stg0,bootindex=1 \ \ -blockdev node-name=prot_stg1,driver=virtio-blk-vhost-vdpa,path=/dev/vhost-vdpa-1,cache.direct=on,auto-read-only=on,read-only=off,force-share=off \ -blockdev node-name=fmt_stg1,driver=raw,file=prot_stg1 \ -device scsi-hd,id=stg1,drive=fmt_stg1,share-rw=on,serial=data1,bootindex=2 \ -monitor stdio \ \ -device virtio-net-pci,mac=9a:b5:b6:b1:b2:b7,id=nic1,netdev=nicpci,bus=pcie-root-port-7 \ -netdev tap,id=nicpci \ -boot menu=on,reboot-timeout=1000,strict=off \ \ -chardev socket,id=socket-serial,path=/var/tmp/socket-serial,logfile=/var/tmp/file-serial.log,mux=on,server=on,wait=off \ -serial chardev:socket-serial \ -chardev file,path=/var/tmp/file-bios.log,id=file-bios \ -device isa-debugcon,chardev=file-bios,iobase=0x402 \ \ -chardev socket,id=socket-qmp,path=/var/tmp/socket-qmp,logfile=/var/tmp/file-qmp.log,mux=on,server=on,wait=off \ -mon chardev=socket-qmp,mode=control \ -chardev socket,id=socket-hmp,path=/var/tmp/socket-hmp,logfile=/var/tmp/file-hmp.log,mux=on,server=on,wait=off \ -mon chardev=socket-hmp,mode=readline \ -vnc :6 -qmp tcp:0:5956,server=on,wait=off (In reply to qing.wang from comment #0) > Description of problem: > Try to migrate VM on one host or Boot two VMs with the same disks on one > host. > > > Boot VM the disk enable share-rw=on, but it failed to boot the second VM > with the same disk. > > -blockdev > node-name=prot_stg0,driver=virtio-blk-vhost-vdpa,path=/dev/vhost-vdpa-0, > cache.direct=on,discard=unmap,detect-zeroes=on \ > -blockdev node-name=fmt_stg0,driver=raw,file=prot_stg0 \ > -device > virtio-blk-pci,iothread=iothread0,share-rw=on,serial=data0,bus=pcie-root- > port-4,addr=0,id=stg0,drive=fmt_stg0,bootindex=1 \ > \ > > > qemu-kvm: -blockdev > node-name=prot_stg0,driver=virtio-blk-vhost-vdpa,path=/dev/vhost-vdpa-0, > cache.direct=on,discard=unmap,detect-zeroes=on: Could not open > '/dev/vhost-vdpa-0': Device or resource busy This is the expected, since a vhost device can't be shared between multiple VMs, so you need to use different vhost device, configured with the same backend. Real HW will allow doing this, and now also the simulator is supporting it starting from commit abebb16254b3 ("vdpa_sim_blk: support shared backend"). That feature is backported through: https://issues.redhat.com/browse/RHEL-806 (I don't know how to put the dependency between BZ and JIRA) So, with that commit backported in the next release, you can load vdpa_sim_blk with `shared_backend` parameter set to true (default is false): $ modprobe vdpa_sim_blk shared_backend=1 $ cat /sys/module/vdpa_sim_blk/parameters/shared_backend Y At this point, all vdpa_sim_blk devices you create will have the same shared ramdisk to support the migration, so I recommend that you use only one device per VM, otherwise the changes you make will be propagated to all devices. (This is just a simulator to help with testing and developing). For example, this is what I did to test it: $ modprobe vhost_vdpa $ modprobe vdpa_sim_blk shared_backend=1 $ vdpa dev add mgmtdev vdpasim_blk name vdpa0 $ vdpa dev add mgmtdev vdpasim_blk name vdpa1 $ /usr/libexec/qemu-kvm ... \ ... -blockdev node-name=drive_src1,driver=virtio-blk-vhost-vdpa,path=/dev/vhost-vdpa-1,cache.direct=on \ ... -incoming tcp:0:3333 $ /usr/libexec/qemu-kvm ... \ ... -blockdev node-name=drive_src1,driver=virtio-blk-vhost-vdpa,path=/dev/vhost-vdpa-0,cache.direct=on # create a fs on the device and a file, then migrate (qemu) migrate -d tcp:0:3333 This of course works only on the same host, for multiple hosts we will need to use a real hardware. I set a TestOnly flag for now, since all the code should be already there or backported through https://issues.redhat.com/browse/RHEL-806 Pass test on Red Hat Enterprise Linux release 9.3 Beta (Plow) 5.14.0-352.el9.x86_64 qemu-kvm-8.0.0-11.el9.x86_64 seabios-bin-1.16.1-1.el9.noarch edk2-ovmf-20230524-2.el9.noarch libvirt-9.5.0-5.el9.x86_64 virtio-win-prewhql-0.1-240.iso Source VM: /usr/libexec/qemu-kvm \ -name src_vm \ -machine q35,memory-backend=mem \ -object memory-backend-memfd,id=mem,size=6G,share=on \ -m 6G \ -smp 2 \ -cpu host,+kvm_pv_unhalt \ -device ich9-usb-ehci1,id=usb1 \ -device usb-tablet,id=usb-tablet1,bus=usb1.0,port=1 \ -device pcie-root-port,id=pcie-root-port-0,multifunction=on,bus=pcie.0,addr=0x3,chassis=1 \ -device pcie-root-port,id=pcie-root-port-1,port=0x1,addr=0x3.0x1,bus=pcie.0,chassis=2 \ -device pcie-root-port,id=pcie-root-port-2,port=0x2,addr=0x3.0x2,bus=pcie.0,chassis=3 \ -device pcie-root-port,id=pcie-root-port-3,port=0x3,addr=0x3.0x3,bus=pcie.0,chassis=4 \ -device pcie-root-port,id=pcie-root-port-4,port=0x4,addr=0x3.0x4,bus=pcie.0,chassis=5 \ -device pcie-root-port,id=pcie-root-port-5,port=0x5,addr=0x3.0x5,bus=pcie.0,chassis=6 \ -device pcie-root-port,id=pcie-root-port-6,port=0x6,addr=0x3.0x6,bus=pcie.0,chassis=7 \ -device pcie-root-port,id=pcie-root-port-7,port=0x7,addr=0x3.0x7,bus=pcie.0,chassis=8 \ -device pcie-root-port,id=pcie_extra_root_port_0,bus=pcie.0,addr=0x4 \ -object iothread,id=iothread0 \ -device virtio-scsi-pci,id=scsi0,bus=pcie-root-port-0,iothread=iothread0 \ -blockdev driver=qcow2,file.driver=file,cache.direct=off,cache.no-flush=on,file.filename=/home/kvm_autotest_root/images/rhel930-64-virtio-scsi.qcow2,node-name=drive_image1,file.aio=threads \ -device scsi-hd,id=os,drive=drive_image1,share-rw=on,bus=scsi0.0,bootindex=0,serial=OS_DISK \ -blockdev node-name=prot_stg0,driver=virtio-blk-vhost-vdpa,path=/dev/vhost-vdpa-0,cache.direct=on,discard=unmap,detect-zeroes=on \ -blockdev node-name=fmt_stg0,driver=raw,file=prot_stg0 \ -device virtio-blk-pci,iothread=iothread0,share-rw=on,serial=data0,bus=pcie-root-port-4,addr=0,id=stg0,drive=fmt_stg0,bootindex=1 \ -monitor stdio \ -device virtio-net-pci,mac=9a:b5:b6:b1:b2:b7,id=nic1,netdev=nicpci,bus=pcie-root-port-7 \ -netdev tap,id=nicpci \ -boot menu=on,reboot-timeout=1000,strict=off \ -chardev socket,id=socket-serial,path=/var/tmp/socket-serial,logfile=/var/tmp/file-serial.log,mux=on,server=on,wait=off \ -serial chardev:socket-serial \ -chardev file,path=/var/tmp/file-bios.log,id=file-bios \ -device isa-debugcon,chardev=file-bios,iobase=0x402 \ -chardev socket,id=socket-qmp,path=/var/tmp/socket-qmp,logfile=/var/tmp/file-qmp.log,mux=on,server=on,wait=off \ -mon chardev=socket-qmp,mode=control \ -chardev socket,id=socket-hmp,path=/var/tmp/socket-hmp,logfile=/var/tmp/file-hmp.log,mux=on,server=on,wait=off \ -mon chardev=socket-hmp,mode=readline \ -vnc :5 \ -qmp tcp:0:5955,server=on,wait=off Dst VM: /usr/libexec/qemu-kvm \ -name dst_vm \ -machine q35,memory-backend=mem \ -object memory-backend-memfd,id=mem,size=6G,share=on \ -m 6G \ -smp 2 \ -cpu host,+kvm_pv_unhalt \ -device ich9-usb-ehci1,id=usb1 \ -device usb-tablet,id=usb-tablet1,bus=usb1.0,port=1 \ -device pcie-root-port,id=pcie-root-port-0,multifunction=on,bus=pcie.0,addr=0x3,chassis=1 \ -device pcie-root-port,id=pcie-root-port-1,port=0x1,addr=0x3.0x1,bus=pcie.0,chassis=2 \ -device pcie-root-port,id=pcie-root-port-2,port=0x2,addr=0x3.0x2,bus=pcie.0,chassis=3 \ -device pcie-root-port,id=pcie-root-port-3,port=0x3,addr=0x3.0x3,bus=pcie.0,chassis=4 \ -device pcie-root-port,id=pcie-root-port-4,port=0x4,addr=0x3.0x4,bus=pcie.0,chassis=5 \ -device pcie-root-port,id=pcie-root-port-5,port=0x5,addr=0x3.0x5,bus=pcie.0,chassis=6 \ -device pcie-root-port,id=pcie-root-port-6,port=0x6,addr=0x3.0x6,bus=pcie.0,chassis=7 \ -device pcie-root-port,id=pcie-root-port-7,port=0x7,addr=0x3.0x7,bus=pcie.0,chassis=8 \ -device pcie-root-port,id=pcie_extra_root_port_0,bus=pcie.0,addr=0x4 \ -object iothread,id=iothread0 \ -device virtio-scsi-pci,id=scsi0,bus=pcie-root-port-0,iothread=iothread0 \ -blockdev driver=qcow2,file.driver=file,cache.direct=off,cache.no-flush=on,file.filename=/home/kvm_autotest_root/images/rhel930-64-virtio-scsi.qcow2,node-name=drive_image1,file.aio=threads \ -device scsi-hd,id=os,drive=drive_image1,share-rw=on,bus=scsi0.0,bootindex=0,serial=OS_DISK \ -blockdev node-name=prot_stg0,driver=virtio-blk-vhost-vdpa,path=/dev/vhost-vdpa-1,cache.direct=on,discard=unmap,detect-zeroes=on \ -blockdev node-name=fmt_stg0,driver=raw,file=prot_stg0 \ -device virtio-blk-pci,iothread=iothread0,share-rw=on,serial=data0,bus=pcie-root-port-4,addr=0,id=stg0,drive=fmt_stg0,bootindex=1 \ -monitor stdio \ -device virtio-net-pci,mac=9a:b5:b6:b1:b2:b7,id=nic1,netdev=nicpci,bus=pcie-root-port-7 \ -netdev tap,id=nicpci \ -boot menu=on,reboot-timeout=1000,strict=off \ -chardev socket,id=socket-serial,path=/var/tmp/socket-serial,logfile=/var/tmp/file-serial.log,mux=on,server=on,wait=off \ -serial chardev:socket-serial \ -chardev file,path=/var/tmp/file-bios.log,id=file-bios \ -device isa-debugcon,chardev=file-bios,iobase=0x402 \ -chardev socket,id=socket-qmp,path=/var/tmp/socket-qmp,logfile=/var/tmp/file-qmp.log,mux=on,server=on,wait=off \ -mon chardev=socket-qmp,mode=control \ -chardev socket,id=socket-hmp,path=/var/tmp/socket-hmp,logfile=/var/tmp/file-hmp.log,mux=on,server=on,wait=off \ -mon chardev=socket-hmp,mode=readline \ -vnc :6 \ -qmp tcp:0:5956,server=on,wait=off \ -incoming defer migration succeed |