BOOT VMs on one host case:
VM1:

usr/libexec/qemu-kvm \
  -name vm1 \
  -machine q35,memory-backend=mem \
  -object memory-backend-memfd,id=mem,size=6G,share=on \
  -m  6G \
  -smp 2 \
  -cpu host,+kvm_pv_unhalt \
  -device ich9-usb-ehci1,id=usb1 \
  -device usb-tablet,id=usb-tablet1,bus=usb1.0,port=1 \
   \
   \
  -device pcie-root-port,id=pcie-root-port-0,multifunction=on,bus=pcie.0,addr=0x3,chassis=1 \
  -device pcie-root-port,id=pcie-root-port-1,port=0x1,addr=0x3.0x1,bus=pcie.0,chassis=2 \
  -device pcie-root-port,id=pcie-root-port-2,port=0x2,addr=0x3.0x2,bus=pcie.0,chassis=3 \
  -device pcie-root-port,id=pcie-root-port-3,port=0x3,addr=0x3.0x3,bus=pcie.0,chassis=4 \
  -device pcie-root-port,id=pcie-root-port-4,port=0x4,addr=0x3.0x4,bus=pcie.0,chassis=5 \
  -device pcie-root-port,id=pcie-root-port-5,port=0x5,addr=0x3.0x5,bus=pcie.0,chassis=6 \
  -device pcie-root-port,id=pcie-root-port-6,port=0x6,addr=0x3.0x6,bus=pcie.0,chassis=7 \
  -device pcie-root-port,id=pcie-root-port-7,port=0x7,addr=0x3.0x7,bus=pcie.0,chassis=8 \
  -device pcie-root-port,id=pcie_extra_root_port_0,bus=pcie.0,addr=0x4  \
  -object iothread,id=iothread0 \
  -device virtio-scsi-pci,id=scsi0,bus=pcie-root-port-0,iothread=iothread0 \
  -blockdev driver=qcow2,file.driver=file,cache.direct=off,cache.no-flush=on,file.filename=/home/kvm_autotest_root/images/rhel930-64-virtio-scsi-1.qcow2,node-name=drive_image1,file.aio=threads   \
  -device scsi-hd,id=os,drive=drive_image1,share-rw=on,bus=scsi0.0,bootindex=0,serial=OS_DISK   \
  \
  -blockdev node-name=prot_stg0,driver=virtio-blk-vhost-vdpa,path=/dev/vhost-vdpa-0,cache.direct=on,discard=unmap,detect-zeroes=on \
  -blockdev node-name=fmt_stg0,driver=raw,file=prot_stg0 \
  -device virtio-blk-pci,iothread=iothread0,share-rw=on,serial=data0,bus=pcie-root-port-4,addr=0,id=stg0,drive=fmt_stg0,bootindex=1 \
  \
  -blockdev node-name=prot_stg1,driver=virtio-blk-vhost-vdpa,path=/dev/vhost-vdpa-1,cache.direct=on,auto-read-only=on,read-only=off,force-share=off \
  -blockdev node-name=fmt_stg1,driver=raw,file=prot_stg1 \
  -device scsi-hd,id=stg1,drive=fmt_stg1,share-rw=on,serial=data1,bootindex=2 \
  -monitor stdio \
  \
  -device virtio-net-pci,mac=9a:b5:b6:b1:b2:b7,id=nic1,netdev=nicpci,bus=pcie-root-port-7 \
  -netdev tap,id=nicpci \
  -boot menu=on,reboot-timeout=1000,strict=off \
  \
  -chardev socket,id=socket-serial,path=/var/tmp/socket-serial,logfile=/var/tmp/file-serial.log,mux=on,server=on,wait=off \
  -serial chardev:socket-serial \
  -chardev file,path=/var/tmp/file-bios.log,id=file-bios \
  -device isa-debugcon,chardev=file-bios,iobase=0x402 \
  \
  -chardev socket,id=socket-qmp,path=/var/tmp/socket-qmp,logfile=/var/tmp/file-qmp.log,mux=on,server=on,wait=off \
  -mon chardev=socket-qmp,mode=control \
  -chardev socket,id=socket-hmp,path=/var/tmp/socket-hmp,logfile=/var/tmp/file-hmp.log,mux=on,server=on,wait=off \
  -mon chardev=socket-hmp,mode=readline \
  -vnc :5 -qmp tcp:0:5955,server=on,wait=off


VM2:

/usr/libexec/qemu-kvm \
  -name vm2 \
  -machine q35,memory-backend=mem \
  -object memory-backend-memfd,id=mem,size=6G,share=on \
  -m  6G \
  -smp 2 \
  -cpu host,+kvm_pv_unhalt \
  -device ich9-usb-ehci1,id=usb1 \
  -device usb-tablet,id=usb-tablet1,bus=usb1.0,port=1 \
   \
   \
  -device pcie-root-port,id=pcie-root-port-0,multifunction=on,bus=pcie.0,addr=0x3,chassis=1 \
  -device pcie-root-port,id=pcie-root-port-1,port=0x1,addr=0x3.0x1,bus=pcie.0,chassis=2 \
  -device pcie-root-port,id=pcie-root-port-2,port=0x2,addr=0x3.0x2,bus=pcie.0,chassis=3 \
  -device pcie-root-port,id=pcie-root-port-3,port=0x3,addr=0x3.0x3,bus=pcie.0,chassis=4 \
  -device pcie-root-port,id=pcie-root-port-4,port=0x4,addr=0x3.0x4,bus=pcie.0,chassis=5 \
  -device pcie-root-port,id=pcie-root-port-5,port=0x5,addr=0x3.0x5,bus=pcie.0,chassis=6 \
  -device pcie-root-port,id=pcie-root-port-6,port=0x6,addr=0x3.0x6,bus=pcie.0,chassis=7 \
  -device pcie-root-port,id=pcie-root-port-7,port=0x7,addr=0x3.0x7,bus=pcie.0,chassis=8 \
  -device pcie-root-port,id=pcie_extra_root_port_0,bus=pcie.0,addr=0x4  \
  -object iothread,id=iothread0 \
  -device virtio-scsi-pci,id=scsi0,bus=pcie-root-port-0,iothread=iothread0 \
  -blockdev driver=qcow2,file.driver=file,cache.direct=off,cache.no-flush=on,file.filename=/home/kvm_autotest_root/images/rhel930-64-virtio-scsi-2.qcow2,node-name=drive_image1,file.aio=threads   \
  -device scsi-hd,id=os,drive=drive_image1,share-rw=on,bus=scsi0.0,bootindex=0,serial=OS_DISK   \
  \
  -blockdev node-name=prot_stg0,driver=virtio-blk-vhost-vdpa,path=/dev/vhost-vdpa-0,cache.direct=on,discard=unmap,detect-zeroes=on \
  -blockdev node-name=fmt_stg0,driver=raw,file=prot_stg0 \
  -device virtio-blk-pci,iothread=iothread0,share-rw=on,serial=data0,bus=pcie-root-port-4,addr=0,id=stg0,drive=fmt_stg0,bootindex=1 \
  \
  -blockdev node-name=prot_stg1,driver=virtio-blk-vhost-vdpa,path=/dev/vhost-vdpa-1,cache.direct=on,auto-read-only=on,read-only=off,force-share=off \
  -blockdev node-name=fmt_stg1,driver=raw,file=prot_stg1 \
  -device scsi-hd,id=stg1,drive=fmt_stg1,share-rw=on,serial=data1,bootindex=2 \
  -monitor stdio \
  \
  -device virtio-net-pci,mac=9a:b5:b6:b1:b2:b7,id=nic1,netdev=nicpci,bus=pcie-root-port-7 \
  -netdev tap,id=nicpci \
  -boot menu=on,reboot-timeout=1000,strict=off \
  \
  -chardev socket,id=socket-serial,path=/var/tmp/socket-serial,logfile=/var/tmp/file-serial.log,mux=on,server=on,wait=off \
  -serial chardev:socket-serial \
  -chardev file,path=/var/tmp/file-bios.log,id=file-bios \
  -device isa-debugcon,chardev=file-bios,iobase=0x402 \
  \
  -chardev socket,id=socket-qmp,path=/var/tmp/socket-qmp,logfile=/var/tmp/file-qmp.log,mux=on,server=on,wait=off \
  -mon chardev=socket-qmp,mode=control \
  -chardev socket,id=socket-hmp,path=/var/tmp/socket-hmp,logfile=/var/tmp/file-hmp.log,mux=on,server=on,wait=off \
  -mon chardev=socket-hmp,mode=readline \
  -vnc :6 -qmp tcp:0:5956,server=on,wait=off

(In reply to qing.wang from comment #0)
> Description of problem:
> Try to migrate VM on one host or Boot two VMs with the same disks on one
> host.
> 
> 
> Boot VM the disk enable share-rw=on, but it failed to boot the second VM
> with the same disk. 
> 
> -blockdev
> node-name=prot_stg0,driver=virtio-blk-vhost-vdpa,path=/dev/vhost-vdpa-0,
> cache.direct=on,discard=unmap,detect-zeroes=on \
>   -blockdev node-name=fmt_stg0,driver=raw,file=prot_stg0 \
>   -device
> virtio-blk-pci,iothread=iothread0,share-rw=on,serial=data0,bus=pcie-root-
> port-4,addr=0,id=stg0,drive=fmt_stg0,bootindex=1 \
>   \
> 
> 
> qemu-kvm: -blockdev
> node-name=prot_stg0,driver=virtio-blk-vhost-vdpa,path=/dev/vhost-vdpa-0,
> cache.direct=on,discard=unmap,detect-zeroes=on: Could not open
> '/dev/vhost-vdpa-0': Device or resource busy


This is the expected, since a vhost device can't be shared between multiple VMs, so you need to use different vhost device, configured with the same backend.
Real HW will allow doing this, and now also the simulator is supporting it starting from commit abebb16254b3 ("vdpa_sim_blk: support shared backend").
That feature is backported through: https://issues.redhat.com/browse/RHEL-806
(I don't know how to put the dependency between BZ and JIRA)

So, with that commit backported in the next release, you can load vdpa_sim_blk with `shared_backend` parameter set to true (default is false):
    $ modprobe vdpa_sim_blk shared_backend=1
    $ cat /sys/module/vdpa_sim_blk/parameters/shared_backend 
    Y

At this point, all vdpa_sim_blk devices you create will have the same shared ramdisk to support the migration, so I recommend that you use only one device per VM, otherwise the changes you make will be propagated to all devices. (This is just a simulator to help with testing and developing).

For example, this is what I did to test it:
    $ modprobe vhost_vdpa
    $ modprobe vdpa_sim_blk shared_backend=1

    $ vdpa dev add mgmtdev vdpasim_blk name vdpa0
    $ vdpa dev add mgmtdev vdpasim_blk name vdpa1

    $ /usr/libexec/qemu-kvm ... \
      ...
      -blockdev node-name=drive_src1,driver=virtio-blk-vhost-vdpa,path=/dev/vhost-vdpa-1,cache.direct=on \
      ...
      -incoming tcp:0:3333

    $ /usr/libexec/qemu-kvm ... \
      ...
      -blockdev node-name=drive_src1,driver=virtio-blk-vhost-vdpa,path=/dev/vhost-vdpa-0,cache.direct=on

      # create a fs on the device and a file, then migrate

      (qemu) migrate -d tcp:0:3333

This of course works only on the same host, for multiple hosts we will need to use a real hardware.

I set a TestOnly flag for now, since all the code should be already there or backported through https://issues.redhat.com/browse/RHEL-806

Pass test on
Red Hat Enterprise Linux release 9.3 Beta (Plow)
5.14.0-352.el9.x86_64
qemu-kvm-8.0.0-11.el9.x86_64
seabios-bin-1.16.1-1.el9.noarch
edk2-ovmf-20230524-2.el9.noarch
libvirt-9.5.0-5.el9.x86_64
virtio-win-prewhql-0.1-240.iso

Source VM:

/usr/libexec/qemu-kvm \
 -name src_vm \
 -machine q35,memory-backend=mem \
 -object memory-backend-memfd,id=mem,size=6G,share=on \
 -m 6G \
 -smp 2 \
 -cpu host,+kvm_pv_unhalt \
 -device ich9-usb-ehci1,id=usb1 \
 -device usb-tablet,id=usb-tablet1,bus=usb1.0,port=1 \
 -device pcie-root-port,id=pcie-root-port-0,multifunction=on,bus=pcie.0,addr=0x3,chassis=1 \
 -device pcie-root-port,id=pcie-root-port-1,port=0x1,addr=0x3.0x1,bus=pcie.0,chassis=2 \
 -device pcie-root-port,id=pcie-root-port-2,port=0x2,addr=0x3.0x2,bus=pcie.0,chassis=3 \
 -device pcie-root-port,id=pcie-root-port-3,port=0x3,addr=0x3.0x3,bus=pcie.0,chassis=4 \
 -device pcie-root-port,id=pcie-root-port-4,port=0x4,addr=0x3.0x4,bus=pcie.0,chassis=5 \
 -device pcie-root-port,id=pcie-root-port-5,port=0x5,addr=0x3.0x5,bus=pcie.0,chassis=6 \
 -device pcie-root-port,id=pcie-root-port-6,port=0x6,addr=0x3.0x6,bus=pcie.0,chassis=7 \
 -device pcie-root-port,id=pcie-root-port-7,port=0x7,addr=0x3.0x7,bus=pcie.0,chassis=8 \
 -device pcie-root-port,id=pcie_extra_root_port_0,bus=pcie.0,addr=0x4 \
 -object iothread,id=iothread0 \
 -device virtio-scsi-pci,id=scsi0,bus=pcie-root-port-0,iothread=iothread0 \
 -blockdev driver=qcow2,file.driver=file,cache.direct=off,cache.no-flush=on,file.filename=/home/kvm_autotest_root/images/rhel930-64-virtio-scsi.qcow2,node-name=drive_image1,file.aio=threads \
 -device scsi-hd,id=os,drive=drive_image1,share-rw=on,bus=scsi0.0,bootindex=0,serial=OS_DISK \
 -blockdev node-name=prot_stg0,driver=virtio-blk-vhost-vdpa,path=/dev/vhost-vdpa-0,cache.direct=on,discard=unmap,detect-zeroes=on \
 -blockdev node-name=fmt_stg0,driver=raw,file=prot_stg0 \
 -device virtio-blk-pci,iothread=iothread0,share-rw=on,serial=data0,bus=pcie-root-port-4,addr=0,id=stg0,drive=fmt_stg0,bootindex=1 \
 -monitor stdio \
 -device virtio-net-pci,mac=9a:b5:b6:b1:b2:b7,id=nic1,netdev=nicpci,bus=pcie-root-port-7 \
 -netdev tap,id=nicpci \
 -boot menu=on,reboot-timeout=1000,strict=off \
 -chardev socket,id=socket-serial,path=/var/tmp/socket-serial,logfile=/var/tmp/file-serial.log,mux=on,server=on,wait=off \
 -serial chardev:socket-serial \
 -chardev file,path=/var/tmp/file-bios.log,id=file-bios \
 -device isa-debugcon,chardev=file-bios,iobase=0x402 \
 -chardev socket,id=socket-qmp,path=/var/tmp/socket-qmp,logfile=/var/tmp/file-qmp.log,mux=on,server=on,wait=off \
 -mon chardev=socket-qmp,mode=control \
 -chardev socket,id=socket-hmp,path=/var/tmp/socket-hmp,logfile=/var/tmp/file-hmp.log,mux=on,server=on,wait=off \
 -mon chardev=socket-hmp,mode=readline \
 -vnc :5 \
 -qmp tcp:0:5955,server=on,wait=off

Dst VM:
/usr/libexec/qemu-kvm \
 -name dst_vm \
 -machine q35,memory-backend=mem \
 -object memory-backend-memfd,id=mem,size=6G,share=on \
 -m 6G \
 -smp 2 \
 -cpu host,+kvm_pv_unhalt \
 -device ich9-usb-ehci1,id=usb1 \
 -device usb-tablet,id=usb-tablet1,bus=usb1.0,port=1 \
 -device pcie-root-port,id=pcie-root-port-0,multifunction=on,bus=pcie.0,addr=0x3,chassis=1 \
 -device pcie-root-port,id=pcie-root-port-1,port=0x1,addr=0x3.0x1,bus=pcie.0,chassis=2 \
 -device pcie-root-port,id=pcie-root-port-2,port=0x2,addr=0x3.0x2,bus=pcie.0,chassis=3 \
 -device pcie-root-port,id=pcie-root-port-3,port=0x3,addr=0x3.0x3,bus=pcie.0,chassis=4 \
 -device pcie-root-port,id=pcie-root-port-4,port=0x4,addr=0x3.0x4,bus=pcie.0,chassis=5 \
 -device pcie-root-port,id=pcie-root-port-5,port=0x5,addr=0x3.0x5,bus=pcie.0,chassis=6 \
 -device pcie-root-port,id=pcie-root-port-6,port=0x6,addr=0x3.0x6,bus=pcie.0,chassis=7 \
 -device pcie-root-port,id=pcie-root-port-7,port=0x7,addr=0x3.0x7,bus=pcie.0,chassis=8 \
 -device pcie-root-port,id=pcie_extra_root_port_0,bus=pcie.0,addr=0x4 \
 -object iothread,id=iothread0 \
 -device virtio-scsi-pci,id=scsi0,bus=pcie-root-port-0,iothread=iothread0 \
 -blockdev driver=qcow2,file.driver=file,cache.direct=off,cache.no-flush=on,file.filename=/home/kvm_autotest_root/images/rhel930-64-virtio-scsi.qcow2,node-name=drive_image1,file.aio=threads \
 -device scsi-hd,id=os,drive=drive_image1,share-rw=on,bus=scsi0.0,bootindex=0,serial=OS_DISK \
 -blockdev node-name=prot_stg0,driver=virtio-blk-vhost-vdpa,path=/dev/vhost-vdpa-1,cache.direct=on,discard=unmap,detect-zeroes=on \
 -blockdev node-name=fmt_stg0,driver=raw,file=prot_stg0 \
 -device virtio-blk-pci,iothread=iothread0,share-rw=on,serial=data0,bus=pcie-root-port-4,addr=0,id=stg0,drive=fmt_stg0,bootindex=1 \
 -monitor stdio \
 -device virtio-net-pci,mac=9a:b5:b6:b1:b2:b7,id=nic1,netdev=nicpci,bus=pcie-root-port-7 \
 -netdev tap,id=nicpci \
 -boot menu=on,reboot-timeout=1000,strict=off \
 -chardev socket,id=socket-serial,path=/var/tmp/socket-serial,logfile=/var/tmp/file-serial.log,mux=on,server=on,wait=off \
 -serial chardev:socket-serial \
 -chardev file,path=/var/tmp/file-bios.log,id=file-bios \
 -device isa-debugcon,chardev=file-bios,iobase=0x402 \
 -chardev socket,id=socket-qmp,path=/var/tmp/socket-qmp,logfile=/var/tmp/file-qmp.log,mux=on,server=on,wait=off \
 -mon chardev=socket-qmp,mode=control \
 -chardev socket,id=socket-hmp,path=/var/tmp/socket-hmp,logfile=/var/tmp/file-hmp.log,mux=on,server=on,wait=off \
 -mon chardev=socket-hmp,mode=readline \
 -vnc :6 \
 -qmp tcp:0:5956,server=on,wait=off \
 -incoming defer


migration succeed