Description of problem: This problem is similar to bug 2093791, but this one is using vfio-pci instead of virtio devices. Version-Release number of selected component (if applicable): Host kernel: kernel-5.14.0-319.el9.aarch64 Guest kernel: kernel-5.14.0-319.el9.aarch64 (4k/64k) QEMU version: qemu-kvm-8.0.0-4.el9.aarch64 edk2 version: edk2-aarch64-20230301gitf80f052277c8-5.el9.noarch How reproducible: always Steps to Reproduce: 1. Setup SR-IOV device modprobe vfio modprobe vfio-pci echo 1 > /sys/bus/pci/devices/0000\:0b\:00.0/sriov_numvfs driverctl set-override 0000:0b:02.0 vfio-pci # driverctl list-devices network 0000:05:00.0 e1000e 0000:0b:00.0 qede 0000:0b:00.1 qede 0000:0b:02.0 vfio-pci [*] 2. Launch a guest with virtio-iommu and vfio-pci MALLOC_PERTURB_=1 /usr/libexec/qemu-kvm \ -name 'avocado-vt-vm1' \ -sandbox on \ -blockdev '{"node-name": "file_aavmf_code", "driver": "file", "filename": "/usr/share/edk2/aarch64/QEMU_EFI-silent-pflash.qcow2", "auto-read-only": true, "discard": "unmap"}' \ -blockdev '{"node-name": "drive_aavmf_code", "driver": "qcow2", "read-only": true, "file": "file_aavmf_code"}' \ -blockdev '{"node-name": "file_aavmf_vars", "driver": "file", "filename": "/root/avocado/data/avocado-vt/avocado-vt-vm1_rhel930-aarch64-64k-virtio-scsi_qcow2_filesystem_VARS.qcow2", "auto-read-only": true, "discard": "unmap"}' \ -blockdev '{"node-name": "drive_aavmf_vars", "driver": "qcow2", "read-only": false, "file": "file_aavmf_vars"}' \ -machine virt,gic-version=host,memory-backend=mem-machine_mem,pflash0=drive_aavmf_code,pflash1=drive_aavmf_vars,its=on,ras=on,acpi=on \ -device '{"id": "viommu", "driver": "virtio-iommu-pci", "bus": "pcie.0", "addr": "0x2"}' \ -device '{"id": "pcie-root-port-0", "driver": "pcie-root-port", "multifunction": true, "bus": "pcie.0", "addr": "0x1", "chassis": 1}' \ -device '{"id": "pcie-pci-bridge-0", "driver": "pcie-pci-bridge", "addr": "0x0", "bus": "pcie-root-port-0"}' \ -nodefaults \ -device '{"id": "pcie-root-port-1", "port": 1, "driver": "pcie-root-port", "addr": "0x1.0x1", "bus": "pcie.0", "chassis": 2}' \ -device '{"driver": "virtio-gpu-pci", "bus": "pcie-root-port-1", "addr": "0x0"}' \ -m 13312 \ -object '{"size": 13958643712, "id": "mem-machine_mem", "qom-type": "memory-backend-ram"}' \ -smp 112,maxcpus=112,cores=56,threads=1,clusters=1,sockets=2 \ -cpu 'host' \ -serial unix:'/var/tmp/serial-serial0',server=on,wait=off \ -device '{"id": "pcie-root-port-2", "port": 2, "driver": "pcie-root-port", "addr": "0x1.0x2", "bus": "pcie.0", "chassis": 3}' \ -device '{"driver": "qemu-xhci", "id": "usb1", "bus": "pcie-root-port-2", "addr": "0x0"}' \ -device '{"driver": "usb-tablet", "id": "usb-tablet1", "bus": "usb1.0", "port": "1"}' \ -device '{"id": "pcie-root-port-3", "port": 3, "driver": "pcie-root-port", "addr": "0x1.0x3", "bus": "pcie.0", "chassis": 4}' \ -device '{"id": "virtio_scsi_pci0", "driver": "virtio-scsi-pci", "bus": "pcie-root-port-3", "addr": "0x0"}' \ -blockdev '{"node-name": "file_image1", "driver": "file", "auto-read-only": true, "discard": "unmap", "aio": "threads", "filename": "/home/kvm_autotest_root/images/rhel930-aarch64-64k-virtio-scsi.qcow2", "cache": {"direct": true, "no-flush": false}}' \ -blockdev '{"node-name": "drive_image1", "driver": "qcow2", "read-only": false, "cache": {"direct": true, "no-flush": false}, "file": "file_image1"}' \ -device '{"driver": "scsi-hd", "id": "image1", "drive": "drive_image1", "write-cache": "on"}' \ -device '{"id": "pcie-root-port-4", "port": 4, "driver": "pcie-root-port", "addr": "0x1.0x4", "bus": "pcie.0", "chassis": 5}' \ -device '{"driver": "vfio-pci", "host": "0000:0b:02.0", "bus": "pcie-root-port-4", "addr": "0x0"}' \ -vnc :0 \ -rtc base=utc,clock=host \ -enable-kvm \ -qmp stdio 3. Check qemu process qemu-kvm: warning: virtio-iommu page mask 0xfffffffffffff000 does not match 0x40201000 Actual results: Although qemu reports a warning message, the guest was started, if reboot the guest, will repeat the message. Expected results: Launched without warning message Additional info:
This is expected because your guest seems to use 4kB page size while the host IOMMU driver uses 64kB
Eric, in comment 0, the host kernel is 4k page, and can reproduce this with 4k and 64k guests. And I believe x86 can also reproduce this problem. Anyway, I will summarise the result for ARM later.
Host: # uname -r 5.14.0-319.el9.aarch64 Guest: # uname -r 5.14.0-319.el9.aarch64 QEMU: # MALLOC_PERTURB_=1 /usr/libexec/qemu-kvm \ -name 'avocado-vt-vm1' \ -S \ -sandbox on \ -blockdev '{"node-name": "file_aavmf_code", "driver": "file", "filename": "/usr/share/edk2/aarch64/QEMU_EFI-silent-pflash.qcow2", "auto-read-only": true, "discard": "unmap"}' \ -blockdev '{"node-name": "drive_aavmf_code", "driver": "qcow2", "read-only": true, "file": "file_aavmf_code"}' \ -blockdev '{"node-name": "file_aavmf_vars", "driver": "file", "filename": "/root/avocado/data/avocado-vt/avocado-vt-vm1_rhel930-aarch64-4k-virtio-scsi_qcow2_filesystem_VARS.qcow2", "auto-read-only": true, "discard": "unmap"}' \ -blockdev '{"node-name": "drive_aavmf_vars", "driver": "qcow2", "read-only": false, "file": "file_aavmf_vars"}' \ -machine virt,gic-version=host,memory-backend=mem-machine_mem,pflash0=drive_aavmf_code,pflash1=drive_aavmf_vars \ -device '{"id": "viommu", "driver": "virtio-iommu-pci", "bus": "pcie.0", "addr": "0x2"}' \ -device '{"id": "pcie-root-port-0", "driver": "pcie-root-port", "multifunction": true, "bus": "pcie.0", "addr": "0x1", "chassis": 1}' \ -device '{"id": "pcie-pci-bridge-0", "driver": "pcie-pci-bridge", "addr": "0x0", "bus": "pcie-root-port-0"}' \ -nodefaults \ -device '{"id": "pcie-root-port-1", "port": 1, "driver": "pcie-root-port", "addr": "0x1.0x1", "bus": "pcie.0", "chassis": 2}' \ -device '{"driver": "virtio-gpu-pci", "bus": "pcie-root-port-1", "addr": "0x0"}' \ -m 8192 \ -object '{"size": 8589934592, "id": "mem-machine_mem", "qom-type": "memory-backend-ram"}' \ -smp 8,maxcpus=8,cores=4,threads=1,clusters=1,sockets=2 \ -cpu 'host' \ -serial unix:'/var/tmp/serial-serial0',server=on,wait=off \ -device '{"id": "pcie-root-port-2", "port": 2, "driver": "pcie-root-port", "addr": "0x1.0x2", "bus": "pcie.0", "chassis": 3}' \ -device '{"driver": "qemu-xhci", "id": "usb1", "bus": "pcie-root-port-2", "addr": "0x0"}' \ -device '{"driver": "usb-tablet", "id": "usb-tablet1", "bus": "usb1.0", "port": "1"}' \ -device '{"id": "pcie-root-port-3", "port": 3, "driver": "pcie-root-port", "addr": "0x1.0x3", "bus": "pcie.0", "chassis": 4}' \ -device '{"id": "virtio_scsi_pci0", "driver": "virtio-scsi-pci", "bus": "pcie-root-port-3", "addr": "0x0"}' \ -blockdev '{"node-name": "file_image1", "driver": "file", "auto-read-only": true, "discard": "unmap", "aio": "threads", "filename": "/home/kvm_autotest_root/images/rhel930-aarch64-4k-virtio-scsi.qcow2", "cache": {"direct": true, "no-flush": false}}' \ -blockdev '{"node-name": "drive_image1", "driver": "qcow2", "read-only": false, "cache": {"direct": true, "no-flush": false}, "file": "file_image1"}' \ -device '{"driver": "scsi-hd", "id": "image1", "drive": "drive_image1", "write-cache": "on"}' \ -device '{"id": "pcie-root-port-4", "port": 4, "driver": "pcie-root-port", "addr": "0x1.0x4", "bus": "pcie.0", "chassis": 5}' \ -device '{"driver": "vfio-pci", "host": "0000:0b:02.0", "bus": "pcie-root-port-4", "addr": "0x0"}' \ -vnc :0 \ -rtc base=utc,clock=host \ -monitor stdio QEMU 8.0.0 monitor - type 'help' for more information (qemu) c (qemu) qemu-kvm: warning: virtio-iommu page mask 0xfffffffffffff000 does not match 0x40201000 Host: # uname -r 5.14.0-319.el9.aarch64 Guest: # uname -r 5.14.0-319.el9.aarch64+64k QEMU: # MALLOC_PERTURB_=1 /usr/libexec/qemu-kvm \ -name 'avocado-vt-vm1' \ -S \ -sandbox on \ -blockdev '{"node-name": "file_aavmf_code", "driver": "file", "filename": "/usr/share/edk2/aarch64/QEMU_EFI-silent-pflash.qcow2", "auto-read-only": true, "discard": "unmap"}' \ -blockdev '{"node-name": "drive_aavmf_code", "driver": "qcow2", "read-only": true, "file": "file_aavmf_code"}' \ -blockdev '{"node-name": "file_aavmf_vars", "driver": "file", "filename": "/root/avocado/data/avocado-vt/avocado-vt-vm1_rhel930-aarch64-64k-virtio-scsi_qcow2_filesystem_VARS.qcow2", "auto-read-only": true, "discard": "unmap"}' \ -blockdev '{"node-name": "drive_aavmf_vars", "driver": "qcow2", "read-only": false, "file": "file_aavmf_vars"}' \ -machine virt,gic-version=host,memory-backend=mem-machine_mem,pflash0=drive_aavmf_code,pflash1=drive_aavmf_vars \ -device '{"id": "viommu", "driver": "virtio-iommu-pci", "bus": "pcie.0", "addr": "0x2"}' \ -device '{"id": "pcie-root-port-0", "driver": "pcie-root-port", "multifunction": true, "bus": "pcie.0", "addr": "0x1", "chassis": 1}' \ -device '{"id": "pcie-pci-bridge-0", "driver": "pcie-pci-bridge", "addr": "0x0", "bus": "pcie-root-port-0"}' \ -nodefaults \ -device '{"id": "pcie-root-port-1", "port": 1, "driver": "pcie-root-port", "addr": "0x1.0x1", "bus": "pcie.0", "chassis": 2}' \ -device '{"driver": "virtio-gpu-pci", "bus": "pcie-root-port-1", "addr": "0x0"}' \ -m 8192 \ -object '{"size": 8589934592, "id": "mem-machine_mem", "qom-type": "memory-backend-ram"}' \ -smp 8,maxcpus=8,cores=4,threads=1,clusters=1,sockets=2 \ -cpu 'host' \ -serial unix:'/var/tmp/serial-serial0',server=on,wait=off \ -device '{"id": "pcie-root-port-2", "port": 2, "driver": "pcie-root-port", "addr": "0x1.0x2", "bus": "pcie.0", "chassis": 3}' \ -device '{"driver": "qemu-xhci", "id": "usb1", "bus": "pcie-root-port-2", "addr": "0x0"}' \ -device '{"driver": "usb-tablet", "id": "usb-tablet1", "bus": "usb1.0", "port": "1"}' \ -device '{"id": "pcie-root-port-3", "port": 3, "driver": "pcie-root-port", "addr": "0x1.0x3", "bus": "pcie.0", "chassis": 4}' \ -device '{"id": "virtio_scsi_pci0", "driver": "virtio-scsi-pci", "bus": "pcie-root-port-3", "addr": "0x0"}' \ -blockdev '{"node-name": "file_image1", "driver": "file", "auto-read-only": true, "discard": "unmap", "aio": "threads", "filename": "/home/kvm_autotest_root/images/rhel930-aarch64-64k-virtio-scsi.qcow2", "cache": {"direct": true, "no-flush": false}}' \ -blockdev '{"node-name": "drive_image1", "driver": "qcow2", "read-only": false, "cache": {"direct": true, "no-flush": false}, "file": "file_image1"}' \ -device '{"driver": "scsi-hd", "id": "image1", "drive": "drive_image1", "write-cache": "on"}' \ -device '{"id": "pcie-root-port-4", "port": 4, "driver": "pcie-root-port", "addr": "0x1.0x4", "bus": "pcie.0", "chassis": 5}' \ -device '{"driver": "vfio-pci", "host": "0000:0b:02.0", "bus": "pcie-root-port-4", "addr": "0x0"}' \ -vnc :0 \ -rtc base=utc,clock=host \ -monitor stdio QEMU 8.0.0 monitor - type 'help' for more information (qemu) c (qemu) qemu-kvm: warning: virtio-iommu page mask 0xfffffffffffff000 does not match 0x40201000
Test with x86 and can reproduce this problem: host & guest:5.14.0-319.el9.x86_64 /usr/libexec/qemu-kvm \ -name guest=v,debug-threads=on \ -S \ -object '{"qom-type":"secret","id":"masterKey0","format":"raw","file":"/var/lib/libvirt/qemu/domain-1-v/master-key.aes"}' \ -blockdev '{"driver":"file","filename":"/usr/share/edk2/ovmf/OVMF_CODE.secboot.fd","node-name":"libvirt-pflash0-storage","auto-read-only":true,"discard":"unmap"}' \ -blockdev '{"node-name":"libvirt-pflash0-format","read-only":true,"driver":"raw","file":"libvirt-pflash0-storage"}' \ -blockdev '{"driver":"file","filename":"/var/lib/libvirt/qemu/nvram/v_VARS.fd","node-name":"libvirt-pflash1-storage","auto-read-only":true,"discard":"unmap"}' \ -blockdev '{"node-name":"libvirt-pflash1-format","read-only":false,"driver":"raw","file":"libvirt-pflash1-storage"}' \ -machine pc-q35-rhel9.2.0,usb=off,smm=on,dump-guest-core=off,memory-backend=pc.ram,pflash0=libvirt-pflash0-format,pflash1=libvirt-pflash1-format,hpet=off,acpi=on \ -accel kvm \ -cpu host,migratable=on \ -global driver=cfi.pflash01,property=secure,value=on \ -m 8192 \ -object '{"qom-type":"memory-backend-ram","id":"pc.ram","size":8589934592}' \ -overcommit mem-lock=off \ -smp 6,sockets=1,dies=1,cores=6,threads=1 \ -uuid 59b43985-fc72-41c8-8979-492f774408ff \ -no-user-config \ -nodefaults \ -chardev socket,id=charmonitor,fd=33,server=on,wait=off \ -mon chardev=charmonitor,id=monitor,mode=control \ -rtc base=utc,driftfix=slew \ -global kvm-pit.lost_tick_policy=delay \ -no-shutdown \ -global ICH9-LPC.disable_s3=1 \ -global ICH9-LPC.disable_s4=1 \ -boot strict=on \ -device '{"driver":"virtio-iommu","id":"iommu0","bus":"pcie.0","addr":"0x4"}' \ -device '{"driver":"pcie-root-port","port":16,"chassis":1,"id":"pci.1","bus":"pcie.0","multifunction":true,"addr":"0x2"}' \ -device '{"driver":"pcie-root-port","port":17,"chassis":2,"id":"pci.2","bus":"pcie.0","addr":"0x2.0x1"}' \ -device '{"driver":"pcie-root-port","port":18,"chassis":3,"id":"pci.3","bus":"pcie.0","addr":"0x2.0x2"}' \ -device '{"driver":"pcie-root-port","port":19,"chassis":4,"id":"pci.4","bus":"pcie.0","addr":"0x2.0x3"}' \ -device '{"driver":"pcie-root-port","port":20,"chassis":5,"id":"pci.5","bus":"pcie.0","addr":"0x2.0x4"}' \ -device '{"driver":"pcie-root-port","port":21,"chassis":6,"id":"pci.6","bus":"pcie.0","addr":"0x2.0x5"}' \ -device '{"driver":"pcie-root-port","port":22,"chassis":7,"id":"pci.7","bus":"pcie.0","addr":"0x2.0x6"}' \ -device '{"driver":"pcie-root-port","port":23,"chassis":8,"id":"pci.8","bus":"pcie.0","addr":"0x2.0x7"}' \ -device '{"driver":"pcie-root-port","port":24,"chassis":9,"id":"pci.9","bus":"pcie.0","multifunction":true,"addr":"0x3"}' \ -device '{"driver":"pcie-root-port","port":25,"chassis":10,"id":"pci.10","bus":"pcie.0","addr":"0x3.0x1"}' \ -device '{"driver":"pcie-root-port","port":26,"chassis":11,"id":"pci.11","bus":"pcie.0","addr":"0x3.0x2"}' \ -device '{"driver":"pcie-root-port","port":27,"chassis":12,"id":"pci.12","bus":"pcie.0","addr":"0x3.0x3"}' \ -device '{"driver":"pcie-root-port","port":28,"chassis":13,"id":"pci.13","bus":"pcie.0","addr":"0x3.0x4"}' \ -device '{"driver":"pcie-root-port","port":29,"chassis":14,"id":"pci.14","bus":"pcie.0","addr":"0x3.0x5"}' \ -device '{"driver":"qemu-xhci","p2":15,"p3":15,"id":"usb","bus":"pci.3","addr":"0x0"}' \ -device '{"driver":"virtio-scsi-pci","iommu_platform":true,"ats":true,"id":"scsi0","bus":"pci.2","addr":"0x0"}' \ -device '{"driver":"virtio-serial-pci","id":"virtio-serial0","bus":"pci.4","addr":"0x0"}' \ -blockdev '{"driver":"file","filename":"/home/RHEL-9.3.0-20230604.26-x86_64-ovmf.qcow2","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}' \ -blockdev '{"node-name":"libvirt-1-format","read-only":false,"driver":"qcow2","file":"libvirt-1-storage","backing":null}' \ -device '{"driver":"scsi-hd","bus":"scsi0.0","channel":0,"scsi-id":0,"lun":0,"device_id":"drive-scsi0-0-0-0","drive":"libvirt-1-format","id":"scsi0-0-0-0","bootindex":1}' \ -netdev '{"type":"tap","fd":"34","vhost":true,"vhostfd":"36","id":"hostnet0"}' \ -device '{"driver":"virtio-net-pci","iommu_platform":true,"ats":true,"netdev":"hostnet0","id":"net0","mac":"52:54:00:f6:db:6f","bus":"pci.1","addr":"0x0"}' \ -chardev pty,id=charserial0 \ -device '{"driver":"isa-serial","chardev":"charserial0","id":"serial0","index":0}' \ -chardev socket,id=charchannel0,fd=32,server=on,wait=off \ -device '{"driver":"virtserialport","bus":"virtio-serial0.0","nr":1,"chardev":"charchannel0","id":"channel0","name":"org.qemu.guest_agent.0"}' \ -chardev socket,id=chrtpm,path=/run/libvirt/qemu/swtpm/1-v-swtpm.sock \ -tpmdev emulator,id=tpm-tpm0,chardev=chrtpm \ -device '{"driver":"tpm-crb","tpmdev":"tpm-tpm0","id":"tpm0"}' \ -device '{"driver":"usb-tablet","id":"input0","bus":"usb.0","port":"1"}' \ -audiodev '{"id":"audio1","driver":"none"}' \ -vnc 0.0.0.0:0,audiodev=audio1 \ -device '{"driver":"virtio-vga","id":"video0","max_outputs":1,"bus":"pcie.0","addr":"0x1"}' \ -global ICH9-LPC.noreboot=off \ -watchdog-action reset \ -device '{"driver":"vfio-pci","host":"0000:01:10.0","id":"hostdev0","bus":"pci.7","addr":"0x0"}' \ -device '{"driver":"virtio-balloon-pci","id":"balloon0","bus":"pci.5","addr":"0x0"}' \ -object '{"qom-type":"rng-random","id":"objrng0","filename":"/dev/urandom"}' \ -device '{"driver":"virtio-rng-pci","rng":"objrng0","id":"rng0","bus":"pci.6","addr":"0x0"}' \ -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \ -msg timestamp=on qemu-kvm: warning: virtio-iommu page mask 0xfffffffffffff000 does not match 0x40201000
The above warning is a benign warning supposed to warn the end-user that the page mask exposed by the VFIO device is not applied (supposed to lead to sub optimal perf) but does not prevent the device from working. Since I don't really understand the purpose of this warning I sent upstream: [PATCH 2/2] virtio-iommu: Rework the trace in virtio_iommu_set_page_size_mask() See https://lore.kernel.org/all/20230704111527.3424992-3-eric.auger@redhat.com/ Eric
QE bot(pre verify): Set 'Verified:Tested,SanityOnly' as gating/tier1 test pass.
Test with host kernel: 5.14.0-339.kpq1.el9.x86_64 qemu-kvm: qemu-kvm-8.0.0-8.el9 boot guest with vfio-pci and virtio-iommu, there is no warning message guest: -name guest=rhel93,debug-threads=on \ -S \ -object '{"qom-type":"secret","id":"masterKey0","format":"raw","file":"/var/lib/libvirt/qemu/domain-2-rhel93/master-key.aes"}' \ -blockdev '{"driver":"file","filename":"/usr/share/edk2/ovmf/OVMF_CODE.secboot.fd","node-name":"libvirt-pflash0-storage","auto-read-only":true,"discard":"unmap"}' \ -blockdev '{"node-name":"libvirt-pflash0-format","read-only":true,"driver":"raw","file":"libvirt-pflash0-storage"}' \ -blockdev '{"driver":"file","filename":"/var/lib/libvirt/qemu/nvram/rhel93_VARS.fd","node-name":"libvirt-pflash1-storage","auto-read-only":true,"discard":"unmap"}' \ -blockdev '{"node-name":"libvirt-pflash1-format","read-only":false,"driver":"raw","file":"libvirt-pflash1-storage"}' \ -machine pc-q35-rhel9.2.0,usb=off,smm=on,dump-guest-core=off,memory-backend=pc.ram,pflash0=libvirt-pflash0-format,pflash1=libvirt-pflash1-format,hpet=off,acpi=on \ -accel kvm \ -cpu host,migratable=on \ -global driver=cfi.pflash01,property=secure,value=on \ -m size=4194304k \ -object '{"qom-type":"memory-backend-ram","id":"pc.ram","size":4294967296}' \ -overcommit mem-lock=off \ -smp 4,sockets=4,cores=1,threads=1 \ -uuid a661a929-447c-4665-9ff2-9accb0e5ab8e \ -no-user-config \ -nodefaults \ -chardev socket,id=charmonitor,fd=32,server=on,wait=off \ -mon chardev=charmonitor,id=monitor,mode=control \ -rtc base=utc,driftfix=slew \ -global kvm-pit.lost_tick_policy=delay \ -no-shutdown \ -global ICH9-LPC.disable_s3=1 \ -global ICH9-LPC.disable_s4=1 \ -boot strict=on \ -device '{"driver":"virtio-iommu","id":"iommu0","bus":"pcie.0","addr":"0x4"}' \ -device '{"driver":"pcie-root-port","port":16,"chassis":1,"id":"pci.1","bus":"pcie.0","multifunction":true,"addr":"0x2"}' \ -device '{"driver":"pcie-root-port","port":17,"chassis":2,"id":"pci.2","bus":"pcie.0","addr":"0x2.0x1"}' \ -device '{"driver":"pcie-root-port","port":18,"chassis":3,"id":"pci.3","bus":"pcie.0","addr":"0x2.0x2"}' \ -device '{"driver":"pcie-root-port","port":19,"chassis":4,"id":"pci.4","bus":"pcie.0","addr":"0x2.0x3"}' \ -device '{"driver":"pcie-root-port","port":20,"chassis":5,"id":"pci.5","bus":"pcie.0","addr":"0x2.0x4"}' \ -device '{"driver":"pcie-root-port","port":21,"chassis":6,"id":"pci.6","bus":"pcie.0","addr":"0x2.0x5"}' \ -device '{"driver":"pcie-root-port","port":22,"chassis":7,"id":"pci.7","bus":"pcie.0","addr":"0x2.0x6"}' \ -device '{"driver":"pcie-root-port","port":23,"chassis":8,"id":"pci.8","bus":"pcie.0","addr":"0x2.0x7"}' \ -device '{"driver":"pcie-root-port","port":24,"chassis":9,"id":"pci.9","bus":"pcie.0","multifunction":true,"addr":"0x3"}' \ -device '{"driver":"pcie-root-port","port":25,"chassis":10,"id":"pci.10","bus":"pcie.0","addr":"0x3.0x1"}' \ -device '{"driver":"pcie-root-port","port":26,"chassis":11,"id":"pci.11","bus":"pcie.0","addr":"0x3.0x2"}' \ -device '{"driver":"pcie-root-port","port":27,"chassis":12,"id":"pci.12","bus":"pcie.0","addr":"0x3.0x3"}' \ -device '{"driver":"pcie-root-port","port":28,"chassis":13,"id":"pci.13","bus":"pcie.0","addr":"0x3.0x4"}' \ -device '{"driver":"pcie-root-port","port":29,"chassis":14,"id":"pci.14","bus":"pcie.0","addr":"0x3.0x5"}' \ -device '{"driver":"ich9-usb-ehci1","id":"usb","bus":"pcie.0","addr":"0x1d.0x7"}' \ -device '{"driver":"ich9-usb-uhci1","masterbus":"usb.0","firstport":0,"bus":"pcie.0","multifunction":true,"addr":"0x1d"}' \ -device '{"driver":"ich9-usb-uhci2","masterbus":"usb.0","firstport":2,"bus":"pcie.0","addr":"0x1d.0x1"}' \ -device '{"driver":"ich9-usb-uhci3","masterbus":"usb.0","firstport":4,"bus":"pcie.0","addr":"0x1d.0x2"}' \ -blockdev '{"driver":"file","filename":"/home/images/RHEL93.qcow2","aio":"threads","node-name":"libvirt-1-storage","cache":{"direct":true,"no-flush":false},"auto-read-only":true,"discard":"unmap"}' \ -blockdev '{"node-name":"libvirt-1-format","read-only":false,"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":"libvirt-1-storage","backing":null}' \ -device '{"driver":"virtio-blk-pci","bus":"pci.2","addr":"0x0","drive":"libvirt-1-format","id":"virtio-disk0","bootindex":1,"write-cache":"on"}' \ -netdev '{"type":"tap","fd":"33","vhost":true,"vhostfd":"35","id":"hostnet0"}' \ -device '{"driver":"virtio-net-pci","iommu_platform":true,"netdev":"hostnet0","id":"net0","mac":"52:54:00:00:93:93","bus":"pci.1","addr":"0x0"}' \ -chardev pty,id=charserial0 \ -device '{"driver":"isa-serial","chardev":"charserial0","id":"serial0","index":0}' \ -chardev socket,id=chrtpm,path=/run/libvirt/qemu/swtpm/2-rhel93-swtpm.sock \ -tpmdev emulator,id=tpm-tpm0,chardev=chrtpm \ -device '{"driver":"tpm-crb","tpmdev":"tpm-tpm0","id":"tpm0"}' \ -device '{"driver":"usb-tablet","id":"input0","bus":"usb.0","port":"1"}' \ -audiodev '{"id":"audio1","driver":"none"}' \ -vnc 0.0.0.0:93,audiodev=audio1 \ -device '{"driver":"bochs-display","id":"video0","vgamem":16777216,"bus":"pcie.0","addr":"0x1"}' \ -global ICH9-LPC.noreboot=off \ -watchdog-action reset \ -device '{"driver":"vfio-pci","host":"0000:01:10.1","id":"hostdev0","bus":"pci.4","addr":"0x0"}' \ -device '{"driver":"virtio-balloon-pci","id":"balloon0","bus":"pci.3","addr":"0x0"}' \ -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \ -msg timestamp=on
based on Comment 14, set BZ to verified.