Bug 2158430

Gerd - could you take a look here. Not sure if the component is right, but I'll it there for now. 

If I'm reading right, this is trying to boot as a Rome (quite a bit older)...

(In reply to John Ferlan from comment #1)
> Gerd - could you take a look here. Not sure if the component is right, but
> I'll it there for now. 
> 
> If I'm reading right, this is trying to boot as a Rome (quite a bit older)...

The issue can also be reproduced on Milan, test steps and test result is the same. 

Version:
qemu-kvm-7.2.0-2.el9, 
kernel: 5.14.0-229.el9.x86_64
edk2-ovmf-20221207gitfff6d81270b5-2.el9.noarch

Steps:
/usr/libexec/qemu-kvm \
-name guest=rhel86_sev,debug-threads=on \
-S \
-object '{"qom-type":"secret","id":"masterKey0","format":"raw","file":"/var/lib/libvirt/qemu/domain-30-rhel86_sev/master-key.aes"}' \
-blockdev '{"driver":"file","filename":"/usr/share/edk2/ovmf/OVMF.amdsev.fd","node-name":"libvirt-pflash0-storage","auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-pflash0-format","read-only":true,"driver":"raw","file":"libvirt-pflash0-storage"}' \
-machine pc-q35-rhel9.2.0,usb=off,dump-guest-core=off,memory-backend=pc.ram,confidential-guest-support=lsec0,pflash0=libvirt-pflash0-format \
-accel kvm \
-cpu EPYC-Milan,x2apic=on,tsc-deadline=on,hypervisor=on,tsc-adjust=on,vaes=on,vpclmulqdq=on,spec-ctrl=on,stibp=on,arch-capabilities=on,ssbd=on,cmp-legacy=on,virt-ssbd=on,lbrv=off,tsc-scale=off,vmcb-clean=off,pause-filter=off,pfthreshold=off,v-vmsave-vmload=off,vgif=off,rdctl-no=on,skip-l1dfl-vmentry=on,mds-no=on,pschange-mc-no=on,svm=off,topoext=on,npt=off,nrip-save=off,svme-addr-chk=off \
-m 4096 \
-object '{"qom-type":"memory-backend-ram","id":"pc.ram","size":4294967296}' \
-overcommit mem-lock=off \
-smp 4,sockets=1,dies=1,cores=4,threads=1 \
-uuid 02e57e1e-a2d5-49a1-be7c-278772063dc4 \
-no-user-config \
-nodefaults \
-chardev socket,id=charmonitor,fd=32,server=on,wait=off \
-mon chardev=charmonitor,id=monitor,mode=control \
-rtc base=utc,driftfix=slew \
-global kvm-pit.lost_tick_policy=delay \
-no-hpet \
-no-shutdown \
-global ICH9-LPC.disable_s3=1 \
-global ICH9-LPC.disable_s4=1 \
-boot strict=on \
-kernel /home/vmlinuz \
-initrd /home/initrd.img \
-append 'ksdevice=eth0 inst.ks=http://beaker.engineering.redhat.com/kickstart/11188299 serial console=tty0 console=ttyS0,115200 net.ifnames=0 biosdevname=0 inst.repo=*' \
-device '{"driver":"pcie-root-port","port":16,"chassis":1,"id":"pci.1","bus":"pcie.0","multifunction":true,"addr":"0x2"}' \
-device '{"driver":"pcie-root-port","port":17,"chassis":2,"id":"pci.2","bus":"pcie.0","addr":"0x2.0x1"}' \
-device '{"driver":"pcie-root-port","port":18,"chassis":3,"id":"pci.3","bus":"pcie.0","addr":"0x2.0x2"}' \
-device '{"driver":"pcie-root-port","port":19,"chassis":4,"id":"pci.4","bus":"pcie.0","addr":"0x2.0x3"}' \
-device '{"driver":"pcie-root-port","port":20,"chassis":5,"id":"pci.5","bus":"pcie.0","addr":"0x2.0x4"}' \
-device '{"driver":"pcie-root-port","port":21,"chassis":6,"id":"pci.6","bus":"pcie.0","addr":"0x2.0x5"}' \
-device '{"driver":"pcie-root-port","port":22,"chassis":7,"id":"pci.7","bus":"pcie.0","addr":"0x2.0x6"}' \
-device '{"driver":"pcie-root-port","port":23,"chassis":8,"id":"pci.8","bus":"pcie.0","addr":"0x2.0x7"}' \
-device '{"driver":"pcie-root-port","port":24,"chassis":9,"id":"pci.9","bus":"pcie.0","multifunction":true,"addr":"0x3"}' \
-device '{"driver":"pcie-root-port","port":25,"chassis":10,"id":"pci.10","bus":"pcie.0","addr":"0x3.0x1"}' \
-device '{"driver":"pcie-root-port","port":26,"chassis":11,"id":"pci.11","bus":"pcie.0","addr":"0x3.0x2"}' \
-device '{"driver":"pcie-root-port","port":27,"chassis":12,"id":"pci.12","bus":"pcie.0","addr":"0x3.0x3"}' \
-device '{"driver":"pcie-root-port","port":28,"chassis":13,"id":"pci.13","bus":"pcie.0","addr":"0x3.0x4"}' \
-device '{"driver":"pcie-root-port","port":29,"chassis":14,"id":"pci.14","bus":"pcie.0","addr":"0x3.0x5"}' \
-device '{"driver":"qemu-xhci","p2":15,"p3":15,"id":"usb","bus":"pci.4","addr":"0x0"}' \
-device '{"driver":"virtio-scsi-pci","id":"scsi0","bus":"pci.2","addr":"0x0"}' \
-device '{"driver":"virtio-serial-pci","id":"virtio-serial0","bus":"pci.3","addr":"0x0"}' \
-blockdev '{"driver":"file","filename":"/home/rhel86_sev.qcow2","node-name":"libvirt-1-storage","cache":{"direct":true,"no-flush":false},"auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-1-format","read-only":false,"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":"libvirt-1-storage","backing":null}' \
-device '{"driver":"scsi-hd","bus":"scsi0.0","channel":0,"scsi-id":0,"lun":0,"device_id":"drive-scsi0-0-0-0","drive":"libvirt-1-format","id":"scsi0-0-0-0","bootindex":1,"write-cache":"on"}' \
-netdev '{"type":"tap","fd":"55","vhost":true,"vhostfd":"56","id":"hostnet0"}' \
-device '{"driver":"virtio-net-pci","netdev":"hostnet0","id":"net0","mac":"52:56:00:00:00:02","bus":"pci.1","addr":"0x0","romfile":""}' \
-add-fd set=0,fd=30,opaque=serial0-source \
-chardev file,id=charserial0,path=/dev/fdset/0,append=on \
-device '{"driver":"isa-serial","chardev":"charserial0","id":"serial0","index":0}' \
-chardev pty,id=charserial1 \
-device '{"driver":"isa-serial","chardev":"charserial1","id":"serial1","index":1}' \
-chardev socket,id=charchannel0,fd=31,server=on,wait=off \
-device '{"driver":"virtserialport","bus":"virtio-serial0.0","nr":1,"chardev":"charchannel0","id":"channel0","name":"org.qemu.guest_agent.0"}' \
-chardev socket,id=chrtpm,path=/run/libvirt/qemu/swtpm/30-rhel86_sev-swtpm.sock \
-tpmdev emulator,id=tpm-tpm0,chardev=chrtpm \
-device '{"driver":"tpm-crb","tpmdev":"tpm-tpm0","id":"tpm0"}' \
-device '{"driver":"usb-tablet","id":"input0","bus":"usb.0","port":"1"}' \
-audiodev '{"id":"audio1","driver":"none"}' \
-vnc 0.0.0.0:0,audiodev=audio1 \
-device '{"driver":"virtio-vga","id":"video0","max_outputs":1,"bus":"pcie.0","addr":"0x1"}' \
-device '{"driver":"virtio-balloon-pci","id":"balloon0","bus":"pci.5","addr":"0x0"}' \
-object '{"qom-type":"rng-random","id":"objrng0","filename":"/dev/random"}' \
-device '{"driver":"virtio-rng-pci","rng":"objrng0","id":"rng0","bus":"pci.6","addr":"0x0"}' \
-object '{"qom-type":"sev-guest","id":"lsec0","cbitpos":51,"reduced-phys-bits":1,"policy":7,"kernel-hashes":true}' \
-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \
-msg timestamp=on


Result:
BdsDxe: failed to load Boot0003 "Grub Bootloader" from Fv(7CB8BDC9-F8EB-4F34-AAEA-3EE4AF6516A1)/FvFile(B5AE312C-BC8A-43B1-9C62-EBB826DD5D07): Not Found

(In reply to zixchen from comment #0)
> Description of problem:
> Measured AMD SEV-ES guest boot with kernel/initrd failed with failed to load
> Boot0003 "Grub Bootloader" from Fv/Fv file
> This is a regression issue which start from qemu7.2
> 
> Version-Release number of selected component (if applicable):
> qemu-kvm-7.2.0-2.el9
> edk2-ovmf-20221207gitfff6d81270b5-1.el9
> kernel-5.14.0-226.el9.x86_64

> Additional info:
> Only observed this from qemu7.2, it should be a qemu issue, as only degrade
> qemu-kvm to qemu-kvm-7.1.0-7.el9 the issue is gone.

My first guess would be the qemu RNG seed patches breaking kernel measurements.

grub load failing is expected, there is only a dummy grub.efi included.
But usually the firmware never reaches that point, it tries direct kernel
load first, this must have failed to land at that point in the boot flow.

(In reply to Gerd Hoffmann from comment #3)
> (In reply to zixchen from comment #0)
> > Description of problem:
> > Measured AMD SEV-ES guest boot with kernel/initrd failed with failed to load
> > Boot0003 "Grub Bootloader" from Fv/Fv file
> > This is a regression issue which start from qemu7.2
> > 
> > Version-Release number of selected component (if applicable):
> > qemu-kvm-7.2.0-2.el9
> > edk2-ovmf-20221207gitfff6d81270b5-1.el9
> > kernel-5.14.0-226.el9.x86_64
> 
> > Additional info:
> > Only observed this from qemu7.2, it should be a qemu issue, as only degrade
> > qemu-kvm to qemu-kvm-7.1.0-7.el9 the issue is gone.
> 
> My first guess would be the qemu RNG seed patches breaking kernel
> measurements.

This can be tested by replacing -machine pc-q35-rhel9.2.0  with -machine pc-q35-rhel9.1.0, which should disable the RNG seed injection logic.

When the RNG seed stuff was proposed originally there was indeed a regression with SEV measurements, but that was addressed in a later posting. I guess there's still a chance it regressed back to broken again though, since there were soo many revisions to this logic

(In reply to Daniel Berrangé from comment #4)
> (In reply to Gerd Hoffmann from comment #3)
> > (In reply to zixchen from comment #0)
> > > Description of problem:
> > > Measured AMD SEV-ES guest boot with kernel/initrd failed with failed to load
> > > Boot0003 "Grub Bootloader" from Fv/Fv file
> > > This is a regression issue which start from qemu7.2
> > > 
> > > Version-Release number of selected component (if applicable):
> > > qemu-kvm-7.2.0-2.el9
> > > edk2-ovmf-20221207gitfff6d81270b5-1.el9
> > > kernel-5.14.0-226.el9.x86_64
> > 
> > > Additional info:
> > > Only observed this from qemu7.2, it should be a qemu issue, as only degrade
> > > qemu-kvm to qemu-kvm-7.1.0-7.el9 the issue is gone.
> > 
> > My first guess would be the qemu RNG seed patches breaking kernel
> > measurements.
> 
> This can be tested by replacing -machine pc-q35-rhel9.2.0  with -machine
> pc-q35-rhel9.1.0, which should disable the RNG seed injection logic.

Comment2 steps on Milan use -machine pc-q35-rhel9.2.0,usb=off,dump-guest-core=off,memory-backend=pc.ram,confidential-guest-support=lsec0,pflash0=libvirt-pflash0-format \, it can reproduce the bug.

> 
> When the RNG seed stuff was proposed originally there was indeed a
> regression with SEV measurements, but that was addressed in a later posting.
> I guess there's still a chance it regressed back to broken again though,
> since there were soo many revisions to this logic

(In reply to zixchen from comment #5)
> (In reply to Daniel Berrangé from comment #4)
> > (In reply to Gerd Hoffmann from comment #3)
> > > (In reply to zixchen from comment #0)
> > > > Description of problem:
> > > > Measured AMD SEV-ES guest boot with kernel/initrd failed with failed to load
> > > > Boot0003 "Grub Bootloader" from Fv/Fv file
> > > > This is a regression issue which start from qemu7.2
> > > > 
> > > > Version-Release number of selected component (if applicable):
> > > > qemu-kvm-7.2.0-2.el9
> > > > edk2-ovmf-20221207gitfff6d81270b5-1.el9
> > > > kernel-5.14.0-226.el9.x86_64
> > > 
> > > > Additional info:
> > > > Only observed this from qemu7.2, it should be a qemu issue, as only degrade
> > > > qemu-kvm to qemu-kvm-7.1.0-7.el9 the issue is gone.
> > > 
> > > My first guess would be the qemu RNG seed patches breaking kernel
> > > measurements.
> > 
> > This can be tested by replacing -machine pc-q35-rhel9.2.0  with -machine
> > pc-q35-rhel9.1.0, which should disable the RNG seed injection logic.
> 
> Comment2 steps on Milan use -machine
> pc-q35-rhel9.2.0,usb=off,dump-guest-core=off,memory-backend=pc.ram,
> confidential-guest-support=lsec0,pflash0=libvirt-pflash0-format \, it can
> reproduce the bug.

pc-q35-rhel9.2.0 has the RNG seed feature. Please test with the previous version, aka pc-q35-rhel9.1.0, which should NOT have the RNG seed feature.

(In reply to Daniel Berrangé from comment #6)
> (In reply to zixchen from comment #5)
> > (In reply to Daniel Berrangé from comment #4)
> > > (In reply to Gerd Hoffmann from comment #3)
> > > > (In reply to zixchen from comment #0)
> > > > > Description of problem:
> > > > > Measured AMD SEV-ES guest boot with kernel/initrd failed with failed to load
> > > > > Boot0003 "Grub Bootloader" from Fv/Fv file
> > > > > This is a regression issue which start from qemu7.2
> > > > > 
> > > > > Version-Release number of selected component (if applicable):
> > > > > qemu-kvm-7.2.0-2.el9
> > > > > edk2-ovmf-20221207gitfff6d81270b5-1.el9
> > > > > kernel-5.14.0-226.el9.x86_64
> > > > 
> > > > > Additional info:
> > > > > Only observed this from qemu7.2, it should be a qemu issue, as only degrade
> > > > > qemu-kvm to qemu-kvm-7.1.0-7.el9 the issue is gone.
> > > > 
> > > > My first guess would be the qemu RNG seed patches breaking kernel
> > > > measurements.
> > > 
> > > This can be tested by replacing -machine pc-q35-rhel9.2.0  with -machine
> > > pc-q35-rhel9.1.0, which should disable the RNG seed injection logic.
> > 
> > Comment2 steps on Milan use -machine
> > pc-q35-rhel9.2.0,usb=off,dump-guest-core=off,memory-backend=pc.ram,
> > confidential-guest-support=lsec0,pflash0=libvirt-pflash0-format \, it can
> > reproduce the bug.
> 
> pc-q35-rhel9.2.0 has the RNG seed feature. Please test with the previous
> version, aka pc-q35-rhel9.1.0, which should NOT have the RNG seed feature.

Thanks Daniel! Replace pc-q35-rhel9.2.0 to pc-q35-rhel9.0.0 because pc-q35-rhel9.1.0 is not a supported machine type. It can boot, sev-es measure boot now works.

Latest upstream proposal (not yet merged) is to revert all the RNG seed patches entirely to purge all the brokeness:

  https://lists.gnu.org/archive/html/qemu-devel/2023-02/msg02333.html

(In reply to Daniel Berrangé from comment #8)
> Latest upstream proposal (not yet merged) is to revert all the RNG seed
> patches entirely to purge all the brokeness:
> 
>   https://lists.gnu.org/archive/html/qemu-devel/2023-02/msg02333.html

Hi Gerd,

This was merged upstream 9 days ago. Any idea when we'll get this fix
downstream ?

(In reply to Greg Kurz from comment #12)
> (In reply to Daniel Berrangé from comment #8)
> > Latest upstream proposal (not yet merged) is to revert all the RNG seed
> > patches entirely to purge all the brokeness:
> > 
> >   https://lists.gnu.org/archive/html/qemu-devel/2023-02/msg02333.html
> 
> Hi Gerd,
> 
> This was merged upstream 9 days ago. Any idea when we'll get this fix
> downstream ?

Dunno, doing firmware not qemu these days.

With the 8.0 qemu release being just around the corner and 9.2 being
in exception + blocker already I guess the process will be to first
have c9s + 9.3 pick up the changes by rebase and then, if needed,
backport into 9.2.z

The way I read things commit 167f4873580d3729565044cda73c3e20997950f2 (last in series) made it prior to upstream v8.0.0-rc0. Adjusting the various fields to ensure inclusion into RHEL 9.3 and assigning to Michael since he authored the commit.

If a z-stream is required, then set the ZTR=9.2.0 with zstream? and provide justification.

QE bot(pre verify): Set 'Verified:Tested,SanityOnly' as gating/tier1 test pass.

Test with qemu-kvm-8.0.0-1.el9.x86_64 on Milan, measure boot with sev-es enabled, no issue found. Wait for bug status changes to ON_QA.

Version:
qemu-kvm-8.0.0-1.el9.x86_64
kernel-5.14.0-293.el9.x86_64
edk2-ovmf-20230301gitf80f052277c8-2.el9.noarch

Steps:
/usr/libexec/qemu-kvm \
     -S  \
     -name 'avocado-vt-vm1'  \
     -sandbox on  \
     -object sev-guest,id=lsec0,policy=7,cbitpos=51,reduced-phys-bits=1,kernel-hashes=on \
     -blockdev node-name=file_ovmf_code,driver=file,filename=/usr/share/edk2/ovmf/OVMF.amdsev.fd,auto-read-only=on,discard=unmap \
     -blockdev node-name=drive_ovmf_code,driver=raw,read-only=on,file=file_ovmf_code \
     -machine q35,memory-backend=mem-machine_mem,confidential-guest-support=lsec0,pflash0=drive_ovmf_code \
     -device pcie-root-port,id=pcie-root-port-0,multifunction=on,bus=pcie.0,addr=0x1,chassis=1 \
     -device pcie-pci-bridge,id=pcie-pci-bridge-0,addr=0x0,bus=pcie-root-port-0  \
     -nodefaults \
     -device VGA,bus=pcie.0,addr=0x2 \
     -m 30720 \
     -object '{"qom-type": "memory-backend-ram", "size": 32212254720, "id": "mem-machine_mem"}'  \
     -smp 16,maxcpus=16,cores=8,threads=1,dies=1,sockets=2  \
     -cpu host,+kvm_pv_unhalt \
     -chardev socket,id=qmp_id_qmpmonitor1,wait=off,path=/var/tmp/avocado_2ksa4d11/monitor-qmpmonitor1-20230104-012730-gZ5d4sTa,server=on  \
     -mon chardev=qmp_id_qmpmonitor1,mode=control \
     -chardev socket,id=qmp_id_catch_monitor,wait=off,path=/var/tmp/avocado_2ksa4d11/monitor-catch_monitor-20230104-012730-gZ5d4sTa,server=on  \
     -mon chardev=qmp_id_catch_monitor,mode=control \
     -device pvpanic,ioport=0x505,id=idtwqVkd \
     -chardev socket,id=chardev_serial0,wait=off,path=/var/tmp/avocado_2ksa4d11/serial-serial0-20230104-012730-gZ5d4sTa,server=on \
     -device isa-serial,id=serial0,chardev=chardev_serial0 \
     -device pcie-root-port,id=pcie-root-port-1,port=0x1,addr=0x1.0x1,bus=pcie.0,chassis=2 \
     -device qemu-xhci,id=usb1,bus=pcie-root-port-1,addr=0x0 \
     -device usb-tablet,id=usb-tablet1,bus=usb1.0,port=1 \
     -device pcie-root-port,id=pcie-root-port-2,port=0x2,addr=0x1.0x2,bus=pcie.0,chassis=3 \
     -device '{"id": "virtio_scsi_pci0", "driver": "virtio-scsi-pci", "bus": "pcie-root-port-2", "addr": "0x0"}' \
     -blockdev '{"node-name": "file_image1", "driver": "file", "auto-read-only": true, "discard": "unmap", "aio": "threads", "filename": "/home/kvm_autotest_root/images/rhel930-64-virtio-scsi.qcow2", "cache": {"direct": true, "no-flush": false}}' \
     -blockdev '{"node-name": "drive_image1", "driver": "qcow2", "read-only": false, "cache": {"direct": true, "no-flush": false}, "file": "file_image1"}' \
     -device '{"driver": "scsi-hd", "id": "image1", "drive": "drive_image1", "write-cache": "on"}' \
     -device pcie-root-port,id=pcie-root-port-3,port=0x3,addr=0x1.0x3,bus=pcie.0,chassis=4 \
     -device virtio-net-pci,mac=9a:32:48:07:37:30,romfile=,id=idHzVGz4,netdev=idIKx0f8,bus=pcie-root-port-3,addr=0x0  \
     -netdev tap,id=idIKx0f8 \
     -blockdev '{"node-name": "file_cd1", "driver": "file", "auto-read-only": true, "discard": "unmap", "aio": "threads", "filename": "/home/kvm_autotest_root/iso/linux/RHEL-9.3.0-20230423.30-x86_64-dvd1.iso", "cache": {"direct": true, "no-flush": false}}' \
     -blockdev '{"node-name": "drive_cd1", "driver": "raw", "read-only": true, "cache": {"direct": true, "no-flush": false}, "file": "file_cd1"}' \
     -device '{"driver": "scsi-cd", "id": "cd1", "drive": "drive_cd1", "write-cache": "on"}' \
     -blockdev '{"node-name": "file_unattended", "driver": "file", "auto-read-only": true, "discard": "unmap", "aio": "threads", "filename": "/home/kvm_autotest_root/images/rhel930-64/ks.iso", "cache": {"direct": true, "no-flush": false}}' \
     -blockdev '{"node-name": "drive_unattended", "driver": "raw", "read-only": true, "cache": {"direct": true, "no-flush": false}, "file": "file_unattended"}' \
     -device '{"driver": "scsi-cd", "id": "unattended", "drive": "drive_unattended", "write-cache": "on"}'  \
     -kernel '/home/kvm_autotest_root/images/rhel930-64/vmlinuz'  \
     -append 'inst.sshd inst.repo=cdrom inst.ks=cdrom:/ks.cfg net.ifnames=0 console=ttyS0,115200'  \
     -initrd '/home/kvm_autotest_root/images/rhel930-64/initrd.img'  \
     -vnc :0  \
     -monitor stdio \
     -rtc base=utc,clock=host,driftfix=slew  \
     -boot menu=off,order=cdn,once=d,strict=off  \
     -no-shutdown \
     -enable-kvm \
     -device pcie-root-port,id=pcie_extra_root_port_0,multifunction=on,bus=pcie.0,addr=0x3,chassis=5

Result:
Guest measured boots, and check VM sev-es enabled.

Since the problem described in this bug report should be
resolved in a recent advisory, it has been closed with a
resolution of ERRATA.

For information on the advisory (Moderate: qemu-kvm security, bug fix, and enhancement update), and where to find the updated
files, follow the link below.

If the solution does not work for you, open a new bug report.

https://access.redhat.com/errata/RHSA-2023:6368