Bug 2034454

Could you please provide your QEMU command line? Assuming it's the same as in https://bugzilla.redhat.com/show_bug.cgi?id=1961761#c23
and you're using 'Skylake-Server' CPU model you may need to add 'xsaves' and 'vmx-xsaves' CPU flags or switch to 'Skylake-Server-v5'.

> Could you please provide your QEMU command line? Assuming it's the same as
> in https://bugzilla.redhat.com/show_bug.cgi?id=1961761#c23
> and you're using 'Skylake-Server' CPU model you may need to add 'xsaves' and
> 'vmx-xsaves' CPU flags or switch to 'Skylake-Server-v5'.

Thanks Vitaly, this is the issue after added CPU model vmx and xsaves, please check the guest XML, xsaves and vmx are added. I am using Icelake server, in my CPU model, there are no flag vmx-xsaves. I explained this bug in https://bugzilla.redhat.com/show_bug.cgi?id=1961761#c36.

<domain type="kvm" id="2">
<name>win2016</name>
<uuid>f840f47d-3217-4dcd-8343-541f6212468f</uuid>
<metadata>
<libosinfo:libosinfo xmlns:libosinfo="http://libosinfo.org/xmlns/libvirt/domain/1.0">
<libosinfo:os id="http://microsoft.com/win/10"/>
</libosinfo:libosinfo>
</metadata>
<memory unit="KiB">4194304</memory>
<currentMemory unit="KiB">4194304</currentMemory>
<vcpu placement="static">2</vcpu>
<resource>
<partition>/machine</partition>
</resource>
<os>
<type arch="x86_64" machine="pc-q35-rhel8.5.0">hvm</type>
<loader readonly="yes" secure="yes" type="pflash">/usr/share/edk2/ovmf/OVMF_CODE.secboot.fd</loader>
<nvram template="/usr/share/edk2/ovmf/OVMF_VARS.secboot.fd">/var/lib/libvirt/qemu/nvram/win2016_VARS.fd</nvram>
<boot dev="hd"/>
</os>
<features>
<acpi/>
<apic/>
<hyperv>
<relaxed state="on"/>
<vapic state="on"/>
<spinlocks state="on" retries="8191"/>
<vpindex state="on"/>
<runtime state="on"/>
<synic state="on"/>
<stimer state="on">
<direct state="on"/>
</stimer>
<reset state="on"/>
<frequencies state="on"/>
<reenlightenment state="on"/>
<tlbflush state="on"/>
<ipi state="on"/>
<evmcs state="on"/>
</hyperv>
<smm state="on"/>
</features>
<cpu mode="custom" match="exact" check="full">
<model fallback="forbid">Icelake-Server</model>
<vendor>Intel</vendor>
<topology sockets="2" dies="1" cores="1" threads="1"/>
<feature policy="require" name="ss"/>
<feature policy="require" name="vmx"/>
<feature policy="require" name="pdcm"/>
<feature policy="require" name="hypervisor"/>
<feature policy="require" name="tsc_adjust"/>
<feature policy="require" name="avx512ifma"/>
<feature policy="require" name="sha-ni"/>
<feature policy="require" name="rdpid"/>
<feature policy="require" name="fsrm"/>
<feature policy="require" name="md-clear"/>
<feature policy="require" name="stibp"/>
<feature policy="require" name="arch-capabilities"/>
<feature policy="require" name="xsaves"/>
<feature policy="require" name="ibpb"/>
<feature policy="require" name="ibrs"/>
<feature policy="require" name="amd-stibp"/>
<feature policy="require" name="amd-ssbd"/>
<feature policy="require" name="rdctl-no"/>
<feature policy="require" name="ibrs-all"/>
<feature policy="require" name="skip-l1dfl-vmentry"/>
<feature policy="require" name="mds-no"/>
<feature policy="require" name="pschange-mc-no"/>
<feature policy="require" name="tsx-ctrl"/>
<feature policy="disable" name="hle"/>
<feature policy="disable" name="rtm"/>
<feature policy="disable" name="mpx"/>
<feature policy="disable" name="intel-pt"/>
</cpu>
<clock offset="localtime">
<timer name="rtc" tickpolicy="catchup"/>
<timer name="pit" tickpolicy="delay"/>
<timer name="hpet" present="no"/>
<timer name="hypervclock" present="yes"/>
</clock>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>destroy</on_crash>
<pm>
<suspend-to-mem enabled="no"/>
<suspend-to-disk enabled="no"/>
</pm>
<devices>
<emulator>/usr/libexec/qemu-kvm</emulator>
<disk type="file" device="disk">
<driver name="qemu" type="qcow2" cache="none"/>
<source file="/home/win2016.qcow2" index="4"/>
<backingStore/>
<target dev="sda" bus="sata"/>
<alias name="sata0-0-0"/>
<address type="drive" controller="0" bus="0" target="0" unit="0"/>
</disk>
<disk type="file" device="cdrom">
<driver name="qemu"/>
<target dev="sdb" bus="sata"/>
<readonly/>
<alias name="sata0-0-1"/>
<address type="drive" controller="0" bus="0" target="0" unit="1"/>
</disk>
<disk type="file" device="cdrom">
<driver name="qemu"/>
<target dev="sdc" bus="sata"/>
<readonly/>
<alias name="sata0-0-2"/>
<address type="drive" controller="0" bus="0" target="0" unit="2"/>
</disk>
<disk type="file" device="cdrom">
<driver name="qemu"/>
<target dev="sdd" bus="sata"/>
<readonly/>
<alias name="sata0-0-3"/>
<address type="drive" controller="0" bus="0" target="0" unit="3"/>
</disk>
<controller type="usb" index="0" model="qemu-xhci" ports="15">
<alias name="usb"/>
<address type="pci" domain="0x0000" bus="0x02" slot="0x00" function="0x0"/>
</controller>
<controller type="sata" index="0">
<alias name="ide"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x1f" function="0x2"/>
</controller>
<controller type="pci" index="0" model="pcie-root">
<alias name="pcie.0"/>
</controller>
<controller type="pci" index="1" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="1" port="0x10"/>
<alias name="pci.1"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x0" multifunction="on"/>
</controller>
<controller type="pci" index="2" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="2" port="0x11"/>
<alias name="pci.2"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x1"/>
</controller>
<controller type="pci" index="3" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="3" port="0x12"/>
<alias name="pci.3"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x2"/>
</controller>
<controller type="pci" index="4" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="4" port="0x13"/>
<alias name="pci.4"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x3"/>
</controller>
<interface type="bridge">
<mac address="52:54:00:18:46:b3"/>
<source bridge="br3"/>
<target dev="vnet1"/>
<model type="e1000e"/>
<alias name="net0"/>
<address type="pci" domain="0x0000" bus="0x01" slot="0x00" function="0x0"/>
</interface>
<serial type="file">
<source path="/tmp/win2016.log"/>
<target type="isa-serial" port="0">
<model name="isa-serial"/>
</target>
<alias name="serial0"/>
</serial>
<console type="file">
<source path="/tmp/win2016.log"/>
<target type="serial" port="0"/>
<alias name="serial0"/>
</console>
<input type="tablet" bus="usb">
<alias name="input0"/>
<address type="usb" bus="0" port="1"/>
</input>
<input type="mouse" bus="ps2">
<alias name="input1"/>
</input>
<input type="keyboard" bus="ps2">
<alias name="input2"/>
</input>
<tpm model="tpm-crb">
<backend type="emulator" version="2.0"/>
<alias name="tpm0"/>
</tpm>
<graphics type="vnc" port="5900" autoport="no" listen="0.0.0.0">
<listen type="address" address="0.0.0.0"/>
</graphics>
<audio id="1" type="none"/>
<video>
<model type="bochs" vram="16384" heads="1" primary="yes"/>
<alias name="video0"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x0"/>
</video>
<memballoon model="virtio">
<alias name="balloon0"/>
<address type="pci" domain="0x0000" bus="0x03" slot="0x00" function="0x0"/>
</memballoon>
</devices>
<seclabel type="dynamic" model="selinux" relabel="yes">
<label>system_u:system_r:svirt_t:s0:c297,c312</label>
<imagelabel>system_u:object_r:svirt_image_t:s0:c297,c312</imagelabel>
</seclabel>
<seclabel type="dynamic" model="dac" relabel="yes">
<label>+107:+107</label>
<imagelabel>+107:+107</imagelabel>
</seclabel>
</domain>(In reply to Vitaly Kuznetsov from comment #2)

(In reply to zixchen from comment #3)
> > Could you please provide your QEMU command line? Assuming it's the same as
> > in https://bugzilla.redhat.com/show_bug.cgi?id=1961761#c23
> > and you're using 'Skylake-Server' CPU model you may need to add 'xsaves' and
> > 'vmx-xsaves' CPU flags or switch to 'Skylake-Server-v5'.
> 
> Thanks Vitaly, this is the issue after added CPU model vmx and xsaves,
> please check the guest XML, xsaves and vmx are added. I am using Icelake
> server, in my CPU model, there are no flag vmx-xsaves. I explained this bug
> in https://bugzilla.redhat.com/show_bug.cgi?id=1961761#c36.
> 

'vmx-xsaves' is essential, please refer to https://bugzilla.redhat.com/show_bug.cgi?id=1942914
Regarding IceLake-Server, you need "Icelake-Server-v5" to make things work. You can also try
adding 'xsaves' and 'vmx-xsaves' manually (I see you already have 'xsaves' in your xml). Also,
if for whatever reason this is not supported in xml, you can try running QEMU manually (and then
we'll figure out why this is not supported by libvirt).

Thanks, Vitaly. I test with qemu command line, when added Icelake-Server-v5, Hyper V works in my L1 guest but with a warning. Qemu just uses "-cpu host + hyperV cpu features" also works for me. 

Could you help to check the warning?

Version:
qemu-kvm-6.1.0-3.module+el8.6.0+12982+5e169f40.x86_64
kernel-4.18.0-356.el8.x86_64
libvirt-7.8.0-1.module+el8.6.0+12982+5e169f40.x86_64

Qemu cmd line:
1. With Icelake-Server-v5, the guest can enable Hyper V and no sticking issue, but there is a warning.
...
-machine pc-q35-rhel8.5.0,accel=kvm,usb=off,smm=on,dump-guest-core=off \
-cpu Icelake-Server-v5,ss=on,vmx=on,pdcm=on,hypervisor=on,tsc-adjust=on,avx512ifma=on,sha-ni=on,rdpid=on,fsrm=on,md-clear=on,stibp=on,arch-capabilities=on,xsaves=on,ibpb=on,ibrs=on,amd-stibp=on,amd-ssbd=on,rdctl-no=on,ibrs-all=on,skip-l1dfl-vmentry=on,mds-no=on,pschange-mc-no=on,tsx-ctrl=on,hle=off,rtm=off,mpx=off,intel-pt=off,hv-time,hv-relaxed,hv-vapic,hv-spinlocks=0x1fff,hv-vpindex,hv-runtime,hv-synic,hv-stimer,hv-stimer-direct,hv-reset,hv-frequencies,hv-reenlightenment,hv-tlbflush,hv-ipi,hv-evmcs,hv-crash,vmx-xsaves \
...
QEMU 6.1.0 monitor - type 'help' for more information
(qemu) 2021-12-31T07:25:58.287999Z qemu-kvm: warning: host doesn't support requested feature: MSR(10AH).taa-no [bit 8]
2021-12-31T07:25:58.289392Z qemu-kvm: warning: host doesn't support requested feature: MSR(10AH).taa-no [bit 8]

2. If just use host model, also no sticking issue without features added like vmx, xsave, and vmx-xsave.
...
-machine pc-q35-rhel8.5.0,accel=kvm,usb=off,smm=on,dump-guest-core=off -cpu host,hv_relaxed,hv_vapic,hv_spinlocks=0x1fff,hv_vpindex,hv_runtime,hv_crash,hv_time,hv_synic,hv_stimer,hv_tlbflush,hv_ipi,hv_reset,hv_frequencies,hv_reenlightenment,hv_stimer_direct,hv_evmcs \
...

3. If using host model + vmx, xsave, and vmx-xsave, will return error
QEMU 6.1.0 monitor - type 'help' for more information
(qemu) 2021-12-31T07:36:32.594872Z qemu-kvm: can't apply global host-x86_64-cpu.vmx-xsave=on: Property 'host-x86_64-cpu.vmx-xsave' not found




Looks like the issue is from libvirt, the sticking issue shows when using cpu host model, 'vmx-xsave' and 'Icelake-Server-v5' not supported on libvirt, returns error like:
error: Failed to start domain 'win2016'
error: unsupported configuration: unknown CPU feature: vmx-xsave

(In reply to zixchen from comment #6)
> Thanks, Vitaly. I test with qemu command line, when added Icelake-Server-v5,
> Hyper V works in my L1 guest but with a warning. Qemu just uses "-cpu host +
> hyperV cpu features" also works for me. 

Thanks, yes,

'-cpu host' is also expected to work.

> 
> Could you help to check the warning?
> 

The warning says you physical CPU lacks a feature which virtual model has,
this is unexpected. BIOS/microcode update may help but maybe our virtual
model is not entirely correct. This deserves investigation but this is not
related to Windows.

I suggest to reassign this BZ to libvirt so newer cpu models/features are added.

Tim, please have a look. Thanks.

Moving to RHEL9 since that's where the dependent bug will be resolved.

Retest with kernel-4.18.0-372.46.1.el8_6.x86_64 and qemu-kvm-6.1.0-5.module+el8.6.0+13430+8fdd5f85.x86_64
(In reply to zixchen from comment #6)
> Thanks, Vitaly. I test with qemu command line, when added Icelake-Server-v5,
> Hyper V works in my L1 guest but with a warning. Qemu just uses "-cpu host +
> hyperV cpu features" also works for me. 
> 
> Could you help to check the warning?
> 
> Version:
> qemu-kvm-6.1.0-3.module+el8.6.0+12982+5e169f40.x86_64
> kernel-4.18.0-356.el8.x86_64
> libvirt-7.8.0-1.module+el8.6.0+12982+5e169f40.x86_64
> 
> Qemu cmd line:
> 1. With Icelake-Server-v5, the guest can enable Hyper V and no sticking
> issue, but there is a warning.
> ...
> -machine pc-q35-rhel8.5.0,accel=kvm,usb=off,smm=on,dump-guest-core=off \
> -cpu
> Icelake-Server-v5,ss=on,vmx=on,pdcm=on,hypervisor=on,tsc-adjust=on,
> avx512ifma=on,sha-ni=on,rdpid=on,fsrm=on,md-clear=on,stibp=on,arch-
> capabilities=on,xsaves=on,ibpb=on,ibrs=on,amd-stibp=on,amd-ssbd=on,rdctl-
> no=on,ibrs-all=on,skip-l1dfl-vmentry=on,mds-no=on,pschange-mc-no=on,tsx-
> ctrl=on,hle=off,rtm=off,mpx=off,intel-pt=off,hv-time,hv-relaxed,hv-vapic,hv-
> spinlocks=0x1fff,hv-vpindex,hv-runtime,hv-synic,hv-stimer,hv-stimer-direct,
> hv-reset,hv-frequencies,hv-reenlightenment,hv-tlbflush,hv-ipi,hv-evmcs,hv-
> crash,vmx-xsaves \
> ...
> QEMU 6.1.0 monitor - type 'help' for more information
> (qemu) 2021-12-31T07:25:58.287999Z qemu-kvm: warning: host doesn't support
> requested feature: MSR(10AH).taa-no [bit 8]
> 2021-12-31T07:25:58.289392Z qemu-kvm: warning: host doesn't support
> requested feature: MSR(10AH).taa-no [bit 8]
> 
This result is the same, CPU warning is still the same, and hper V can be enabled and reboot is ok. 


> 2. If just use host model, also no sticking issue without features added
> like vmx, xsave, and vmx-xsave.
> ...
> -machine pc-q35-rhel8.5.0,accel=kvm,usb=off,smm=on,dump-guest-core=off -cpu
> host,hv_relaxed,hv_vapic,hv_spinlocks=0x1fff,hv_vpindex,hv_runtime,hv_crash,
> hv_time,hv_synic,hv_stimer,hv_tlbflush,hv_ipi,hv_reset,hv_frequencies,
> hv_reenlightenment,hv_stimer_direct,hv_evmcs \
> ...
> 
The result is the same. 

> 3. If using host model + vmx, xsave, and vmx-xsave, will return error
> QEMU 6.1.0 monitor - type 'help' for more information
> (qemu) 2021-12-31T07:36:32.594872Z qemu-kvm: can't apply global
> host-x86_64-cpu.vmx-xsave=on: Property 'host-x86_64-cpu.vmx-xsave' not found
> 
-cpu host,ss=on,vmx=on,vmx-xsaves \
Also No issue with host model. vmx-xsave should be a typo.

Also check latest 8.6 compose, vmx-xsaves is still not in libvirt capabilities.

Version:
libvirt-8.0.0-5.9.module+el8.6.0+18552+b96c5a91.x86_64
qemu-kvm-6.2.0-11.module+el8.6.0+16538+01ea313d.6.x86_64

Steps:
# virsh domcapabilities|grep vmx-xsaves
# virsh domcapabilities|grep vmx
      <feature policy='require' name='vmx'/>
# virsh domcapabilities|grep xsaves
      <feature policy='require' name='xsaves'/>

<mode name='host-model' supported='yes'>
      <model fallback='forbid'>Icelake-Server</model>
      <vendor>Intel</vendor>
      <feature policy='require' name='ss'/>
      <feature policy='require' name='vmx'/>
      <feature policy='require' name='pdcm'/>
      <feature policy='require' name='hypervisor'/>
      <feature policy='require' name='tsc_adjust'/>
      <feature policy='require' name='avx512ifma'/>
      <feature policy='require' name='sha-ni'/>
      <feature policy='require' name='rdpid'/>
      <feature policy='require' name='fsrm'/>
      <feature policy='require' name='md-clear'/>
      <feature policy='require' name='stibp'/>
      <feature policy='require' name='arch-capabilities'/>
      <feature policy='require' name='xsaves'/>
      <feature policy='require' name='invtsc'/>
      <feature policy='require' name='ibpb'/>
      <feature policy='require' name='ibrs'/>
      <feature policy='require' name='amd-stibp'/>
      <feature policy='require' name='amd-ssbd'/>
      <feature policy='require' name='rdctl-no'/>
      <feature policy='require' name='ibrs-all'/>
      <feature policy='require' name='skip-l1dfl-vmentry'/>
      <feature policy='require' name='mds-no'/>
      <feature policy='require' name='pschange-mc-no'/>
      <feature policy='require' name='tsx-ctrl'/>
      <feature policy='disable' name='hle'/>
      <feature policy='disable' name='rtm'/>
      <feature policy='disable' name='mpx'/>
      <feature policy='disable' name='intel-pt'/>
    </mode>