Bug 1840008

Hi Jiri I Tested the following scenario cause I can not find the physical machines which can satisfy the confitions.
But the test results are not as expected, could you pls have a look? Thank you in advance. ：）

Version:
qemu-kvm-4.2.0-25.module+el8.2.1+6985+9fd9d514.x86_64
libvirt-6.0.0-24.module+el8.2.1+6997+c666f621.x86_64
kernel-4.18.0-193.11.1.el8_2.x86_64

Steps：
1.Physical host info (The physical host contains hle and rtm cpu flags)
# diff /usr/share/libvirt/cpu_map/x86_Icelake-Server.xml /usr/share/libvirt/cpu_map/x86_Icelake-Server-noTSX.xml 
2,3c2,3
<   <model name='Icelake-Server'>
<     <decode host='on' guest='on'/>
---
>   <model name='Icelake-Server-noTSX'>
>     <decode host='on' guest='off'/>
40d39
<     <feature name='hle'/>
68d66
<     <feature name='rtm'/>

# lscpu 
...
CPU family:          6
Model:               106
Model name:          Genuine Intel(R) CPU $0000%@
Stepping:            4
...
Flags:               fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cdp_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid md_clear pconfig flush_l1d arch_capabilities

# virsh domcapabilities
...
  <cpu>
    <mode name='host-passthrough' supported='yes'/>
    <mode name='host-model' supported='yes'>
      <model fallback='forbid'>Icelake-Server</model>
      <vendor>Intel</vendor>
      <feature policy='require' name='ss'/>
      <feature policy='require' name='vmx'/>
      <feature policy='require' name='hypervisor'/>
      <feature policy='require' name='tsc_adjust'/>
      <feature policy='require' name='avx512ifma'/>
      <feature policy='require' name='sha-ni'/>
      <feature policy='require' name='md-clear'/>
      <feature policy='require' name='stibp'/>
      <feature policy='require' name='arch-capabilities'/>
      <feature policy='require' name='xsaves'/>
      <feature policy='require' name='invtsc'/>
      <feature policy='require' name='ibpb'/>
      <feature policy='require' name='amd-ssbd'/>
      <feature policy='require' name='rdctl-no'/>
      <feature policy='require' name='ibrs-all'/>
      <feature policy='require' name='skip-l1dfl-vmentry'/>
      <feature policy='require' name='mds-no'/>
      <feature policy='require' name='pschange-mc-no'/>
      <feature policy='disable' name='mpx'/>
    </mode>

2. L1 guest info
# virsh domstate jiyan
shut off

# virsh dumpxml jiyan --inactive | grep "<cpu" -A5
  <cpu mode='host-passthrough' check='none'>
    <cache mode='passthrough'/>
    <feature policy='disable' name='hle'/>
    <feature policy='disable' name='rtm'/>
  </cpu>

# virsh start jiyan
Domain jiyan started

# virsh console jiyan
Connected to domain jiyan
Escape character is ^]

Red Hat Enterprise Linux 8.2 (Ootpa)
Kernel 4.18.0-193.el8.x86_64 on an x86_64

localhost login: root
Password: 
Last login: Wed Jul  1 15:13:25 on ttyS0

[root@localhost ~]# rpm -qa libvirt qemu-kvm kernel
qemu-kvm-4.2.0-28.module+el8.2.1+7211+16dfe810.x86_64
libvirt-6.0.0-25.module+el8.2.1+7154+47ffd890.x86_64
kernel-4.18.0-193.el8.x86_64

[root@localhost ~]# lscpu   [**The guest does not contain hle and rtm info.**]
...
CPU family:          6
Model:               106
Model name:          Genuine Intel(R) CPU $0000%@
Stepping:            4
...
Flags:               fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology cpuid tsc_known_freq pni pclmulqdq vmx ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch cpuid_fault invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves wbnoinvd arat avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq la57 md_clear arch_capabilities

[root@localhost ~]# virsh domcapabilities
...
  <cpu>
    <mode name='host-passthrough' supported='yes'/>
    <mode name='host-model' supported='yes'>
      <model fallback='forbid'>Icelake-Server</model>   ** There is no no-tsx here **
      <vendor>Intel</vendor>
      <feature policy='require' name='ss'/>
      <feature policy='require' name='vmx'/>
      <feature policy='require' name='hypervisor'/>
      <feature policy='require' name='tsc_adjust'/>
      <feature policy='require' name='avx512ifma'/>
      <feature policy='require' name='sha-ni'/>
      <feature policy='require' name='md-clear'/>
      <feature policy='require' name='stibp'/>
      <feature policy='require' name='arch-capabilities'/>
      <feature policy='require' name='xsaves'/>
      <feature policy='require' name='ibpb'/>
      <feature policy='require' name='amd-ssbd'/>
      <feature policy='require' name='rdctl-no'/>
      <feature policy='require' name='ibrs-all'/>
      <feature policy='require' name='skip-l1dfl-vmentry'/>
      <feature policy='require' name='mds-no'/>
      <feature policy='require' name='pschange-mc-no'/>
      <feature policy='disable' name='hle'/>
      <feature policy='disable' name='rtm'/>
      <feature policy='disable' name='mpx'/>
      <feature policy='disable' name='intel-pt'/>
    </mode>

And I also tested a physical host without hle and rtm cpu flags, the cpu model displayed seems has some problems too.
There are too many disabled cpu features showed in "virsh domcapabilities".

# rpm -qa libvirt qemu-kvm kernel
qemu-kvm-4.2.0-28.module+el8.2.1+7211+16dfe810.x86_64
kernel-4.18.0-193.10.1.el8_2.x86_64
libvirt-6.0.0-25.module+el8.2.1+7154+47ffd890.x86_64

# lscpu 
...
CPU family:          6
Model:               94
Model name:          Intel(R) Celeron(R) CPU G3900 @ 2.80GHz
Stepping:            3
...
Flags:               fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx est tm2 ssse3 sdbg cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb invpcid_single pti ssbd ibrs ibpb stibp tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust erms invpcid rdseed smap clflushopt intel_pt xsaveopt xsavec xgetbv1 xsaves dtherm arat pln pts hwp hwp_notify hwp_act_window hwp_epp md_clear flush_l1d

# lscpu | grep hle
No output

# lscpu | grep rtm
No output

# virsh domcapabilities 
...
  <cpu>
    <mode name='host-passthrough' supported='yes'/>
    <mode name='host-model' supported='yes'>
      <model fallback='forbid'>Skylake-Client-IBRS</model>
      <vendor>Intel</vendor>
      <feature policy='require' name='ss'/>
      <feature policy='require' name='hypervisor'/>
      <feature policy='require' name='tsc_adjust'/>
      <feature policy='require' name='clflushopt'/>
      <feature policy='require' name='umip'/>
      <feature policy='require' name='md-clear'/>
      <feature policy='require' name='stibp'/>
      <feature policy='require' name='arch-capabilities'/>
      <feature policy='require' name='ssbd'/>
      <feature policy='require' name='xsaves'/>
      <feature policy='require' name='pdpe1gb'/>
      <feature policy='require' name='invtsc'/>
      <feature policy='require' name='ibpb'/>
      <feature policy='require' name='amd-ssbd'/>
      <feature policy='require' name='skip-l1dfl-vmentry'/>
      <feature policy='require' name='pschange-mc-no'/>
      <feature policy='disable' name='fma'/>
      <feature policy='disable' name='avx'/>
      <feature policy='disable' name='f16c'/>
      <feature policy='disable' name='bmi1'/>
      <feature policy='disable' name='hle'/>
      <feature policy='disable' name='avx2'/>
      <feature policy='disable' name='smep'/>
      <feature policy='disable' name='bmi2'/>
      <feature policy='disable' name='rtm'/>
      <feature policy='disable' name='mpx'/>
      <feature policy='disable' name='adx'/>
    </mode>

(In reply to jiyan from comment #6)
> [root@localhost ~]# virsh domcapabilities
> ...
>   <cpu>
>     <mode name='host-passthrough' supported='yes'/>
>     <mode name='host-model' supported='yes'>
>       <model fallback='forbid'>Icelake-Server</model>   ** There is no no-tsx here **

This is correct. For migration compatibility with older libvirt (which did not
support all the new noTSX models) the noTSX variant will never be used
automatically as a host-model. The base model with hle and rtm disabled will
be used instead.

But you should see that the noTSX variant is usable (while the base model is
unusable) in domain capabilities XML:

    <cpu>
      ...
      <mode name='custom' supported='yes'>
        ...
        <model usable='yes'>Icelake-Server-noTSX</model>
        <model usable='no'>Icelake-Server</model>

Also tested "Cascadelake-Server-notsx" on a physical machine with the following steps.
The output of "virsh capabilities" displays "Cascadelake-Server-notsx" cpu model but the output of "virsh domcapabilities" does not display "Cascadelake-Server-notsx".

Version:
qemu-kvm-4.2.0-29.module+el8.3.0+7212+401047e6.x86_64
kernel-4.18.0-221.el8.x86_64
libvirt-libs-6.0.0-25.module+el8.3.0+7176+57f10f42.x86_64

Steps:
# lscpu 
Architecture:        x86_64
CPU op-mode(s):      32-bit, 64-bit
Byte Order:          Little Endian
CPU(s):              192
On-line CPU(s) list: 0-191
Thread(s) per core:  2
Core(s) per socket:  48
Socket(s):           2
NUMA node(s):        4
Vendor ID:           GenuineIntel
CPU family:          6
Model:               85
Model name:          Intel(R) Xeon(R) Platinum 9242 CPU @ 2.30GHz
Stepping:            7
CPU MHz:             3366.021
CPU max MHz:         3800.0000
CPU min MHz:         1000.0000
BogoMIPS:            4600.00
Virtualization:      VT-x
L1d cache:           32K
L1i cache:           32K
L2 cache:            1024K
L3 cache:            36608K
NUMA node0 CPU(s):   0-23,96-119
NUMA node1 CPU(s):   24-47,120-143
NUMA node2 CPU(s):   48-71,144-167
NUMA node3 CPU(s):   72-95,168-191
Flags:               fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cdp_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid cqm mpx rdt_a avx512f avx512dq rdseed adx smap clflushopt clwb intel_pt avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req pku ospke avx512_vnni md_clear flush_l1d arch_capabilities

# lscpu | grep arch_capabilities
Output here

# lscpu | grep hle
Output here

# lscpu | grep rtm
No output

# virsh capabilities
<capabilities>

  <host>
    <uuid>00597cf4-0879-e911-906e-001635649f5c</uuid>
    <cpu>
      <arch>x86_64</arch>
      <model>Cascadelake-Server-noTSX</model>
      <vendor>Intel</vendor>
      <microcode version='83898113'/>
      <counter name='tsc' frequency='2294609000' scaling='yes'/>
      <topology sockets='1' dies='1' cores='24' threads='2'/>
      <feature name='ds'/>
      <feature name='acpi'/>
      <feature name='ss'/>
      <feature name='ht'/>
      <feature name='tm'/>
      <feature name='pbe'/>
      <feature name='dtes64'/>
      <feature name='monitor'/>
      <feature name='ds_cpl'/>
      <feature name='vmx'/>
      <feature name='smx'/>
      <feature name='est'/>
      <feature name='tm2'/>
      <feature name='xtpr'/>
      <feature name='pdcm'/>
      <feature name='dca'/>
      <feature name='osxsave'/>
      <feature name='tsc_adjust'/>
      <feature name='cmt'/>
      <feature name='intel-pt'/>
      <feature name='pku'/>
      <feature name='ospke'/>
      <feature name='md-clear'/>
      <feature name='stibp'/>
      <feature name='arch-capabilities'/>
      <feature name='xsaves'/>
      <feature name='mbm_total'/>
      <feature name='mbm_local'/>
      <feature name='invtsc'/>
      <feature name='rdctl-no'/>
      <feature name='ibrs-all'/>
      <feature name='skip-l1dfl-vmentry'/>
      <feature name='mds-no'/>
      <feature name='tsx-ctrl'/>
      <pages unit='KiB' size='4'/>
      <pages unit='KiB' size='2048'/>
      <pages unit='KiB' size='1048576'/>
    </cpu>


# virsh domcapabilities
...
  <cpu>
    <mode name='host-passthrough' supported='yes'/>
    <mode name='host-model' supported='yes'>
      <model fallback='forbid'>Cascadelake-Server</model>
      <vendor>Intel</vendor>
      <feature policy='require' name='ss'/>
      <feature policy='require' name='vmx'/>
      <feature policy='require' name='hypervisor'/>
      <feature policy='require' name='tsc_adjust'/>
      <feature policy='require' name='umip'/>
      <feature policy='require' name='pku'/>
      <feature policy='require' name='md-clear'/>
      <feature policy='require' name='stibp'/>
      <feature policy='require' name='arch-capabilities'/>
      <feature policy='require' name='xsaves'/>
      <feature policy='require' name='invtsc'/>
      <feature policy='require' name='ibpb'/>
      <feature policy='require' name='amd-ssbd'/>
      <feature policy='require' name='rdctl-no'/>
      <feature policy='require' name='ibrs-all'/>
      <feature policy='require' name='skip-l1dfl-vmentry'/>
      <feature policy='require' name='mds-no'/>
      <feature policy='require' name='pschange-mc-no'/>
      <feature policy='disable' name='hle'/>
      <feature policy='disable' name='rtm'/>
    </mode>

(In reply to jiyan from comment #10)
> Also tested "Cascadelake-Server-notsx" on a physical machine with the
> following steps.
> The output of "virsh capabilities" displays "Cascadelake-Server-notsx" cpu
> model but the output of "virsh domcapabilities" does not display
> "Cascadelake-Server-notsx".

This is expected, see comment 8 for details.

Jiri

I did not notice comment 8 and another question here, 
whether it will be enough to verify this bug by starting VM with Cascadelake-Server-noTSX cpu model on the physical host (The output of "virsh capabilities" displays "Cascadelake-Server-notsx" cpu model but the output of "virsh domcapabilities" does not display "Cascadelake-Server-notsx".) showed in comment 10.

Test with below package version for L1 VM and host:
# rpm -q libvirt-libs qemu-kvm 
libvirt-libs-6.0.0-25.module+el8.2.1+7154+47ffd890.x86_64
qemu-kvm-4.2.0-29.module+el8.2.1+7297+a825794d.x86_64

For Skylake-Server-IBRS-noTSX:
1. Start L1 vm on host with "Skylake-Server-IBRS" cpu model with "hle" and "rtm" disabled:

# virsh dumpxml rhel --inactive | grep /cpu -B3
  <cpu mode='host-passthrough' check='partial'>
    <feature policy='disable' name='rtm'/>
    <feature policy='disable' name='hle'/>
  </cpu>

2. Check on L1 vm, the capabilities recognize it as Skylake-Server-noTSX-IBRS, and domcapabilities shows Skylake-Server-IBRS which is as expected.
# virsh capabilities 
<capabilities>

  <host>
    <uuid>4334211a-358f-482b-afd9-5f603382b219</uuid>
    <cpu>
      <arch>x86_64</arch>
  **    <model>Skylake-Server-noTSX-IBRS</model>  **
      <vendor>Intel</vendor>
      <microcode version='33581318'/>
      <topology sockets='1' dies='1' cores='1' threads='1'/>
      <feature name='ss'/>
      <feature name='vmx'/>
      <feature name='osxsave'/>
      <feature name='hypervisor'/>
      <feature name='tsc_adjust'/>
      <feature name='clflushopt'/>
      <feature name='umip'/>
      <feature name='pku'/>
      <feature name='ospke'/>
      <feature name='md-clear'/>
      <feature name='stibp'/>
      <feature name='arch-capabilities'/>
      <feature name='ssbd'/>
      <feature name='xsaves'/>
      <feature name='ibpb'/>
      <feature name='amd-ssbd'/>
      <feature name='skip-l1dfl-vmentry'/>
      <feature name='pschange-mc-no'/>
      <pages unit='KiB' size='4'/>
      <pages unit='KiB' size='2048'/>
      <pages unit='KiB' size='1048576'/>
    </cpu>
...

# virsh domcapabilities
...
  <cpu>
    <mode name='host-passthrough' supported='yes'/>
    <mode name='host-model' supported='yes'>
  **    <model fallback='forbid'>Skylake-Server-IBRS</model> **
      <vendor>Intel</vendor>
      <feature policy='require' name='ss'/>
      <feature policy='require' name='vmx'/>
      <feature policy='require' name='hypervisor'/>
      <feature policy='require' name='tsc_adjust'/>
      <feature policy='require' name='clflushopt'/>
      <feature policy='require' name='umip'/>
      <feature policy='require' name='pku'/>
      <feature policy='require' name='md-clear'/>
      <feature policy='require' name='stibp'/>
      <feature policy='require' name='arch-capabilities'/>
      <feature policy='require' name='ssbd'/>
      <feature policy='require' name='xsaves'/>
      <feature policy='require' name='ibpb'/>
      <feature policy='require' name='amd-ssbd'/>
      <feature policy='require' name='skip-l1dfl-vmentry'/>
      <feature policy='require' name='pschange-mc-no'/>
      <feature policy='disable' name='hle'/>
      <feature policy='disable' name='rtm'/>
      <feature policy='disable' name='mpx'/>
    </mode>
....
3. start a L2 vm with host-model , 
on L1 guest:
# virsh dumpxml new --inactive
<cpu mode='host-model' check='partial'/>
... 	

# virsh dumpxml new
<domain type='kvm' id='1'>
 <cpu mode='custom' match='exact' check='full'>
    <model fallback='forbid'>Skylake-Server-IBRS</model>
    <vendor>Intel</vendor>
    <feature policy='require' name='ss'/>
    <feature policy='require' name='vmx'/>
    <feature policy='require' name='hypervisor'/>
    <feature policy='require' name='tsc_adjust'/>
    <feature policy='require' name='clflushopt'/>
    <feature policy='require' name='umip'/>
    <feature policy='require' name='pku'/>
    <feature policy='require' name='md-clear'/>
    <feature policy='require' name='stibp'/>
    <feature policy='require' name='arch-capabilities'/>
    <feature policy='require' name='ssbd'/>
    <feature policy='require' name='xsaves'/>
    <feature policy='require' name='ibpb'/>
    <feature policy='require' name='amd-ssbd'/>
    <feature policy='require' name='skip-l1dfl-vmentry'/>
    <feature policy='require' name='pschange-mc-no'/>
    <feature policy='disable' name='hle'/>
    <feature policy='disable' name='rtm'/>
    <feature policy='disable' name='mpx'/>
  </cpu>

Since the problem described in this bug report should be
resolved in a recent advisory, it has been closed with a
resolution of ERRATA.

For information on the advisory, and where to find the updated
files, follow the link below.

If the solution does not work for you, open a new bug report.

https://access.redhat.com/errata/RHBA-2020:3172