Bug 2169333

Description of problem:
i40e card: testpmd start failed after reboot guest when running testpmd as switch case

Version-Release number of selected component (if applicable):
[root@dell-per730-52 ~]# rpm -qa|grep dpdk
dpdk-22.11-1.el9.x86_64
dpdk-tools-22.11-1.el9.x86_64
[root@dell-per730-52 ~]# uname -r
5.14.0-197.el9.x86_64


How reproducible:


Steps to Reproduce:
1. Use following script to start testpmd on host
[root@dell-per730-52 ~]# cat start_testpmd.py
import sys
import pexpect
testpmd_cmd=sys.argv[1]
cpu1=int(sys.argv[2])
cpu2=int(sys.argv[3])
cpu3=int(sys.argv[4])
port1=int(sys.argv[5])
port2=int(sys.argv[6])
port3=int(sys.argv[7])
port4=int(sys.argv[8])
#exe.logfile_read = sys.stdout
#exe.logfile = sys.stdout
#cmd
cmd="{} -l {},{},{} -n 4 --socket-mem 1024,1024 --vdev net_vhost0,iface=/tmp/vhost0,client=1,iommu-support=1,queues=1 --vdev net_vhost1,iface=/tmp/vhost1,client=1,iommu-support=1,queues=1 -- -i --nb-cores=2 --txq=1 --rxq=1 --forward-mode=io".format(testpmd_cmd,cpu1,cpu2,cpu3)
exe=pexpect.spawn(cmd, encoding='utf-8')
exe.logfile = sys.stdout
exe.expect('Done')
#exe.sendline('set portlist 0,2,1,3')
exe.sendline('set portlist {},{},{},{}'.format(port1,port2,port3,port4))
exe.expect('testpmd>')
exe.sendline('start')
exe.expect('testpmd>')
#exe.sendline('show port summary all')
exe.expect('testpmd>')
import time
time.sleep(6000)

[root@dell-per730-52 perf]# python3 /root/start_testpmd.py dpdk-testpmd 54 26 52 0 2 1 3
EAL: Detected CPU lcores: 56
EAL: Detected NUMA nodes: 2
EAL: Detected shared linkage of DPDK
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
EAL: Selected IOVA mode 'VA'
EAL: VFIO support initialized
EAL: Using IOMMU type 1 (Type 1)
EAL: Probe PCI driver: net_i40e (8086:158b) device: 0000:07:00.0 (socket 0)
EAL: Probe PCI driver: net_i40e (8086:158b) device: 0000:07:00.1 (socket 0)
TELEMETRY: No legacy callbacks, legacy socket not created
Interactive-mode selected
Set io packet forwarding mode
testpmd: create a new mbuf pool <mb_pool_0>: n=163456, size=2176, socket=0
testpmd: preferred mempool ops selected: ring_mp_mc
Configuring Port 0 (socket 0)
i40e_set_mac_max_frame(): Set max frame size at port level not applicable on link down
Port 0: 3C:FD:FE:AD:BC:E8
Configuring Port 1 (socket 0)
i40e_set_mac_max_frame(): Set max frame size at port level not applicable on link down
Port 1: 3C:FD:FE:AD:BC:E9
Configuring Port 2 (socket 0)
VHOST_CONFIG: (/tmp/vhost0) vhost-user client: socket created, fd: 35
VHOST_CONFIG: (/tmp/vhost0) failed to connect: No such file or directory
VHOST_CONFIG: (/tmp/vhost0) reconnecting...
Port 2: 56:48:4F:53:54:02
Configuring Port 3 (socket 0)
VHOST_CONFIG: (/tmp/vhost1) vhost-user client: socket created, fd: 39
VHOST_CONFIG: (/tmp/vhost1) failed to connect: No such file or directory
VHOST_CONFIG: (/tmp/vhost1) reconnecting...
Port 3: 56:48:4F:53:54:03
Checking link statuses...
Done
set portlist 0,2,1,3
testpmd> set portlist 0,2,1,3
start
testpmd> start
io packet forwarding - ports=4 - cores=2 - streams=4 - NUMA support enabled, MP allocation mode: native
Logical Core 52 (socket 0) forwards packets on 2 streams:
  RX P=0/Q=0 (socket 0) -> TX P=2/Q=0 (socket 0) peer=02:00:00:00:00:02
  RX P=2/Q=0 (socket 0) -> TX P=0/Q=0 (socket 0) peer=02:00:00:00:00:00
Logical Core 54 (socket 0) forwards packets on 2 streams:
  RX P=1/Q=0 (socket 0) -> TX P=3/Q=0 (socket 0) peer=02:00:00:00:00:03
  RX P=3/Q=0 (socket 0) -> TX P=1/Q=0 (socket 0) peer=02:00:00:00:00:01

  io packet forwarding packets/burst=32
  nb forwarding cores=2 - nb forwarding ports=4
  port 0: RX queue number: 1 Tx queue number: 1
    Rx offloads=0x0 Tx offloads=0x10000
    RX queue: 0
      RX desc=256 - RX free threshold=32
      RX threshold registers: pthresh=0 hthresh=0  wthresh=0
      RX Offloads=0x0
    TX queue: 0
      TX desc=256 - TX free threshold=32
      TX threshold registers: pthresh=32 hthresh=0  wthresh=0
      TX offloads=0x10000 - TX RS bit threshold=32
  port 1: RX queue number: 1 Tx queue number: 1
    Rx offloads=0x0 Tx offloads=0x10000
    RX queue: 0
      RX desc=256 - RX free threshold=32
      RX threshold registers: pthresh=0 hthresh=0  wthresh=0
      RX Offloads=0x0
    TX queue: 0
      TX desc=256 - TX free threshold=32
      TX threshold registers: pthresh=32 hthresh=0  wthresh=0
      TX offloads=0x10000 - TX RS bit threshold=32
  port 2: RX queue number: 1 Tx queue number: 1
    Rx offloads=0x0 Tx offloads=0x0
    RX queue: 0
      RX desc=0 - RX free threshold=0
      RX threshold registers: pthresh=0 hthresh=0  wthresh=0
      RX Offloads=0x0
    TX queue: 0
      TX desc=0 - TX free threshold=0
      TX threshold registers: pthresh=0 hthresh=0  wthresh=0
      TX offloads=0x0 - TX RS bit threshold=0
  port 3: RX queue number: 1 Tx queue number: 1
    Rx offloads=0x0 Tx offloads=0x0
    RX queue: 0
      RX desc=0 - RX free threshold=0
      RX threshold registers: pthresh=0 hthresh=0  wthresh=0
      RX Offloads=0x0
    TX queue: 0
      TX desc=0 - TX free threshold=0
      TX threshold registers: pthresh=0 hthresh=0  wthresh=0
      TX offloads=0x0 - TX RS bit threshold=0
testpmd> 

2. start guest with following xml
[root@dell-per730-52 perf]# cat g1.xml 
<domain type='kvm'>
  <name>g1</name>
  <memory unit='KiB'>8388608</memory>
  <currentMemory unit='KiB'>8388608</currentMemory>
  <memoryBacking>
    <hugepages>
      <page size='1048576' unit='KiB'/>
    </hugepages>
    <locked/>
    <access mode='shared'/>
  </memoryBacking>
  <vcpu placement='static'>3</vcpu>
  <cputune>
    <vcpupin vcpu='0' cpuset='2'/>
    <vcpupin vcpu='1' cpuset='30'/>
    <vcpupin vcpu='2' cpuset='4'/>
    <emulatorpin cpuset='0,28'/>
  </cputune>
  <numatune>
    <memory mode='strict' nodeset='0'/>
  </numatune>
  <resource>
    <partition>/machine</partition>
  </resource>
  <os>
    <type arch='x86_64' machine='q35'>hvm</type>
    <boot dev='hd'/>
  </os>
  <features>
    <acpi/>
    <apic/>
    <pmu state='off'/>
    <vmport state='off'/>
    <ioapic driver='qemu'/>
  </features>
  <cpu mode='host-passthrough' check='none'>
    <feature policy='require' name='tsc-deadline'/>
    <numa>
      <cell id='0' cpus='0-2' memory='8388608' unit='KiB' memAccess='shared'/>
    </numa>
  </cpu>
  <clock offset='utc'>
    <timer name='rtc' tickpolicy='catchup'/>
    <timer name='pit' tickpolicy='delay'/>
    <timer name='hpet' present='no'/>
  </clock>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>restart</on_crash>
  <pm>
    <suspend-to-mem enabled='no'/>
    <suspend-to-disk enabled='no'/>
  </pm>
  <devices>
    <emulator>/usr/libexec/qemu-kvm</emulator>
    <disk type='file' device='disk'>
      <driver name='qemu' type='qcow2'/>
      <source file='/var/lib/libvirt/images/g1.qcow2'/>
      <backingStore/>
      <target dev='vda' bus='virtio'/>
      <alias name='virtio-disk0'/>
      <address type='pci' domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
    </disk>
    <controller type='usb' index='0' model='none'>
      <alias name='usb'/>
    </controller>
    <controller type='pci' index='0' model='pcie-root'>
      <alias name='pcie.0'/>
    </controller>
    <controller type='pci' index='1' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='1' port='0x10'/>
      <alias name='pci.1'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
    </controller>
    <controller type='pci' index='2' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='2' port='0x11'/>
      <alias name='pci.2'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
    </controller>
    <controller type='pci' index='3' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='3' port='0x8'/>
      <alias name='pci.3'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
    </controller>
    <controller type='pci' index='4' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='4' port='0x9'/>
      <alias name='pci.4'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
    </controller>
    <controller type='pci' index='5' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='5' port='0xa'/>
      <alias name='pci.5'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'/>
    </controller>
    <controller type='pci' index='6' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='6' port='0xb'/>
      <alias name='pci.6'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/>
    </controller>
    <controller type='sata' index='0'>
      <alias name='ide'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x1f' function='0x2'/>
    </controller>
    <interface type='bridge'>
      <mac address='52:54:00:01:02:03'/>
      <source bridge='virbr0'/>
      <model type='virtio'/>
    </interface>
    <interface type='vhostuser'>
      <mac address='00:de:ad:00:00:01'/>
      <source type='unix' path='/tmp/vhost0' mode='server'/>
      <model type='virtio'/>
      <driver name='vhost' rx_queue_size='1024' tx_queue_size='1024' iommu='on' ats='on'>
  <host mrg_rxbuf='off'/>
  </driver>
      <address type='pci' domain='0x0000' bus='0x3' slot='0x00' function='0x0'/>
    </interface>
    <interface type='vhostuser'>
      <mac address='00:de:ad:00:00:02'/>
      <source type='unix' path='/tmp/vhost1' mode='server'/>
      <model type='virtio'/>
      <driver name='vhost' rx_queue_size='1024' tx_queue_size='1024' iommu='on' ats='on'>
  <host mrg_rxbuf='off'/>
  </driver>
      <address type='pci' domain='0x0000' bus='0x4' slot='0x00' function='0x0'/>
    </interface>
    <serial type='pty'>
      <source path='/dev/pts/1'/>
      <target type='isa-serial' port='0'>
        <model name='isa-serial'/>
      </target>
      <alias name='serial0'/>
    </serial>
    <console type='pty' tty='/dev/pts/1'>
      <source path='/dev/pts/1'/>
      <target type='serial' port='0'/>
      <alias name='serial0'/>
    </console>
    <input type='mouse' bus='ps2'>
      <alias name='input0'/>
    </input>
    <input type='keyboard' bus='ps2'>
      <alias name='input1'/>
    </input>
    <graphics type='vnc' port='5900' autoport='yes' listen='0.0.0.0'>
      <listen type='address' address='0.0.0.0'/>
    </graphics>
    <video>
      <model type='cirrus' vram='16384' heads='1' primary='yes'/>
      <alias name='video0'/>
      <address type='pci' domain='0x0000' bus='0x05' slot='0x00' function='0x0'/>
    </video>
    <memballoon model='virtio'>
      <alias name='balloon0'/>
      <address type='pci' domain='0x0000' bus='0x06' slot='0x00' function='0x0'/>
    </memballoon>
    <iommu model='intel'>
      <driver intremap='on' caching_mode='on' iotlb='on'/>
    </iommu>
  </devices>
  <seclabel type='dynamic' model='selinux' relabel='yes'/>
</domain>
[root@dell-per730-52 perf]# virsh create g1.xml 
Domain 'g1' created from g1.xml
 
3. virsh console to access the g1 and run reboot
[root@localhost ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: enp2s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 52:54:00:01:02:03 brd ff:ff:ff:ff:ff:ff
    inet 192.168.122.123/24 brd 192.168.122.255 scope global dynamic noprefixroute enp2s0
       valid_lft 3520sec preferred_lft 3520sec
    inet6 fe80::ff89:9b1b:ce42:384d/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever

[root@localhost ~]# reboot
 
4. after reboot the guest and start testpmd inside guest
[root@localhost ~]# driverctl -v list-overrides
0000:03:00.0 vfio-pci (Virtio network device)
0000:04:00.0 vfio-pci (Virtio network device)
[root@localhost ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: enp2s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 52:54:00:01:02:03 brd ff:ff:ff:ff:ff:ff
    inet 192.168.122.123/24 brd 192.168.122.255 scope global dynamic noprefixroute enp2s0
       valid_lft 3569sec preferred_lft 3569sec
    inet6 fe80::ff89:9b1b:ce42:384d/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
[root@localhost ~]# dpdk-testpmd -l 0-2 -n 1 --socket-mem 1024 -- -i --forward-mode=io --burst=32 --rxd=8192 --txd=8192 --max-pkt-len=9600 --mbuf-size=9728 --nb-cores=2 --rxq=1 --txq=1 --mbcache=512 --auto-start


Actual results:
testpmd start hung as follows.
[root@localhost ~]# dpdk-testpmd -l 0-2 -n 1 --socket-mem 1024 -- -i --forward-mode=io --burst=32 --rxd=8192 --txd=8192 --max-pkt-len=9600 --mbuf-size=9728 --nb-cores=2 --rxq=1 --txq=1 --mbcache=512 --auto-start
EAL: Detected CPU lcores: 3
EAL: Detected NUMA nodes: 1
EAL: Detected shared linkage of DPDK
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
EAL: Selected IOVA mode 'VA'
EAL: VFIO support initialized
EAL: Probe PCI driver: net_virtio (1af4:1041) device: 0000:02:00.0 (socket -1)
eth_virtio_pci_init(): Failed to init PCI device
EAL: Requested device 0000:02:00.0 cannot be used
EAL: Probe PCI driver: net_virtio (1af4:1041) device: 0000:03:00.0 (socket -1)
EAL: Using IOMMU type 1 (Type 1)
EAL: Probe PCI driver: net_virtio (1af4:1041) device: 0000:04:00.0 (socket -1)


Expected results:
Testpmd start succesfully inside guest.
When change dpdk to use dpdk-21.11.2-1.el9_1, it work well. so it only exist on dpdk-22.11-1.el9.x86_64.

Additional info:
https://beaker.engineering.redhat.com/jobs/7524366