Description of problem: create vf on pf bind vf to vfio-pci driver add ovs bond with vf which was bind to vfio-pci find that it does not work Version-Release number of selected component (if applicable): [root@dell-per730-18 bond]# rpm -qa | grep dpdk dpdk-tools-18.11-4.el7_6.x86_64 dpdk-18.11-4.el7_6.x86_64 [root@dell-per730-18 bond]# rpm -qa | grep openv openvswitch-selinux-extra-policy-1.0-13.el7fdp.noarch openvswitch-2.9.0-110.el7fdp.x86_64 [root@dell-per730-18 bond]# uname -a Linux dell-per730-18.rhts.eng.pek2.redhat.com 3.10.0-957.el7.x86_64 #1 SMP Thu Oct 4 20:48:51 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux [root@dell-per730-18 bond]# cat /proc/cmdline BOOT_IMAGE=/vmlinuz-3.10.0-957.el7.x86_64 root=/dev/mapper/rhel_dell--per730--18-root ro intel_iommu=on kpti ksdevice=bootif crashkernel=auto rd.lvm.lv=rhel_dell-per730-18/root rd.lvm.lv=rhel_dell-per730-18/swap console=ttyS0,115200n81 LANG=en_US.UTF-8 skew_tick=1 nohz=on nohz_full=2,4,6,8,10,12,14,16,18,20,22,24,26,28,30 rcu_nocbs=2,4,6,8,10,12,14,16,18,20,22,24,26,28,30 tuned.non_isolcpus=aaaaaaab intel_pstate=disable nosoftlockup nohz=on default_hugepagesz=1G hugepagesz=1G hugepages=24 iommu=pt modprobe.blacklist=qedi modprobe.blacklist=qedf modprobe.blacklist=qedr [root@dell-per730-18 bond]# cat /proc/meminfo MemTotal: 65750584 kB MemFree: 36812256 kB MemAvailable: 36998256 kB Buffers: 4192 kB Cached: 506464 kB SwapCached: 0 kB Active: 432324 kB Inactive: 135652 kB Active(anon): 72652 kB Inactive(anon): 9152 kB Active(file): 359672 kB Inactive(file): 126500 kB Unevictable: 105336 kB Mlocked: 105336 kB SwapTotal: 29298684 kB SwapFree: 29298684 kB Dirty: 1472 kB Writeback: 0 kB AnonPages: 162720 kB Mapped: 53272 kB Shmem: 10864 kB Slab: 259964 kB SReclaimable: 70184 kB SUnreclaim: 189780 kB KernelStack: 10512 kB PageTables: 6652 kB NFS_Unstable: 0 kB Bounce: 0 kB WritebackTmp: 0 kB CommitLimit: 49591064 kB Committed_AS: 723580 kB VmallocTotal: 34359738367 kB VmallocUsed: 467872 kB VmallocChunk: 34325379068 kB HardwareCorrupted: 0 kB AnonHugePages: 4096 kB CmaTotal: 0 kB CmaFree: 0 kB HugePages_Total: 24 HugePages_Free: 16 HugePages_Rsvd: 0 HugePages_Surp: 0 Hugepagesize: 1048576 kB DirectMap4k: 367344 kB DirectMap2M: 5828608 kB DirectMap1G: 62914560 kB [root@dell-per730-18 bond]# How reproducible: always Steps to Reproduce: [root@dell-per730-18 bond]# echo 0 > /sys/bus/pci/devices/0000:07:00.0/sriov_numvfs [root@dell-per730-18 bond]# echo 1 > /sys/bus/pci/devices/0000:07:00.0/sriov_numvfs [root@dell-per730-18 bond]# ip link set p5p1 up [root@dell-per730-18 bond]# ip link show p5p1 34: p5p1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000 link/ether 3c:fd:fe:a0:3b:f0 brd ff:ff:ff:ff:ff:ff vf 0 MAC 00:00:00:00:00:00, spoof checking on, link-state auto, trust off [root@dell-per730-18 bond]# [root@dell-per730-18 bond]# [root@dell-per730-18 bond]# echo 0 > /sys/bus/pci/devices/0000:07:00.1/sriov_numvfs [root@dell-per730-18 bond]# echo 1 > /sys/bus/pci/devices/0000:07:00.1/sriov_numvfs [root@dell-per730-18 bond]# ip link set p5p2 up [root@dell-per730-18 bond]# ip link show p5p2 35: p5p2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000 link/ether 3c:fd:fe:a0:3b:f1 brd ff:ff:ff:ff:ff:ff vf 0 MAC 00:00:00:00:00:00, spoof checking on, link-state auto, trust off [root@dell-per730-18 bond]# [root@dell-per730-18 bond]# modprobe -r vfio-pci [root@dell-per730-18 bond]# modprobe -r vfio modprobe: ERROR: could not remove 'vfio': No such file or directory [root@dell-per730-18 bond]# modprobe vfio-pci [root@dell-per730-18 bond]# modprobe vfio [root@dell-per730-18 bond]# /usr/share/dpdk/usertools/dpdk-devbind.py -b vfio-pci 0000:07:02.0 [root@dell-per730-18 bond]# /usr/share/dpdk/usertools/dpdk-devbind.py -b vfio-pci 0000:07:0a.0 [root@dell-per730-18 bond]# /usr/share/dpdk/usertools/dpdk-devbind.py --status Network devices using DPDK-compatible driver ============================================ 0000:07:02.0 'Ethernet Virtual Function 700 Series 154c' drv=vfio-pci unused=i40evf 0000:07:0a.0 'Ethernet Virtual Function 700 Series 154c' drv=vfio-pci unused=i40evf Network devices using kernel driver =================================== 0000:01:00.0 'NetXtreme BCM5720 Gigabit Ethernet PCIe 165f' if=em1 drv=tg3 unused=vfio-pci *Active* 0000:01:00.1 'NetXtreme BCM5720 Gigabit Ethernet PCIe 165f' if=em2 drv=tg3 unused=vfio-pci 0000:02:00.0 'NetXtreme BCM5720 Gigabit Ethernet PCIe 165f' if=em3 drv=tg3 unused=vfio-pci 0000:02:00.1 'NetXtreme BCM5720 Gigabit Ethernet PCIe 165f' if=em4 drv=tg3 unused=vfio-pci 0000:04:00.0 'MT27800 Family [ConnectX-5] 1017' if=p6p1 drv=mlx5_core unused=vfio-pci 0000:04:00.1 'MT27800 Family [ConnectX-5] 1017' if=p6p2 drv=mlx5_core unused=vfio-pci 0000:05:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection 10fb' if=p7p1 drv=ixgbe unused=vfio-pci 0000:05:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection 10fb' if=p7p2 drv=ixgbe unused=vfio-pci 0000:07:00.0 'Ethernet Controller XL710 for 40GbE QSFP+ 1583' if=p5p1 drv=i40e unused=vfio-pci 0000:07:00.1 'Ethernet Controller XL710 for 40GbE QSFP+ 1583' if=p5p2 drv=i40e unused=vfio-pci 0000:83:00.0 'Ethernet Controller XXV710 for 25GbE SFP28 158b' if=p4p1 drv=i40e unused=vfio-pci 0000:83:00.1 'Ethernet Controller XXV710 for 25GbE SFP28 158b' if=p4p2 drv=i40e unused=vfio-pci 0000:86:00.0 'Ethernet Controller X710 for 10GbE SFP+ 1572' if=p2p1 drv=i40e unused=vfio-pci 0000:86:00.1 'Ethernet Controller X710 for 10GbE SFP+ 1572' if=p2p2 drv=i40e unused=vfio-pci No 'Crypto' devices detected ============================ No 'Eventdev' devices detected ============================== No 'Mempool' devices detected ============================= No 'Compress' devices detected ============================== [root@dell-per730-18 bond]# [root@dell-per730-18 bond]# [root@dell-per730-18 bond]# [root@dell-per730-18 bond]# [root@dell-per730-18 bond]# [root@dell-per730-18 bond]# modprobe openvswitch [root@dell-per730-18 bond]# systemctl stop openvswitch [root@dell-per730-18 bond]# sleep 3 [root@dell-per730-18 bond]# systemctl start openvswitch [root@dell-per730-18 bond]# sleep 3 [root@dell-per730-18 bond]# ovs-vsctl --if-exists del-br ovsbr0 [root@dell-per730-18 bond]# ovs-vsctl --if-exists del-br br-int [root@dell-per730-18 bond]# sleep 5 [root@dell-per730-18 bond]# ovs-vsctl set Open_vSwitch . other_config={} [root@dell-per730-18 bond]# ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true [root@dell-per730-18 bond]# ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-socket-mem=4096,4096 [root@dell-per730-18 bond]# ovs-vsctl set Open_vSwitch . other_config:pmd-cpu-mask=0x55555554 [root@dell-per730-18 bond]# ovs-vsctl --no-wait set Open_vSwitch . other_config:vhost-iommu-support=true [root@dell-per730-18 bond]# systemctl restart openvswitch [root@dell-per730-18 bond]# sleep 3 [root@dell-per730-18 bond]# ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev [root@dell-per730-18 bond]# [root@dell-per730-18 bond]# [root@dell-per730-18 bond]# [root@dell-per730-18 bond]# [root@dell-per730-18 bond]# ovs-vsctl add-bond ovsbr0 dpdkbond dpdk0 dpdk1 bond_mode=active-backup -- set Interface dpdk0 type=dpdk options:dpdk-devargs=class=eth,mac=36:c4:59:dd:20:b2 mtu_request=1500 -- set Interface dpdk1 type=dpdk options:dpdk-devargs=class=eth,mac=0a:cd:a7:95:e8:e0 mtu_request=1500 ovs-vsctl: Error detected while setting up 'dpdk0': Error attaching device 'class=eth,mac=36:c4:59:dd:20:b2' to DPDK. See ovs-vswitchd log for details. ovs-vsctl: Error detected while setting up 'dpdk1': Error attaching device 'class=eth,mac=0a:cd:a7:95:e8:e0' to DPDK. See ovs-vswitchd log for details. ovs-vsctl: The default log directory is "/var/log/openvswitch". [root@dell-per730-18 bond]# ovs-vsctl show adaf10ac-4a1a-4821-8d37-c36940e4cf57 Bridge "ovsbr0" Port dpdkbond Interface "dpdk1" type: dpdk options: {dpdk-devargs="class=eth,mac=0a:cd:a7:95:e8:e0"} error: "Error attaching device 'class=eth,mac=0a:cd:a7:95:e8:e0' to DPDK" Interface "dpdk0" type: dpdk options: {dpdk-devargs="class=eth,mac=36:c4:59:dd:20:b2"} error: "Error attaching device 'class=eth,mac=36:c4:59:dd:20:b2' to DPDK" Port "ovsbr0" Interface "ovsbr0" type: internal ovs_version: "2.9.0" [root@dell-per730-18 bond]# 1. 2. 3. Actual results: Expected results: Additional info:
Can you please check (and paste) /var/log/openvswitch/ovs-vswitchd.log? Did you tried to use the PCI address instead of the mac address?
PCI is the correct way to specify DPDK device. MAC addr was only introduced as a workaround for Mellanox ConnectX-3 as for these devices there are two ports associated with one PCI address and hence PCI is not enough to specify the port. Please continue to use PCI address to specify port for all NICs except Mellanox CX3. Will close as you have successfully configured with PCI.