Created attachment 1404249 [details]
XML of VM

Description of problem:
This is testing with vIOMMU. When ovs acts as vhost-user server, VM acts as vhost-user client, dpdk's testpmd can not receive packets.


Version-Release number of selected component (if applicable):
kernel-3.10.0-855.el7.x86_64
qemu-kvm-rhev-2.10.0-21.el7.x86_64
libvirt-3.9.0-13.el7.x86_64
dpdk-17.11-7.el7.x86_64
openvswitch-2.9.0-1.el7fdb.x86_64
microcode-20180108.tgz

How reproducible:
100%

Steps to Reproduce:
1. Boot OVS with vhost-iommu-support=true and acts as vhost-user server. Full script refer to[1]

ovs-vsctl --no-wait set Open_vSwitch . other_config:vhost-iommu-support=true

2. Boot VM acts as vhost-user client, full xml is attached to this Description.

3. In guest, load vfio, refer to[3]

4. In guest, start dpdk's testpmd, refer to[4]

5. In another host, start TRex as packets generator, refer to [5]. Testpmd fail to receive packets.


Actual results:
Guest dpdk's testpmd fail to receive packets when testing with vIOMMU and ovs vhost-user server.


Expected results:
Guest dpdk's testpmd should be able to receive packets when testing with vIOMMU and ovs vhost-user server.


Additional info:
1. When ovs acts as vhost-user client mode, guest dpdk's testpmd can receive packets, and everything works well, this scenario was tested/verified in below bug:
Bug 1532956 - Boot guest with vhost-user setting "iommu='on' ats='on'", dpdk's testpmd can not receive packets.


Reference:
[1]
# cat boot_ovs_server.sh 
#!/bin/bash

set -e

echo "killing old ovs process"
pkill -f ovs- || true
pkill -f ovsdb || true

echo "probing ovs kernel module"
modprobe -r openvswitch || true
modprobe openvswitch

rm -rf /var/run/openvswitch
mkdir /var/run/openvswitch

echo "clean env"
DB_FILE=/etc/openvswitch/conf.db
rm -rf /var/run/openvswitch
rm -f $DB_FILE
mkdir /var/run/openvswitch

echo "init ovs db and boot db server"
export DB_SOCK=/var/run/openvswitch/db.sock
ovsdb-tool create /etc/openvswitch/conf.db /usr/share/openvswitch/vswitch.ovsschema
ovsdb-server --remote=punix:$DB_SOCK --remote=db:Open_vSwitch,Open_vSwitch,manager_options --pidfile --detach --log-file
ovs-vsctl --no-wait init

echo "start ovs vswitch daemon"
ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true
ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-socket-mem="1024,1024"
ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-lcore-mask="0x1"
ovs-vsctl --no-wait set Open_vSwitch . other_config:vhost-iommu-support=true
ovs-vswitchd unix:$DB_SOCK --pidfile --detach --log-file=/var/log/openvswitch/ovs-vswitchd.log

echo "creating bridge and ports"

ovs-vsctl --if-exists del-br ovsbr0
ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev
ovs-vsctl add-port ovsbr0 dpdk0 -- set Interface dpdk0 type=dpdk options:dpdk-devargs=0000:81:00.0
ovs-vsctl add-port ovsbr0 vhost-user0 -- set Interface vhost-user0 type=dpdkvhostuser
ovs-ofctl del-flows ovsbr0
ovs-ofctl add-flow ovsbr0 "in_port=1,idle_timeout=0 actions=output:2"
ovs-ofctl add-flow ovsbr0 "in_port=2,idle_timeout=0 actions=output:1"

ovs-vsctl --if-exists del-br ovsbr1
ovs-vsctl add-br ovsbr1 -- set bridge ovsbr1 datapath_type=netdev
ovs-vsctl add-port ovsbr1 dpdk1 -- set Interface dpdk1 type=dpdk options:dpdk-devargs=0000:81:00.1
ovs-vsctl add-port ovsbr1 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuser
ovs-ofctl del-flows ovsbr1
ovs-ofctl add-flow ovsbr1 "in_port=1,idle_timeout=0 actions=output:2"
ovs-ofctl add-flow ovsbr1 "in_port=2,idle_timeout=0 actions=output:1"

echo "all done"

# ovs-vsctl show
16a8d9db-288b-4730-b1d3-04ab3522d132
    Bridge "ovsbr0"
        Port "dpdk0"
            Interface "dpdk0"
                type: dpdk
                options: {dpdk-devargs="0000:81:00.0", n_rxq="2"}
        Port "ovsbr0"
            Interface "ovsbr0"
                type: internal
        Port "vhost-user0"
            Interface "vhost-user0"
                type: dpdkvhostuser
    Bridge "ovsbr1"
        Port "ovsbr1"
            Interface "ovsbr1"
                type: internal
        Port "vhost-user1"
            Interface "vhost-user1"
                type: dpdkvhostuser
        Port "dpdk1"
            Interface "dpdk1"
                type: dpdk
                options: {dpdk-devargs="0000:81:00.1", n_rxq="2"}

[3]

# modprobe vfio
# modprobe vfio-pci
# dpdk-devbind --bind=vfio-pci 0000:03:00.0
# dpdk-devbind --bind=vfio-pci 0000:04:00.0

[4]
# cat testpmd.sh 
/usr/bin/testpmd \
-l 1,2,3,4,5 \
-n 4 \
-d /usr/lib64/librte_pmd_virtio.so \
-w 0000:03:00.0 -w 0000:04:00.0 \
-- \
--nb-cores=4 \
--disable-hw-vlan \
-i \
--disable-rss \
--rxq=2 --txq=2

[5]
# cat start_throughput.sh 
DIRECTORY=~/src/lua-trafficgen
cd $DIRECTORY
./binary-search.py \
        --traffic-generator=trex-txrx \
        --search-runtime=30 \
        --validation-runtime=60 \
        --rate-unit=mpps \
        --rate=0 \
        --run-bidirec=1 \
        --run-revunidirec=0 \
        --frame-size=64 \
        --num-flows=1024 \
        --one-shot=0 \
        --max-loss-pct=0 \
        --measure-latency=0