Bug 1609643

Created attachment 1471414 [details]
Guest XML

Description of problem:

This is testing with enabling vIOMMU support.

Boot ovs, then boot VM. Start testpmd in guest, then kill it, start it again will cause ovs-vswitchd segfault. 


Version-Release number of selected component (if applicable):
openvswitch-2.9.0-55.el7fdp.x86_64
3.10.0-927.el7.x86_64
qemu-kvm-rhev-2.12.0-8.el7.x86_64
libvirt-4.5.0-4.el7.x86_64
tuned-2.9.0-1.el7.noarch
dpdk-17.11-10.el7fdb.x86_64


How reproducible:
100%


Steps to Reproduce:
1. Boot ovs, refer to[1]

2. Boot VM, full xml is attached.

3. Start testpmd in guest, refer to[3]

4. Destroy testpmd

# ps aux | grep testpmd
root      1653  370  0.8 4406036 70496 pts/0   SLl+ 13:33   3:34 /usr/bin/testpmd -l 1,2,3,4,5 -n 4 -d /usr/lib64/librte_pmd_virtio.so.1 -w 0000:06:00.0 -w 0000:07:00.0 -- --nb-cores=4 --disable-hw-vlan -i --disable-rss --rxq=2 --txq=2
...
# kill -9 1653


5. Start testpmd again, refer to[5], ovs segfault.

# dmesg (in host)
...
[11531.550929] ovs-vswitchd[11947]: segfault at 2ab07ffb1002 ip 0000557a3552d360 sp 00007f5a037fd4d0 error 4 in ovs-vswitchd[557a3533b000+4b8000]


Actual results:

Guest testpmd operations can cause host ovs crash.


Expected results:

ovs should not crash.


Reference:
[1]
# cat boot_ovs_client.sh 
#!/bin/bash

set -e

echo "killing old ovs process"
pkill -f ovs-vswitchd || true
sleep 5
pkill -f ovsdb-server || true

echo "probing ovs kernel module"
modprobe -r openvswitch || true
modprobe openvswitch

echo "clean env"
DB_FILE=/etc/openvswitch/conf.db
rm -rf /var/run/openvswitch
mkdir /var/run/openvswitch
rm -f $DB_FILE

echo "init ovs db and boot db server"
export DB_SOCK=/var/run/openvswitch/db.sock
ovsdb-tool create /etc/openvswitch/conf.db /usr/share/openvswitch/vswitch.ovsschema
ovsdb-server --remote=punix:$DB_SOCK --remote=db:Open_vSwitch,Open_vSwitch,manager_options --pidfile --detach --log-file
ovs-vsctl --no-wait init

echo "start ovs vswitch daemon"
ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true
ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-socket-mem="1024,1024"
ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-lcore-mask="0x1"
ovs-vsctl --no-wait set Open_vSwitch . other_config:vhost-iommu-support=true
ovs-vswitchd unix:$DB_SOCK --pidfile --detach --log-file=/var/log/openvswitch/ovs-vswitchd.log

echo "creating bridge and ports"

ovs-vsctl --if-exists del-br ovsbr0
ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev
ovs-vsctl add-port ovsbr0 dpdk0 -- set Interface dpdk0 type=dpdk options:dpdk-devargs=0000:04:00.0 
ovs-vsctl add-port ovsbr0 vhost-user0 -- set Interface vhost-user0 type=dpdkvhostuserclient options:vhost-server-path=/tmp/vhostuser0.sock
ovs-ofctl del-flows ovsbr0
ovs-ofctl add-flow ovsbr0 "in_port=1,idle_timeout=0 actions=output:2"
ovs-ofctl add-flow ovsbr0 "in_port=2,idle_timeout=0 actions=output:1"

ovs-vsctl --if-exists del-br ovsbr1
ovs-vsctl add-br ovsbr1 -- set bridge ovsbr1 datapath_type=netdev
ovs-vsctl add-port ovsbr1 dpdk1 -- set Interface dpdk1 type=dpdk options:dpdk-devargs=0000:04:00.1
ovs-vsctl add-port ovsbr1 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/vhostuser1.sock
ovs-ofctl del-flows ovsbr1
ovs-ofctl add-flow ovsbr1 "in_port=1,idle_timeout=0 actions=output:2"
ovs-ofctl add-flow ovsbr1 "in_port=2,idle_timeout=0 actions=output:1"

ovs-vsctl set Open_vSwitch . other_config={}
ovs-vsctl set Open_vSwitch . other_config:dpdk-lcore-mask=0x1
ovs-vsctl set Open_vSwitch . other_config:pmd-cpu-mask=0x15554
ovs-vsctl set Interface dpdk0 options:n_rxq=2
ovs-vsctl set Interface dpdk1 options:n_rxq=2


echo "all done"

# sh boot_ovs_client.sh

# ovs-vsctl show
9b34b335-067e-493e-bc00-d8ba41e0c82d
    Bridge "ovsbr1"
        Port "vhost-user1"
            Interface "vhost-user1"
                type: dpdkvhostuserclient
                options: {vhost-server-path="/tmp/vhostuser1.sock"}
        Port "dpdk1"
            Interface "dpdk1"
                type: dpdk
                options: {dpdk-devargs="0000:04:00.1", n_rxq="2"}
        Port "ovsbr1"
            Interface "ovsbr1"
                type: internal
    Bridge "ovsbr0"
        Port "ovsbr0"
            Interface "ovsbr0"
                type: internal
        Port "dpdk0"
            Interface "dpdk0"
                type: dpdk
                options: {dpdk-devargs="0000:04:00.0", n_rxq="2"}
        Port "vhost-user0"
            Interface "vhost-user0"
                type: dpdkvhostuserclient
                options: {vhost-server-path="/tmp/vhostuser0.sock"}


[3]
# modprobe vfio
# modprobe vfio-pci
# echo 4 >  /sys/devices/system/node/node0/hugepages/hugepages-1048576kB/nr_hugepages
# dpdk-devbind --bind=vfio-pci 0000:06:00.0
# dpdk-devbind --bind=vfio-pci 0000:07:00.0

# /usr/bin/testpmd \
-l 1,2,3,4,5 \
-n 4 \
-d /usr/lib64/librte_pmd_virtio.so.1 \
-w 0000:06:00.0 -w 0000:07:00.0 \
-- \
--nb-cores=4 \
--disable-hw-vlan \
-i \
--disable-rss \
--rxq=2 --txq=2

testpmd> start

[5]
# /usr/bin/testpmd \
-l 1,2,3,4,5 \
-n 4 \
-d /usr/lib64/librte_pmd_virtio.so.1 \
-w 0000:06:00.0 -w 0000:07:00.0 \
-- \
--nb-cores=4 \
--disable-hw-vlan \
-i \
--disable-rss \
--rxq=2 --txq=2

testpmd> start

Additional info: