Bug 1861042
| Summary: | Traffic is broken between to lports in the same chassis if the second lport VIF is deleted and added again. | ||
|---|---|---|---|
| Product: | Red Hat Enterprise Linux Fast Datapath | Reporter: | Numan Siddique <nusiddiq> |
| Component: | ovn2.13 | Assignee: | Numan Siddique <nusiddiq> |
| Status: | CLOSED ERRATA | QA Contact: | Jianlin Shi <jishi> |
| Severity: | unspecified | Docs Contact: | |
| Priority: | unspecified | ||
| Version: | RHEL 8.0 | CC: | ctrautma, jishi, ralongi |
| Target Milestone: | --- | ||
| Target Release: | --- | ||
| Hardware: | Unspecified | ||
| OS: | Unspecified | ||
| Whiteboard: | |||
| Fixed In Version: | Doc Type: | If docs needed, set a value | |
| Doc Text: | Story Points: | --- | |
| Clone Of: | Environment: | ||
| Last Closed: | 2020-08-18 11:23:51 UTC | Type: | Bug |
| Regression: | --- | Mount Type: | --- |
| Documentation: | --- | CRM: | |
| Verified Versions: | Category: | --- | |
| oVirt Team: | --- | RHEL 7.3 requirements from Atomic Host: | |
| Cloudforms Team: | --- | Target Upstream Version: | |
| Embargoed: | |||
I missed out the ACL commands..
Here are the commands to reproduce the issue
ovs-vsctl del-br br-int
# Delete all ovn resources. Or start frest.
ovn-nbctl ls-add sw0
ovn-nbctl lsp-add sw0 sw0-port1
ovn-nbctl lsp-set-addresses sw0-port1 "10:54:00:00:00:03 10.0.0.3"
ovn-nbctl lsp-add sw0 sw0-port2
ovn-nbctl lsp-set-addresses sw0-port2 "10:54:00:00:00:04 10.0.0.4"
ovn-nbctl lsp-add sw0 sw0-port3
ovn-nbctl lsp-set-addresses sw0-port3 "10:54:00:00:00:05 10.0.0.5"
ovn-nbctl acl-add sw0 from-lport 1002 "ip" allow-related
ovn-nbctl acl-add sw0 to-lport 1002 "ip" allow-related
ovs-vsctl add-port br-int sw0p1 -- set interface sw0p1 type=internal
ip netns add sw0p1
ip link set sw0p1 netns sw0p1
ip netns exec sw0p1 ip link set lo up
ip netns exec sw0p1 ip link set sw0p1 up
ip netns exec sw0p1 ip link set sw0p1 address 10:54:00:00:00:03
ip netns exec sw0p1 ip addr add 10.0.0.3/24 dev sw0p1
ip netns exec sw0p1 ip route add default via 10.0.0.1 dev sw0p1
ovs-vsctl set Interface sw0p1 external_ids:iface-id=sw0-port1
# Create the interface for lport sw0-port2
ovs-vsctl add-port br-int sw0p2 -- set interface sw0p2 type=internal
ip netns add sw0p2
ip link set sw0p2 netns sw0p2
ip netns exec sw0p2 ip link set lo up
ip netns exec sw0p2 ip link set sw0p2 up
ip netns exec sw0p2 ip link set sw0p2 address 10:54:00:00:00:04
ip netns exec sw0p2 ip addr add 10.0.0.4/24 dev sw0p2
ip netns exec sw0p2 ip route add default via 10.0.0.1 dev sw0p2
ovs-vsctl set Interface sw0p2 external_ids:iface-id=sw0-port2
# Create the interface for lport sw0-port3
ovs-vsctl add-port br-int sw0p3 -- set interface sw0p3 type=internal
ip netns add sw0p3
ip link set sw0p3 netns sw0p3
ip netns exec sw0p3 ip link set lo up
ip netns exec sw0p3 ip link set sw0p3 up
ip netns exec sw0p3 ip link set sw0p3 address 10:54:00:00:00:05
ip netns exec sw0p3 ip addr add 10.0.0.5/24 dev sw0p3
ip netns exec sw0p3 ip route add default via 10.0.0.1 dev sw0p3
ovs-vsctl set Interface sw0p3 external_ids:iface-id=sw0-port3
ovs-vsctl remove interface sw0p2 external_ids iface-id
ovs-vsctl remove interface sw0p3 external_ids iface-id
sleep 1
ovs-vsctl set interface sw0p3 external_ids:iface-id=sw0-port3
sleep 1
# ping should work, but doesn't work.
ip netns exec sw0p1 ping -c3 10.0.0.5
reproduced with following script:
systemctl start openvswitch
systemctl start ovn-northd
ovn-nbctl set-connection ptcp:6641
ovn-sbctl set-connection ptcp:6642
ovs-vsctl set open . external_ids:system-id=hv1 external_ids:ovn-remote=tcp:20.0.50.25:6642 external_ids:ovn-encap-type=geneve external_ids:ovn-encap-ip=20.0.50.25
systemctl restart ovn-controller
ovs-vsctl del-br br-int
ovn-nbctl ls-add sw0
ovn-nbctl lsp-add sw0 sw0-port1
ovn-nbctl lsp-set-addresses sw0-port1 "10:54:00:00:00:03 10.0.0.3"
ovn-nbctl lsp-add sw0 sw0-port2
ovn-nbctl lsp-set-addresses sw0-port2 "10:54:00:00:00:04 10.0.0.4"
ovn-nbctl lsp-add sw0 sw0-port3
ovn-nbctl lsp-set-addresses sw0-port3 "10:54:00:00:00:05 10.0.0.5"
ovn-nbctl acl-add sw0 from-lport 1002 "ip" allow-related
ovn-nbctl acl-add sw0 to-lport 1002 "ip" allow-related
ovs-vsctl add-port br-int sw0p1 -- set interface sw0p1 type=internal
ip netns add sw0p1
ip link set sw0p1 netns sw0p1
ip netns exec sw0p1 ip link set lo up
ip netns exec sw0p1 ip link set sw0p1 up
ip netns exec sw0p1 ip link set sw0p1 address 10:54:00:00:00:03
ip netns exec sw0p1 ip addr add 10.0.0.3/24 dev sw0p1
ip netns exec sw0p1 ip route add default via 10.0.0.1 dev sw0p1
ovs-vsctl set Interface sw0p1 external_ids:iface-id=sw0-port1
# Create the interface for lport sw0-port2
ovs-vsctl add-port br-int sw0p2 -- set interface sw0p2 type=internal
ip netns add sw0p2
ip link set sw0p2 netns sw0p2
ip netns exec sw0p2 ip link set lo up
ip netns exec sw0p2 ip link set sw0p2 up
ip netns exec sw0p2 ip link set sw0p2 address 10:54:00:00:00:04
ip netns exec sw0p2 ip addr add 10.0.0.4/24 dev sw0p2
ip netns exec sw0p2 ip route add default via 10.0.0.1 dev sw0p2
ovs-vsctl set Interface sw0p2 external_ids:iface-id=sw0-port2
# must sleep here to 100% reproduce
sleep 2
ovs-vsctl get bridge br-int external_ids
# Create the interface for lport sw0-port3
ovs-vsctl add-port br-int sw0p3 -- set interface sw0p3 type=internal
ip netns add sw0p3
ip link set sw0p3 netns sw0p3
ip netns exec sw0p3 ip link set lo up
ip netns exec sw0p3 ip link set sw0p3 up
ip netns exec sw0p3 ip link set sw0p3 address 10:54:00:00:00:05
ip netns exec sw0p3 ip addr add 10.0.0.5/24 dev sw0p3
ip netns exec sw0p3 ip route add default via 10.0.0.1 dev sw0p3
ovs-vsctl set Interface sw0p3 external_ids:iface-id=sw0-port3
sleep 1
ovn-sbctl show
ovs-vsctl get bridge br-int external_ids
sw0_dpkey=$(ovn-sbctl --bare --columns tunnel_key list datapath_binding sw0)
p3_dpkey=$(ovn-sbctl --bare --columns tunnel_key list port_binding sw0-port3)
ovs-ofctl dump-flows br-int table=33,metadata=${sw0_dpkey},reg15=0x${p3_dpkey} | grep REG13
ovs-vsctl remove interface sw0p3 external_ids iface-id
ovs-vsctl remove interface sw0p2 external_ids iface-id
ovs-vsctl get bridge br-int external_ids
sleep 1
ovs-ofctl dump-flows br-int table=33,metadata=${sw0_dpkey},reg15=0x${p3_dpkey} | grep REG13
ovs-vsctl set interface sw0p3 external_ids:iface-id=sw0-port3
sleep 1
ovs-vsctl get bridge br-int external_ids
# ping should work, but doesn't work.
ip netns exec sw0p1 ping -c3 10.0.0.5
reproduced on ovn2.13.0-20.06.1-4:
[root@dell-per740-42 bz1861042]# rpm -qa | grep -E "openvswitch|ovn"
openvswitch-selinux-extra-policy-1.0-23.el8fdp.noarch
ovn2.13-central-20.06.1-4.el8fdp.x86_64
openvswitch2.13-2.13.0-48.el8fdp.x86_64
ovn2.13-20.06.1-4.el8fdp.x86_64
ovn2.13-host-20.06.1-4.el8fdp.x86_64
+ ovs-vsctl set Interface sw0p3 external_ids:iface-id=sw0-port3
+ sleep 1
+ ovn-sbctl show
Chassis hv1
hostname: dell-per740-42.rhts.eng.pek2.redhat.com
Encap geneve
ip: "20.0.50.25"
options: {csum="true"}
Port_Binding sw0-port2
Port_Binding sw0-port3
Port_Binding sw0-port1
+ ovs-vsctl get bridge br-int external_ids
{ct-zone-272297b2-b726-4470-9082-c5c3c473994b_dnat="4", ct-zone-272297b2-b726-4470-9082-c5c3c473994b_snat="3", ct-zone-sw0-port1="1", ct-zone-sw0-port2="2", ct-zone-sw0-port3="5"}
++ ovn-sbctl --bare --columns tunnel_key list datapath_binding sw0
+ sw0_dpkey=1
++ ovn-sbctl --bare --columns tunnel_key list port_binding sw0-port3
+ p3_dpkey=3
+ ovs-ofctl dump-flows br-int table=33,metadata=1,reg15=0x3
+ grep REG13
cookie=0x0, duration=1.054s, table=33, n_packets=0, n_bytes=0, idle_age=1, priority=100,reg15=0x3,metadata=0x1 actions=load:0x5->NXM_NX_REG13[],load:0x4->NXM_NX_REG11[],load:0x3->NXM_NX_REG12[],resubmit(
,34)
+ ovs-vsctl remove interface sw0p3 external_ids iface-id
+ ovs-vsctl remove interface sw0p2 external_ids iface-id
+ ovs-vsctl get bridge br-int external_ids
{ct-zone-272297b2-b726-4470-9082-c5c3c473994b_dnat="4", ct-zone-272297b2-b726-4470-9082-c5c3c473994b_snat="3", ct-zone-sw0-port1="1"}
+ sleep 1
+ ovs-ofctl dump-flows br-int table=33,metadata=1,reg15=0x3
+ grep REG13
cookie=0x0, duration=2.094s, table=33, n_packets=0, n_bytes=0, idle_age=2, priority=100,reg15=0x3,metadata=0x1 actions=load:0x5->NXM_NX_REG13[],load:0x4->NXM_NX_REG11[],load:0x3->NXM_NX_REG12[],resubmit(
,34)
<=== flow is not cleared after delete VIF
+ ovs-vsctl set interface sw0p3 external_ids:iface-id=sw0-port3
+ sleep 1
+ ovs-vsctl get bridge br-int external_ids
{ct-zone-272297b2-b726-4470-9082-c5c3c473994b_dnat="4", ct-zone-272297b2-b726-4470-9082-c5c3c473994b_snat="3", ct-zone-sw0-port1="1", ct-zone-sw0-port3="2"}
<=== ct-zone id changed
+ ip netns exec sw0p1 ping -c3 10.0.0.5
PING 10.0.0.5 (10.0.0.5) 56(84) bytes of data.
--- 10.0.0.5 ping statistics ---
3 packets transmitted, 0 received, 100% packet loss, time 43ms
<=== ping failed
Verified on ovn2.13.0-20.06.1-6:
[root@dell-per740-42 bz1861042]# rpm -qa | grep -E "openvswitch|ovn"
openvswitch-selinux-extra-policy-1.0-23.el8fdp.noarch
ovn2.13-host-20.06.1-6.el8fdp.x86_64
openvswitch2.13-2.13.0-48.el8fdp.x86_64
ovn2.13-central-20.06.1-6.el8fdp.x86_64
ovn2.13-20.06.1-6.el8fdp.x86_64
+ ovs-vsctl set Interface sw0p3 external_ids:iface-id=sw0-port3
+ sleep 1
+ ovn-sbctl show
Chassis hv1
hostname: dell-per740-42.rhts.eng.pek2.redhat.com
Encap geneve
ip: "20.0.50.25"
options: {csum="true"}
Port_Binding sw0-port3
Port_Binding sw0-port2
Port_Binding sw0-port1
+ ovs-vsctl get bridge br-int external_ids
{ct-zone-4e218f14-d970-4790-9b45-b3ef13787d46_dnat="4", ct-zone-4e218f14-d970-4790-9b45-b3ef13787d46_snat="3", ct-zone-sw0-port1="1", ct-zone-sw0-port2="2", ct-zone-sw0-port3="5"}
++ ovn-sbctl --bare --columns tunnel_key list datapath_binding sw0
+ sw0_dpkey=1
++ ovn-sbctl --bare --columns tunnel_key list port_binding sw0-port3
+ p3_dpkey=3
+ grep REG13
+ ovs-ofctl dump-flows br-int table=33,metadata=1,reg15=0x3
cookie=0x0, duration=1.046s, table=33, n_packets=0, n_bytes=0, idle_age=1, priority=100,reg15=0x3,metadata=0x1 actions=load:0x5->NXM_NX_REG13[],load:0x4->NXM_NX_REG11[],load:0x3->NXM_NX_REG12[],resubmit(,34)
+ ovs-vsctl remove interface sw0p3 external_ids iface-id
+ ovs-vsctl remove interface sw0p2 external_ids iface-id
+ ovs-vsctl get bridge br-int external_ids
{ct-zone-4e218f14-d970-4790-9b45-b3ef13787d46_dnat="4", ct-zone-4e218f14-d970-4790-9b45-b3ef13787d46_snat="3", ct-zone-sw0-port1="1"}
+ sleep 1
+ ovs-ofctl dump-flows br-int table=33,metadata=1,reg15=0x3
+ grep REG13
<=== flow is deleted after delete VIF
+ ovs-vsctl set interface sw0p3 external_ids:iface-id=sw0-port3
+ sleep 1
+ ovs-vsctl get bridge br-int external_ids
{ct-zone-4e218f14-d970-4790-9b45-b3ef13787d46_dnat="4", ct-zone-4e218f14-d970-4790-9b45-b3ef13787d46_snat="3", ct-zone-sw0-port1="1", ct-zone-sw0-port3="2"}
<=== ct-zone id changed
+ ip netns exec sw0p1 ping -c3 10.0.0.5
PING 10.0.0.5 (10.0.0.5) 56(84) bytes of data.
64 bytes from 10.0.0.5: icmp_seq=1 ttl=64 time=1.39 ms
64 bytes from 10.0.0.5: icmp_seq=2 ttl=64 time=0.539 ms
64 bytes from 10.0.0.5: icmp_seq=3 ttl=64 time=0.078 ms
--- 10.0.0.5 ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 31ms
rtt min/avg/max/mdev = 0.078/0.669/1.391/0.544 ms
<=== ping passed
Verified on rhel7 version:
+ ip netns exec sw0p3 ip route add default via 10.0.0.1 dev sw0p3
+ ovs-vsctl set Interface sw0p3 external_ids:iface-id=sw0-port3
+ sleep 1
+ ovn-sbctl show
Chassis hv1
hostname: hp-dl380pg8-13.rhts.eng.pek2.redhat.com
Encap geneve
ip: "20.0.50.26"
options: {csum="true"}
Port_Binding sw0-port2
Port_Binding sw0-port1
Port_Binding sw0-port3
+ ovs-vsctl get bridge br-int external_ids
{ct-zone-570df576-79e7-4c47-938f-9f63df09104b_dnat="4", ct-zone-570df576-79e7-4c47-938f-9f63df09104b_snat="3", ct-zone-sw0-port1="1", ct-zone-sw0-port2="2", ct-zone-sw0-port3="5"}
++ ovn-sbctl --bare --columns tunnel_key list datapath_binding sw0
+ sw0_dpkey=1
++ ovn-sbctl --bare --columns tunnel_key list port_binding sw0-port3
+ p3_dpkey=3
+ ovs-ofctl dump-flows br-int table=33,metadata=1,reg15=0x3
+ grep REG13
cookie=0x0, duration=1.025s, table=33, n_packets=0, n_bytes=0, idle_age=1, priority=100,reg15=0x3,metadata=0x1 actions=load:0x5->NXM_NX_REG13[],load:0x4->NXM_NX_REG11[],load:0x3->NXM_NX_REG12[],resubmit(,34)
+ ovs-vsctl remove interface sw0p3 external_ids iface-id
+ ovs-vsctl remove interface sw0p2 external_ids iface-id
+ ovs-vsctl get bridge br-int external_ids
{ct-zone-570df576-79e7-4c47-938f-9f63df09104b_dnat="4", ct-zone-570df576-79e7-4c47-938f-9f63df09104b_snat="3", ct-zone-sw0-port1="1"}
+ sleep 1
+ ovs-ofctl dump-flows br-int table=33,metadata=1,reg15=0x3
+ grep REG13
+ ovs-vsctl set interface sw0p3 external_ids:iface-id=sw0-port3
+ sleep 1
+ ovs-vsctl get bridge br-int external_ids
{ct-zone-570df576-79e7-4c47-938f-9f63df09104b_dnat="4", ct-zone-570df576-79e7-4c47-938f-9f63df09104b_snat="3", ct-zone-sw0-port1="1", ct-zone-sw0-port3="2"}
+ ip netns exec sw0p1 ping -c3 10.0.0.5
PING 10.0.0.5 (10.0.0.5) 56(84) bytes of data.
64 bytes from 10.0.0.5: icmp_seq=1 ttl=64 time=0.808 ms
64 bytes from 10.0.0.5: icmp_seq=2 ttl=64 time=0.190 ms
64 bytes from 10.0.0.5: icmp_seq=3 ttl=64 time=0.032 ms
--- 10.0.0.5 ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 2000ms
rtt min/avg/max/mdev = 0.032/0.343/0.808/0.335 ms
[root@hp-dl380pg8-13 bz1861042]# rpm -qa | grep -E "openvswitch|ovn"
openvswitch2.13-2.13.0-39.el7fdp.x86_64
ovn2.13-host-20.06.1-6.el7fdp.x86_64
openvswitch-selinux-extra-policy-1.0-15.el7fdp.noarch
ovn2.13-central-20.06.1-6.el7fdp.x86_64
ovn2.13-20.06.1-6.el7fdp.x86_64
Since the problem described in this bug report should be resolved in a recent advisory, it has been closed with a resolution of ERRATA. For information on the advisory (ovn2.13 bug fix and enhancement update), and where to find the updated files, follow the link below. If the solution does not work for you, open a new bug report. https://access.redhat.com/errata/RHBA-2020:3488 |
Description of problem: Stepts to reproduce ------ ovn-nbctl ls-add sw0 ovn-nbctl lsp-add sw0 sw0-port1 ovn-nbctl lsp-set-addresses sw0-port1 "10:54:00:00:00:03 10.0.0.3" ovn-nbctl lsp-add sw0 sw0-port2 ovn-nbctl lsp-set-addresses sw0-port2 "10:54:00:00:00:04 10.0.0.4" ovn-nbctl lsp-add sw0 sw0-port3 ovn-nbctl lsp-set-addresses sw0-port3 "10:54:00:00:00:05 10.0.0.5" ovs-vsctl add-port br-int sw0p1 -- set interface sw0p1 type=internal ip netns add sw0p1 ip link set sw0p1 netns sw0p1 ip netns exec sw0p1 ip link set lo up ip netns exec sw0p1 ip link set sw0p1 up ip netns exec sw0p1 ip link set sw0p1 address 10:54:00:00:00:03 ip netns exec sw0p1 ip addr add 10.0.0.3/24 dev sw0p1 ip netns exec sw0p1 ip route add default via 10.0.0.1 dev sw0p1 ovs-vsctl set Interface sw0p1 external_ids:iface-id=sw0-port1 # Create the interface for lport sw0-port2 ovs-vsctl add-port br-int sw0p2 -- set interface sw0p2 type=internal ip netns add sw0p2 ip link set sw0p2 netns sw0p2 ip netns exec sw0p2 ip link set lo up ip netns exec sw0p2 ip link set sw0p2 up ip netns exec sw0p2 ip link set sw0p2 address 10:54:00:00:00:04 ip netns exec sw0p2 ip addr add 10.0.0.4/24 dev sw0p2 ip netns exec sw0p2 ip route add default via 10.0.0.1 dev sw0p2 ovs-vsctl set Interface sw0p2 external_ids:iface-id=sw0-port2 # Create the interface for lport sw0-port3 ovs-vsctl add-port br-int sw0p3 -- set interface sw0p3 type=internal ip netns add sw0p3 ip link set sw0p3 netns sw0p3 ip netns exec sw0p3 ip link set lo up ip netns exec sw0p3 ip link set sw0p3 up ip netns exec sw0p3 ip link set sw0p3 address 10:54:00:00:00:05 ip netns exec sw0p3 ip addr add 10.0.0.5/24 dev sw0p3 ip netns exec sw0p3 ip route add default via 10.0.0.1 dev sw0p3 ovs-vsctl set Interface sw0p3 external_ids:iface-id=sw0-port3 # ping should work ip netns exec sw0p1 ping -c3 10.0.0.5 ovs-vsctl remove interface sw0p2 external_ids iface-id ovs-vsctl remove interface sw0p3 external_ids iface-id sleep 1 ovs-vsctl set interface sw0p3 external_ids:iface-id=sw0-port3 sleep 1 # ping should work, but doesn't work. ip netns exec sw0p1 ping -c3 10.0.0.5 Version-Release number of selected component (if applicable): How reproducible: Steps to Reproduce: 1. 2. 3. Actual results: Expected results: Additional info: