Bug 1815009

Summary: [2.13]fail to ping6 FIP on external namespace after destroy mac_binding of external ipv6 with ovn-sbctl
Product: Red Hat Enterprise Linux Fast Datapath Reporter: Jianlin Shi <jishi>
Component: ovn2.13Assignee: lorenzo bianconi <lorenzo.bianconi>
Status: CLOSED ERRATA QA Contact: Jianlin Shi <jishi>
Severity: medium Docs Contact:
Priority: medium    
Version: FDP 20.ACC: ctrautma, dcbw, jishi, mmichels, nusiddiq, ovs-team, ralongi
Target Milestone: ---   
Target Release: ---   
Hardware: Unspecified   
OS: Unspecified   
Whiteboard:
Fixed In Version: ovn2.13-2.13.0-12.el7fdn Doc Type: If docs needed, set a value
Doc Text:
Story Points: ---
Clone Of: 1815006 Environment:
Last Closed: 2020-05-26 14:07:17 UTC Type: ---
Regression: --- Mount Type: ---
Documentation: --- CRM:
Verified Versions: Category: ---
oVirt Team: --- RHEL 7.3 requirements from Atomic Host:
Cloudforms Team: --- Target Upstream Version:
Embargoed:
Bug Depends On: 1815006, 1815316    
Bug Blocks:    

Description Jianlin Shi 2020-03-19 10:10:29 UTC 
following issue also exists on ovn2.13.0-4:

+++ This bug was initially created as a clone of Bug #1815006 +++

Description of problem:
fail to ping6 FIP on external namespace after destroy mac_binding of external ipv6 with ovn-sbctl

Version-Release number of selected component (if applicable):
ovn2.12.0-36

How reproducible:
Always

Steps to Reproduce:
1.client:
#!/bin/bash                          
                                                     
systemctl start openvswitch                          
systemctl start ovn-northd 
ovn-nbctl set-connection ptcp:6641                 
ovn-sbctl set-connection ptcp:6642
ovs-vsctl set open . external_ids:system-id=hv1 external_ids:ovn-remote=tcp:20.0.50.25:6642 external_ids:ovn-encap-type=geneve external_ids:ovn-encap-ip=20.0.50.25
systemctl restart ovn-controller
ip netns add server0         
ip link add veth0_s0 netns server0 type veth peer name veth0_s0_p
ip netns exec server0 ip link set lo up
ip netns exec server0 ip link set veth0_s0 up        
ip netns exec server0 ip link set veth0_s0 address 00:00:00:01:01:02
ip netns exec server0 ip addr add 192.168.1.1/24 dev veth0_s0
ip netns exec server0 ip addr add 2000::1/64 dev veth0_s0
ip netns exec server0 ip route add default via 192.168.1.254 dev veth0_s0
ip netns exec server0 ip -6 route add default via 2000::a dev veth0_s0

                                                                      
ovs-vsctl add-port br-int veth0_s0_p
ip link set veth0_s0_p up 
ovs-vsctl set interface veth0_s0_p external_ids:iface-id=ls1p1      
ovn-nbctl ls-add ls1 
ovn-nbctl lsp-add ls1 ls1p1
ovn-nbctl lsp-set-addresses ls1p1 "00:00:00:01:01:02 192.168.1.1 2000::1"
ovn-nbctl lsp-add ls1 ls1p2                           
ovn-nbctl lsp-set-addresses ls1p2 "00:00:00:01:02:02 192.168.1.2 2000::2"
ovn-nbctl lr-add lr1                 
ovn-nbctl lrp-add lr1 lr1-ls1 00:00:00:00:00:01 192.168.1.254/24 2000::a/64
ovn-nbctl lsp-add ls1 ls1-lr1                    
ovn-nbctl lsp-set-type ls1-lr1 router
ovn-nbctl lsp-set-options ls1-lr1 router-port=lr1-ls1
ovn-nbctl lsp-set-addresses ls1-lr1 00:00:00:00:00:01
ovn-nbctl lsp-add ls1 ls1p3   
ovn-nbctl lsp-set-addresses ls1p3 00:00:00:01:03:02
                                                                                       
ovn-nbctl lrp-add lr1 lr1-ls0 00:de:ad:ff:01:03 172.16.1.254/24 3000::a/64      
ovn-nbctl ls-add ls0
ovn-nbctl lsp-add ls0 ls0-lr1   
ovn-nbctl lsp-set-addresses ls0-lr1 router
ovn-nbctl lsp-set-type ls0-lr1 router
ovn-nbctl lsp-set-options ls0-lr1 router-port=lr1-ls0

ovn-nbctl lsp-add ls0 ln_port
ovn-nbctl lsp-set-addresses ln_port unknown
ovn-nbctl lsp-set-type ln_port localnet
ovn-nbctl lsp-set-options ln_port network_name=nattest

ovn-nbctl set Logical_Router_Port lr1-ls0 options:redirect-chassis=hv1

ovs-vsctl add-br br-nat
ovs-vsctl set open . external_ids:ovn-bridge-mappings=nattest:br-nat
ip link set br-nat up

ip netns add p1
ip link add v0_p1_p type veth peer name v0_p1 netns p1
ip netns exec p1 ip link set lo up
ip netns exec p1 ip link set v0_p1 up
ip netns exec p1 ip addr add 172.16.1.1/24 dev v0_p1
ip netns exec p1 ip addr add 3000::1/64 dev v0_p1

ovs-vsctl add-port br-nat v0_p1_p 
ip link set v0_p1_p up
ovs-vsctl add-port br-nat p3p2

ovn-nbctl lr-nat-add lr1 dnat_and_snat 172.16.1.100 192.168.1.1 ls1p1 00:00:00:01:01:02
ovn-nbctl lr-nat-add lr1 dnat_and_snat 3000::100 2000::1 ls1p1 00:00:00:01:01:02


2.server:
#!/bin/bash


systemctl restart openvswitch
ovs-vsctl set open . external_ids:system-id=hv0 external_ids:ovn-remote=tcp:20.0.50.25:6642 external_ids:ovn-encap-type=geneve external_ids:ovn-encap-ip=20.0.50.26
systemctl restart ovn-controller

ip netns add client0
ip link add veth0_c0 netns client0 type veth peer name veth0_c0_p
ip netns exec client0 ip link set lo up
ip netns exec client0 ip link set veth0_c0 address 00:00:00:01:02:02
ip netns exec client0 ip link set veth0_c0 up                                                         
ip netns exec client0 ip addr add 192.168.1.2/24 dev veth0_c0
ip netns exec client0 ip addr add 2000::2/64 dev veth0_c0
ip netns exec client0 ip route add default via 192.168.1.254 dev veth0_c0
ip netns exec client0 ip -6 route add default via 2000::a dev veth0_c0
                                                                                                      
ovs-vsctl add-port br-int veth0_c0_p
ip link set veth0_c0_p up
ovs-vsctl set interface veth0_c0_p external_ids:iface-id=ls1p2

ovs-vsctl add-br br-nat                                                                               
ovs-vsctl set open . external_ids:ovn-bridge-mappings=nattest:br-nat
ip link set br-nat up
                                                                                                      
ip netns add p2
ip link add v0_p2_p type veth peer name v0_p2 netns p2                                                
ip netns exec p2 ip link set lo up
ip netns exec p2 ip link set v0_p2 up                                                                 
ip netns exec p2 ip addr add 172.16.1.2/24 dev v0_p2                                                  
ip netns exec p2 ip addr add 3000::2/64 dev v0_p2                                                     
                                                                                                      
ovs-vsctl add-port br-nat v0_p2_p
ip link set v0_p2_p up
                                                                                                      
ovs-vsctl add-port br-nat ens1f4d1

3. ping6 FIP on netns p2 on server:
[root@hp-dl380pg8-13 bz1809629]# ip netns exec p2 ping6 3000::100 -c 3   
PING 3000::100(3000::100) 56 data bytes                               
64 bytes from 3000::100: icmp_seq=1 ttl=63 time=7.37 ms               
64 bytes from 3000::100: icmp_seq=2 ttl=63 time=1.06 ms                                                                                                              
64 bytes from 3000::100: icmp_seq=3 ttl=63 time=0.133 ms
                                                              
--- 3000::100 ping statistics ---                                  
3 packets transmitted, 3 received, 0% packet loss, time 2002ms
rtt min/avg/max/mdev = 0.133/2.858/7.376/3.217 ms 

4. destroy mac_binding of 3000::2 with ovn-sbctl
[root@dell-per740-42 bz1809629]# ovn-sbctl find mac_binding ip=\"3000::2\"
_uuid               : 9e14fbf0-7c9d-4238-863d-f1da9f3b2a35                                            
datapath            : 23bf7055-6682-4cce-8059-6c7001947705                                            
ip                  : "3000::2"                                                                       
logical_port        : lr1-ls0                                       
mac                 : "36:8c:2f:1f:d1:2f"                     
[root@dell-per740-42 bz1809629]# ovn-sbctl destroy mac_binding 9e14fbf0-7c9d-4238-863d-f1da9f3b2a35 

5. ping6 FIP on netns p2 on server again:

[root@hp-dl380pg8-13 bz1809629]# ip netns exec p2 ping6 3000::100 -c 3
PING 3000::100(3000::100) 56 data bytes                                                               

--- 3000::100 ping statistics ---
3 packets transmitted, 0 received, 100% packet loss, time 1999ms

Actual results:
fail to ping6

Expected results:
ping6 succeed

Additional info:

[root@hp-dl380pg8-13 bz1809629]# rpm -qa | grep -E "openvswitch|ovn"
openvswitch2.12-2.12.0-23.el7fdp.x86_64
ovn2.12-2.12.0-36.el7fdp.x86_64
openvswitch-selinux-extra-policy-1.0-15.el7fdp.noarch
kernel-kernel-networking-openvswitch-ovn-common-1.0-7.noarch
python-openvswitch2.12-2.12.0-23.el7fdp.x86_64
ovn2.12-central-2.12.0-36.el7fdp.x86_64
kernel-kernel-networking-openvswitch-ovn-basic-1.0-21.noarch
ovn2.12-host-2.12.0-36.el7fdp.x86_64
Comment 3 Jianlin Shi 2020-04-22 06:22:10 UTC 
Verified on ovn2.13.0-18:

[root@hp-dl380pg8-13 bz1815009]# ip netns exec p2 ping6 3000::100 -c 3
PING 3000::100(3000::100) 56 data bytes
64 bytes from 3000::100: icmp_seq=1 ttl=63 time=5.91 ms                                               
64 bytes from 3000::100: icmp_seq=2 ttl=63 time=1.27 ms
64 bytes from 3000::100: icmp_seq=3 ttl=63 time=0.259 ms

--- 3000::100 ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 4ms
rtt min/avg/max/mdev = 0.259/2.480/5.914/2.462 ms

[root@dell-per740-42 bz1815009]# ovn-sbctl find mac_binding ip=\"3000::2\"
_uuid               : e64980cc-34b3-451e-9885-a5b5e56dad8e
datapath            : f3cafc9a-f604-44e8-9fca-de7d4dca8003
ip                  : "3000::2"                                                                       
logical_port        : lr1-ls0                                                                         
mac                 : "de:0d:20:f9:c1:cf"                                                             
[root@dell-per740-42 bz1815009]# ovn-sbctl destroy mac_binding e64980cc-34b3-451e-9885-a5b5e56dad8e

<=== destroy mac_binding

[root@hp-dl380pg8-13 bz1815009]# ip netns exec p2 ping6 3000::100 -c 3
PING 3000::100(3000::100) 56 data bytes
64 bytes from 3000::100: icmp_seq=1 ttl=63 time=3.92 ms
64 bytes from 3000::100: icmp_seq=2 ttl=63 time=1.29 ms
64 bytes from 3000::100: icmp_seq=3 ttl=63 time=0.241 ms

--- 3000::100 ping statistics ---                                                                     
3 packets transmitted, 3 received, 0% packet loss, time 5ms
rtt min/avg/max/mdev = 0.241/1.816/3.916/1.545 ms

<=== ping passed

[root@hp-dl380pg8-13 bz1815009]# rpm -qa | grep -E "openvswitch|ovn"
kernel-kernel-networking-openvswitch-ovn-load_balance-1.0-4.noarch
openvswitch-selinux-extra-policy-1.0-23.el8fdp.noarch
ovn2.13-2.13.0-18.el8fdp.x86_64
python3-openvswitch2.13-2.13.0-9.el8fdp.x86_64
ovn2.13-host-2.13.0-18.el8fdp.x86_64
kernel-kernel-networking-openvswitch-ovn-common-1.0-7.noarch
ovn2.13-central-2.13.0-18.el8fdp.x86_64
openvswitch2.13-2.13.0-9.el8fdp.x86_64
Comment 4 Jianlin Shi 2020-04-27 01:56:35 UTC 
Verified on rhel7 version:

[root@dell-per740-42 bz1815009]# ip netns exec p2 ping6 3000::100 -c 3
PING 3000::100(3000::100) 56 data bytes                                                               
64 bytes from 3000::100: icmp_seq=1 ttl=63 time=2.29 ms
64 bytes from 3000::100: icmp_seq=2 ttl=63 time=0.533 ms
64 bytes from 3000::100: icmp_seq=3 ttl=63 time=0.185 ms
                                                                                                      
--- 3000::100 ping statistics ---                                                                     
3 packets transmitted, 3 received, 0% packet loss, time 59ms
rtt min/avg/max/mdev = 0.185/1.002/2.290/0.922 ms 

[root@hp-dl380pg8-13 bz1815009]# ovn-sbctl find mac_binding ip=\"3000::2\"                            
_uuid               : 56666463-91ef-4fa5-a223-8d3b41d065d0                                            
datapath            : e94b4122-3c14-4c81-b6a2-8948edb23a9f                                            
ip                  : "3000::2"                                                                       
logical_port        : lr1-ls0                                                                         
mac                 : "66:22:2a:de:93:0e"                                                             
[root@hp-dl380pg8-13 bz1815009]# ovn-sbctl destroy mac_binding 56666463-91ef-4fa5-a223-8d3b41d065d0

[root@dell-per740-42 bz1815009]# ip netns exec p2 ping6 3000::100 -c 3
PING 3000::100(3000::100) 56 data bytes                                                               
64 bytes from 3000::100: icmp_seq=1 ttl=63 time=2.25 ms
64 bytes from 3000::100: icmp_seq=2 ttl=63 time=0.475 ms
64 bytes from 3000::100: icmp_seq=3 ttl=63 time=0.121 ms
                                                                                                      
--- 3000::100 ping statistics ---                                                                     
3 packets transmitted, 3 received, 0% packet loss, time 38ms
rtt min/avg/max/mdev = 0.121/0.948/2.249/0.931 ms

[root@hp-dl380pg8-13 bz1815009]# rpm -qa | grep -E "openvswitch|ovn"                                  
ovn2.13-central-2.13.0-18.el7fdp.x86_64                                                               
openvswitch2.13-2.13.0-17.el7fdp.x86_64                                                               
ovn2.13-host-2.13.0-18.el7fdp.x86_64                                                                  
ovn2.13-2.13.0-18.el7fdp.x86_64                                                                       
openvswitch-selinux-extra-policy-1.0-15.el7fdp.noarch
Comment 6 errata-xmlrpc 2020-05-26 14:07:17 UTC 
Since the problem described in this bug report should be
resolved in a recent advisory, it has been closed with a
resolution of ERRATA.

For information on the advisory, and where to find the updated
files, follow the link below.

If the solution does not work for you, open a new bug report.

https://access.redhat.com/errata/RHBA-2020:2317