Bug 1815316

Summary: [2.11]fail to ping6 FIP on external namespace after destroy mac_binding of external ipv6 with ovn-sbctl
Product: Red Hat Enterprise Linux Fast Datapath Reporter: Jianlin Shi <jishi>
Component: ovn2.11Assignee: lorenzo bianconi <lorenzo.bianconi>
Status: CLOSED ERRATA QA Contact: Jianlin Shi <jishi>
Severity: medium Docs Contact:
Priority: medium    
Version: FDP 20.ACC: ctrautma, jishi, lorenzo.bianconi, mmichels, nusiddiq, ralongi, tredaelli
Target Milestone: ---   
Target Release: ---   
Hardware: Unspecified   
OS: Unspecified   
Whiteboard:
Fixed In Version: ovn2.11-2.11.1-40.el7 Doc Type: If docs needed, set a value
Doc Text:
Story Points: ---
Clone Of: 1815006 Environment:
Last Closed: 2020-05-26 14:07:41 UTC Type: ---
Regression: --- Mount Type: ---
Documentation: --- CRM:
Verified Versions: Category: ---
oVirt Team: --- RHEL 7.3 requirements from Atomic Host:
Cloudforms Team: --- Target Upstream Version:
Embargoed:
Bug Depends On: 1815006    
Bug Blocks: 1815009    

Description Jianlin Shi 2020-03-20 00:52:52 UTC 
following issue also exist on ovn2.11.1-37

+++ This bug was initially created as a clone of Bug #1815006 +++

Description of problem:
fail to ping6 FIP on external namespace after destroy mac_binding of external ipv6 with ovn-sbctl

Version-Release number of selected component (if applicable):
ovn2.12.0-36

How reproducible:
Always

Steps to Reproduce:
1.client:
#!/bin/bash                          
                                                     
systemctl start openvswitch                          
systemctl start ovn-northd 
ovn-nbctl set-connection ptcp:6641                 
ovn-sbctl set-connection ptcp:6642
ovs-vsctl set open . external_ids:system-id=hv1 external_ids:ovn-remote=tcp:20.0.50.25:6642 external_ids:ovn-encap-type=geneve external_ids:ovn-encap-ip=20.0.50.25
systemctl restart ovn-controller
ip netns add server0         
ip link add veth0_s0 netns server0 type veth peer name veth0_s0_p
ip netns exec server0 ip link set lo up
ip netns exec server0 ip link set veth0_s0 up        
ip netns exec server0 ip link set veth0_s0 address 00:00:00:01:01:02
ip netns exec server0 ip addr add 192.168.1.1/24 dev veth0_s0
ip netns exec server0 ip addr add 2000::1/64 dev veth0_s0
ip netns exec server0 ip route add default via 192.168.1.254 dev veth0_s0
ip netns exec server0 ip -6 route add default via 2000::a dev veth0_s0

                                                                      
ovs-vsctl add-port br-int veth0_s0_p
ip link set veth0_s0_p up 
ovs-vsctl set interface veth0_s0_p external_ids:iface-id=ls1p1      
ovn-nbctl ls-add ls1 
ovn-nbctl lsp-add ls1 ls1p1
ovn-nbctl lsp-set-addresses ls1p1 "00:00:00:01:01:02 192.168.1.1 2000::1"
ovn-nbctl lsp-add ls1 ls1p2                           
ovn-nbctl lsp-set-addresses ls1p2 "00:00:00:01:02:02 192.168.1.2 2000::2"
ovn-nbctl lr-add lr1                 
ovn-nbctl lrp-add lr1 lr1-ls1 00:00:00:00:00:01 192.168.1.254/24 2000::a/64
ovn-nbctl lsp-add ls1 ls1-lr1                    
ovn-nbctl lsp-set-type ls1-lr1 router
ovn-nbctl lsp-set-options ls1-lr1 router-port=lr1-ls1
ovn-nbctl lsp-set-addresses ls1-lr1 00:00:00:00:00:01
ovn-nbctl lsp-add ls1 ls1p3   
ovn-nbctl lsp-set-addresses ls1p3 00:00:00:01:03:02
                                                                                       
ovn-nbctl lrp-add lr1 lr1-ls0 00:de:ad:ff:01:03 172.16.1.254/24 3000::a/64      
ovn-nbctl ls-add ls0
ovn-nbctl lsp-add ls0 ls0-lr1   
ovn-nbctl lsp-set-addresses ls0-lr1 router
ovn-nbctl lsp-set-type ls0-lr1 router
ovn-nbctl lsp-set-options ls0-lr1 router-port=lr1-ls0

ovn-nbctl lsp-add ls0 ln_port
ovn-nbctl lsp-set-addresses ln_port unknown
ovn-nbctl lsp-set-type ln_port localnet
ovn-nbctl lsp-set-options ln_port network_name=nattest

ovn-nbctl set Logical_Router_Port lr1-ls0 options:redirect-chassis=hv1

ovs-vsctl add-br br-nat
ovs-vsctl set open . external_ids:ovn-bridge-mappings=nattest:br-nat
ip link set br-nat up

ip netns add p1
ip link add v0_p1_p type veth peer name v0_p1 netns p1
ip netns exec p1 ip link set lo up
ip netns exec p1 ip link set v0_p1 up
ip netns exec p1 ip addr add 172.16.1.1/24 dev v0_p1
ip netns exec p1 ip addr add 3000::1/64 dev v0_p1

ovs-vsctl add-port br-nat v0_p1_p 
ip link set v0_p1_p up
ovs-vsctl add-port br-nat p3p2

ovn-nbctl lr-nat-add lr1 dnat_and_snat 172.16.1.100 192.168.1.1 ls1p1 00:00:00:01:01:02
ovn-nbctl lr-nat-add lr1 dnat_and_snat 3000::100 2000::1 ls1p1 00:00:00:01:01:02


2.server:
#!/bin/bash


systemctl restart openvswitch
ovs-vsctl set open . external_ids:system-id=hv0 external_ids:ovn-remote=tcp:20.0.50.25:6642 external_ids:ovn-encap-type=geneve external_ids:ovn-encap-ip=20.0.50.26
systemctl restart ovn-controller

ip netns add client0
ip link add veth0_c0 netns client0 type veth peer name veth0_c0_p
ip netns exec client0 ip link set lo up
ip netns exec client0 ip link set veth0_c0 address 00:00:00:01:02:02
ip netns exec client0 ip link set veth0_c0 up                                                         
ip netns exec client0 ip addr add 192.168.1.2/24 dev veth0_c0
ip netns exec client0 ip addr add 2000::2/64 dev veth0_c0
ip netns exec client0 ip route add default via 192.168.1.254 dev veth0_c0
ip netns exec client0 ip -6 route add default via 2000::a dev veth0_c0
                                                                                                      
ovs-vsctl add-port br-int veth0_c0_p
ip link set veth0_c0_p up
ovs-vsctl set interface veth0_c0_p external_ids:iface-id=ls1p2

ovs-vsctl add-br br-nat                                                                               
ovs-vsctl set open . external_ids:ovn-bridge-mappings=nattest:br-nat
ip link set br-nat up
                                                                                                      
ip netns add p2
ip link add v0_p2_p type veth peer name v0_p2 netns p2                                                
ip netns exec p2 ip link set lo up
ip netns exec p2 ip link set v0_p2 up                                                                 
ip netns exec p2 ip addr add 172.16.1.2/24 dev v0_p2                                                  
ip netns exec p2 ip addr add 3000::2/64 dev v0_p2                                                     
                                                                                                      
ovs-vsctl add-port br-nat v0_p2_p
ip link set v0_p2_p up
                                                                                                      
ovs-vsctl add-port br-nat ens1f4d1

3. ping6 FIP on netns p2 on server:
[root@hp-dl380pg8-13 bz1809629]# ip netns exec p2 ping6 3000::100 -c 3   
PING 3000::100(3000::100) 56 data bytes                               
64 bytes from 3000::100: icmp_seq=1 ttl=63 time=7.37 ms               
64 bytes from 3000::100: icmp_seq=2 ttl=63 time=1.06 ms                                                                                                              
64 bytes from 3000::100: icmp_seq=3 ttl=63 time=0.133 ms
                                                              
--- 3000::100 ping statistics ---                                  
3 packets transmitted, 3 received, 0% packet loss, time 2002ms
rtt min/avg/max/mdev = 0.133/2.858/7.376/3.217 ms 

4. destroy mac_binding of 3000::2 with ovn-sbctl
[root@dell-per740-42 bz1809629]# ovn-sbctl find mac_binding ip=\"3000::2\"
_uuid               : 9e14fbf0-7c9d-4238-863d-f1da9f3b2a35                                            
datapath            : 23bf7055-6682-4cce-8059-6c7001947705                                            
ip                  : "3000::2"                                                                       
logical_port        : lr1-ls0                                       
mac                 : "36:8c:2f:1f:d1:2f"                     
[root@dell-per740-42 bz1809629]# ovn-sbctl destroy mac_binding 9e14fbf0-7c9d-4238-863d-f1da9f3b2a35 

5. ping6 FIP on netns p2 on server again:

[root@hp-dl380pg8-13 bz1809629]# ip netns exec p2 ping6 3000::100 -c 3
PING 3000::100(3000::100) 56 data bytes                                                               

--- 3000::100 ping statistics ---
3 packets transmitted, 0 received, 100% packet loss, time 1999ms

Actual results:
fail to ping6

Expected results:
ping6 succeed

Additional info:

[root@hp-dl380pg8-13 bz1809629]# rpm -qa | grep -E "openvswitch|ovn"
openvswitch2.12-2.12.0-23.el7fdp.x86_64
ovn2.12-2.12.0-36.el7fdp.x86_64
openvswitch-selinux-extra-policy-1.0-15.el7fdp.noarch
kernel-kernel-networking-openvswitch-ovn-common-1.0-7.noarch
python-openvswitch2.12-2.12.0-23.el7fdp.x86_64
ovn2.12-central-2.12.0-36.el7fdp.x86_64
kernel-kernel-networking-openvswitch-ovn-basic-1.0-21.noarch
ovn2.12-host-2.12.0-36.el7fdp.x86_64
Comment 3 Jianlin Shi 2020-04-22 06:42:57 UTC 
verified on ovn2.11.1-44:

[root@hp-dl380pg8-12 bz1815316]# ip netns exec p2 ping6 3000::100 -c 3
PING 3000::100(3000::100) 56 data bytes
64 bytes from 3000::100: icmp_seq=1 ttl=63 time=7.14 ms
64 bytes from 3000::100: icmp_seq=2 ttl=63 time=1.50 ms                                               
64 bytes from 3000::100: icmp_seq=3 ttl=63 time=0.280 ms                                              

--- 3000::100 ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 5ms
rtt min/avg/max/mdev = 0.280/2.974/7.139/2.987 ms

[root@dell-per740-12 bz1815316]# ovn-sbctl find mac_binding ip=\"3000::2\"
_uuid               : c991ebe3-408e-4ba1-a2d9-a9659aed8132
datapath            : e4448130-a971-4095-a331-5fdb66d3c3f8
ip                  : "3000::2"                                                                       
logical_port        : "lr1-ls0"                                                                       
mac                 : "8a:60:e0:9a:53:d9"                                                             
[root@dell-per740-12 bz1815316]# ovn-sbctl destroy mac_binding c991ebe3-408e-4ba1-a2d9-a9659aed8132

<==== destroy mac_binding

[root@hp-dl380pg8-12 bz1815316]# ip netns exec p2 ping6 3000::100 -c 3
PING 3000::100(3000::100) 56 data bytes
64 bytes from 3000::100: icmp_seq=1 ttl=63 time=4.74 ms                                               
64 bytes from 3000::100: icmp_seq=2 ttl=63 time=1.47 ms                                               
64 bytes from 3000::100: icmp_seq=3 ttl=63 time=0.282 ms                                              

--- 3000::100 ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 6ms
rtt min/avg/max/mdev = 0.282/2.161/4.737/1.884 ms

<=== ping passed

[root@hp-dl380pg8-12 bz1815316]# rpm -qa | grep -E "ovn|openvswitch"
ovn2.11-central-2.11.1-44.el8fdp.x86_64                                                               
kernel-kernel-networking-openvswitch-ovn-basic-1.0-23.noarch                                          
openvswitch-selinux-extra-policy-1.0-23.el8fdp.noarch                                                 
python3-openvswitch2.11-2.11.0-50.el8fdp.x86_64                                                       
openvswitch2.11-2.11.0-50.el8fdp.x86_64                                                               
kernel-kernel-networking-openvswitch-ovn-add_setting-1.0-1.noarch                                     
ovn2.11-host-2.11.1-44.el8fdp.x86_64                                                                  
kernel-kernel-networking-openvswitch-ovn-common-1.0-7.noarch                                          
ovn2.11-2.11.1-44.el8fdp.x86_64
Comment 4 Jianlin Shi 2020-04-27 02:10:42 UTC 
Verified on rhel7 version:

[root@dell-per740-42 bz1815009]# ip netns exec p2 ping6 3000::100 -c 3                                
PING 3000::100(3000::100) 56 data bytes                                                               
64 bytes from 3000::100: icmp_seq=1 ttl=63 time=2.45 ms                                               
64 bytes from 3000::100: icmp_seq=2 ttl=63 time=0.536 ms                                              
64 bytes from 3000::100: icmp_seq=3 ttl=63 time=0.150 ms                                              
                                                                                                      
--- 3000::100 ping statistics ---                                                                     
3 packets transmitted, 3 received, 0% packet loss, time 24ms                                          
rtt min/avg/max/mdev = 0.150/1.045/2.449/1.005 ms

[root@hp-dl380pg8-13 bz1815009]# ovn-sbctl find mac_binding ip=\"3000::2\"                            
_uuid               : 96f6cd7f-8794-4ecb-aea7-a771ac692a90                                            
datapath            : b4493d80-87f5-4847-9bf1-d42f3ebbcc36                                            
ip                  : "3000::2"                                                                       
logical_port        : "lr1-ls0"                                                                       
mac                 : "ba:57:54:d2:45:d3"                                                             
[root@hp-dl380pg8-13 bz1815009]# ovn-sbctl destroy mac_binding 96f6cd7f-8794-4ecb-aea7-a771ac692a90

[root@dell-per740-42 bz1815009]# ip netns exec p2 ping6 3000::100 -c 3
PING 3000::100(3000::100) 56 data bytes                                                               
64 bytes from 3000::100: icmp_seq=1 ttl=63 time=2.35 ms
64 bytes from 3000::100: icmp_seq=2 ttl=63 time=0.535 ms
64 bytes from 3000::100: icmp_seq=3 ttl=63 time=0.502 ms
                                                                                                      
--- 3000::100 ping statistics ---                                                                     
3 packets transmitted, 3 received, 0% packet loss, time 66ms
rtt min/avg/max/mdev = 0.502/1.127/2.345/0.861 ms

[root@hp-dl380pg8-13 bz1815009]# rpm -qa | grep -E "openvswitch|ovn"
ovn2.11-host-2.11.1-44.el7fdp.x86_64
ovn2.11-central-2.11.1-44.el7fdp.x86_64
openvswitch2.11-2.11.0-53.20200327gita4efc59.el7fdp.x86_64
openvswitch-selinux-extra-policy-1.0-15.el7fdp.noarch
ovn2.11-2.11.1-44.el7fdp.x86_64
Comment 6 errata-xmlrpc 2020-05-26 14:07:41 UTC 
Since the problem described in this bug report should be
resolved in a recent advisory, it has been closed with a
resolution of ERRATA.

For information on the advisory, and where to find the updated
files, follow the link below.

If the solution does not work for you, open a new bug report.

https://access.redhat.com/errata/RHBA-2020:2318