Bug 1762341

Summary: [OVN] Keepalived handover doesn't work on single-node deployment with virtual interface type
Product: Red Hat Enterprise Linux Fast Datapath Reporter: Maciej Józefczyk <mjozefcz>
Component: ovn2.11Assignee: Numan Siddique <nusiddiq>
Status: CLOSED ERRATA QA Contact: Jianlin Shi <jishi>
Severity: unspecified Docs Contact:
Priority: unspecified    
Version: FDP 20.ACC: bperkins, ctrautma, dalvarez, jishi, jlibosva, kfida, nusiddiq, qding
Target Milestone: ---   
Target Release: ---   
Hardware: Unspecified   
OS: Unspecified   
Whiteboard:
Fixed In Version: ovn2.11-2.11.1-13.el7fdn Doc Type: If docs needed, set a value
Doc Text:
Story Points: ---
Clone Of: Environment:
Last Closed: 2020-01-21 17:02:44 UTC Type: Bug
Regression: --- Mount Type: ---
Documentation: --- CRM:
Verified Versions: Category: ---
oVirt Team: --- RHEL 7.3 requirements from Atomic Host:
Cloudforms Team: --- Target Upstream Version:
Embargoed:

Description Maciej Józefczyk 2019-10-16 14:49:59 UTC
Description of problem:

Keepalived handover doesn't work while instances are on the same chassis.

How reproducible:


Steps to Reproduce:
1. Spawn devstack with https://review.opendev.org/#/c/676223/
2. Spawn 2 instances, follow procedure described here: https://blog.codecentric.de/en/2016/11/highly-available-vips-openstack-vms-vrrp/
3. Try to move VIP address to second VM
4. Check that ping stopped working on VIP FIP

Actual results:
Handover doesn't work. VIP FIP is not reachable

Expected results:
FIP reachable

Comment 1 Numan Siddique 2019-11-04 13:11:59 UTC
The fix is available in downstream - ovn2.11-2.11.1-13.el7fdn.
The fix should be available in next FDP 20.A

Comment 3 Jianlin Shi 2019-12-20 02:44:39 UTC
reproduced on ovn2.11-2.11.1-8.el7fdp.x86_64:

[root@dell-per740-12 bz1762341]# rpm -qa | grep -E "openvswitch|ovn"                                  
openvswitch2.11-2.11.0-35.el7fdp.x86_64                                                               
openvswitch-selinux-extra-policy-1.0-14.el7fdp.noarch                                                 
ovn2.11-2.11.1-8.el7fdp.x86_64                                                                        
ovn2.11-host-2.11.1-8.el7fdp.x86_64                                                                   
ovn2.11-central-2.11.1-8.el7fdp.x86_64

[root@dell-per740-12 bz1762341]# bash -x rep.sh                                                       
+ systemctl start openvswitch                                                                         
+ systemctl start ovn-northd                                                                          
+ ovn-nbctl set-connection ptcp:6641                                                                  
+ ovn-sbctl set-connection ptcp:6642
+ ovs-vsctl set open . external_ids:system-id=hv1 external_ids:ovn-remote=tcp:20.0.30.25:6642 external_ids:ovn-encap-type=geneve external_ids:ovn-encap-ip=20.0.30.25
+ systemctl restart ovn-controller
+ ip netns add server0
+ ip link add veth0_s0 type veth peer name veth0_s0_p
+ ip link set veth0_s0 netns server0                                                                  
+ ip netns exec server0 ip link set lo up                                                             
+ ip netns exec server0 ip link set veth0_s0 up                                                       
+ ip netns exec server0 ip link set veth0_s0 address 00:00:00:01:01:02                                
+ ip netns exec server0 ip addr add 172.16.2.1/24 dev veth0_s0                                        
+ ip netns exec server0 ip addr add 2001::1/64 dev veth0_s0                                           
+ ip netns exec server0 ip route add default via 172.16.2.254 dev veth0_s0                            
+ ip netns exec server0 ip -6 route add default via 2001::a dev veth0_s0                              
+ ip netns add server2
+ ip link add veth0_s2 type veth peer name veth0_s2_p                                                 
+ ip link set veth0_s2 netns server2                                                                  
+ ip netns exec server2 ip link set lo up                                                             
+ ip netns exec server2 ip link set veth0_s2 up                                                       
+ ip netns exec server2 ip link set veth0_s2 address 00:00:00:01:01:03                                
+ ip netns exec server2 ip addr add 172.16.2.2/24 dev veth0_s2
+ ip netns exec server2 ip addr add 2001::2/64 dev veth0_s2
+ ip netns exec server2 ip route add default via 172.16.2.254 dev veth0_s2
+ ip netns exec server2 ip -6 route add default via 2001::a dev veth0_s2                              
+ ovs-vsctl add-port br-int veth0_s0_p                                                                
+ ip link set veth0_s0_p up                                                                           
+ ovs-vsctl set interface veth0_s0_p external_ids:iface-id=ls2p1                                      
+ ovs-vsctl add-port br-int veth0_s2_p                                                                
+ ip link set veth0_s2_p up
+ ovs-vsctl set interface veth0_s2_p external_ids:iface-id=ls2p2                                      
+ ovn-nbctl ls-add ls2
+ ovn-nbctl lsp-add ls2 ls2p1                                                                         
+ ovn-nbctl lsp-set-addresses ls2p1 00:00:00:01:01:02                                                 
+ ovn-nbctl lsp-add ls2 ls2p2                                                                         
+ ovn-nbctl lsp-set-addresses ls2p2 00:00:00:01:01:03                                                 
+ ovn-nbctl lsp-add ls2 ls2-vir                                                                       
+ ovn-nbctl lsp-set-addresses ls2-vir '00:00:00:01:01:10 172.16.1.10'                                 
+ ovn-nbctl lsp-set-type ls2-vir virtual
+ ovn-nbctl set logical_switch_port ls2-vir options:virtual-ip=172.16.1.10
+ ovn-nbctl set logical_switch_port ls2-vir options:virtual-parents=ls2p1,ls2p2

[root@dell-per740-12 bz1762341]# cat garp.py 
from scapy.all import *                                                                               
sendp(Ether(src="00:00:00:01:01:02",dst="ff:ff:ff:ff:ff:ff")/ARP(op=1,hwsrc="00:00:00:01:01:02",hwdst="00:00:00:00:00:00",psrc="172.16.1.10",pdst="172.16.1.10"),iface="veth0_s0")                         
[root@dell-per740-12 bz1762341]# cat garp2.py 
from scapy.all import *                                                                               
sendp(Ether(src="00:00:00:01:01:03",dst="ff:ff:ff:ff:ff:ff")/ARP(op=1,hwsrc="00:00:00:01:01:03",hwdst="00:00:00:00:00:00",psrc="172.16.1.10",pdst="172.16.1.10"),iface="veth0_s2")

[root@dell-per740-12 bz1762341]# ip netns exec server0 python garp.py                                 
.
Sent 1 packets.

<==== send garp on port ls2p1

[root@dell-per740-12 ~]# ovn-sbctl list port_binding 
_uuid               : 28d6248a-52ab-4482-86d5-29414a427e54
chassis             : 12f9f1f2-b5e6-4700-85c5-03db3cdd9b7e                                            
datapath            : 7e1747bf-7118-4fd1-80b7-05530a4b51cb                     
encap               : []                                                                              
external_ids        : {}                                                                              
gateway_chassis     : []                                                                              
ha_chassis_group    : []                                                                              
logical_port        : "ls2-vir"                                                                       
mac                 : ["00:00:00:01:01:10 172.16.1.10"]                                               
nat_addresses       : []                                  
options             : {virtual-ip="172.16.1.10", virtual-parents="ls2p1,ls2p2"}
parent_port         : []                                  
tag                 : []                                                                              
tunnel_key          : 3                                                                               
type                : virtual                                                                         
virtual_parent      : "ls2p1"

<=== ls2p1 is bound

[root@dell-per740-12 bz1762341]# ip netns exec server2 python garp2.py
.
Sent 1 packets.

<==== send garp on port ls2p2

[root@dell-per740-12 ~]# ovn-sbctl list port_binding
_uuid               : 28d6248a-52ab-4482-86d5-29414a427e54
chassis             : 12f9f1f2-b5e6-4700-85c5-03db3cdd9b7e
datapath            : 7e1747bf-7118-4fd1-80b7-05530a4b51cb
encap               : []                                                                              
external_ids        : {}                                                                              
gateway_chassis     : []                                                                              
ha_chassis_group    : []                                                                              
logical_port        : "ls2-vir"                                                                       
mac                 : ["00:00:00:01:01:10 172.16.1.10"]
nat_addresses       : []
options             : {virtual-ip="172.16.1.10", virtual-parents="ls2p1,ls2p2"}
parent_port         : []                                                                              
tag                 : []
tunnel_key          : 3
type                : virtual
virtual_parent      : "ls2p1"

<=== bound port is still ls2p1

Verified on ovn2.11-2.11.1-24.el7fdp.x86_64:

[root@dell-per740-12 bz1762341]# rpm -qa | grep -E "openvswitch|ovn"                                  
openvswitch2.11-2.11.0-35.el7fdp.x86_64
ovn2.11-2.11.1-24.el7fdp.x86_64
openvswitch-selinux-extra-policy-1.0-14.el7fdp.noarch
ovn2.11-central-2.11.1-24.el7fdp.x86_64
ovn2.11-host-2.11.1-24.el7fdp.x86_64

[root@dell-per740-12 bz1762341]# ip netns exec server0 python garp.py
.
Sent 1 packets.

<==== send garp on port ls2p1

[root@dell-per740-12 ~]# ovn-sbctl find port_binding logical_port=ls2-vir
_uuid               : 27ff7834-7a58-46cc-a3a9-50a0c7298242
chassis             : b5e6e487-3f57-4245-bd00-1397ee592ab5
datapath            : 7349f797-d0e2-40b1-b2ba-5e0623afd77b
encap               : []                                                                              
external_ids        : {}                                                                              
gateway_chassis     : []                                                                              
ha_chassis_group    : []                                                                              
logical_port        : "ls2-vir"                                                                       
mac                 : ["00:00:00:01:01:10 172.16.1.10"]
nat_addresses       : []                                                                              
options             : {virtual-ip="172.16.1.10", virtual-parents="ls2p1,ls2p2"}
parent_port         : []                                                                              
tag                 : []                                                                              
tunnel_key          : 3                                                                               
type                : virtual                                                                         
virtual_parent      : "ls2p1"

<=== port ls2p1 is bound

[root@dell-per740-12 bz1762341]# ip netns exec server2 python garp2.py                                
.
Sent 1 packets.

<==== send garp on port ls2p2

[root@dell-per740-12 ~]# ovn-sbctl find port_binding logical_port=ls2-vir
_uuid               : 27ff7834-7a58-46cc-a3a9-50a0c7298242
chassis             : b5e6e487-3f57-4245-bd00-1397ee592ab5
datapath            : 7349f797-d0e2-40b1-b2ba-5e0623afd77b
encap               : []                                                                              
external_ids        : {}                                                                              
gateway_chassis     : []                                                                              
ha_chassis_group    : []                                                                              
logical_port        : "ls2-vir"                                                                       
mac                 : ["00:00:00:01:01:10 172.16.1.10"]
nat_addresses       : []                                                                              
options             : {virtual-ip="172.16.1.10", virtual-parents="ls2p1,ls2p2"}
parent_port         : []                                                                              
tag                 : []                                                                              
tunnel_key          : 3                                                                               
type                : virtual                                                                         
virtual_parent      : "ls2p2"

<=== port ls2p2 is bound

Comment 4 Jianlin Shi 2019-12-20 02:59:43 UTC
also verified on rhel8 version:

[root@hp-dl380pg8-12 bz1762341]# ip netns exec server0 python garp.py
.                                                                                                     
Sent 1 packets.                                                                                       
[root@hp-dl380pg8-12 bz1762341]# ovn-sbctl find port_binding logical_port=ls2-vir
_uuid               : 01053304-6f8f-4bc1-abe6-63a256a1fc4b
chassis             : 01580c4a-383e-4525-9261-21b7bc28dee3
datapath            : 62a7353c-e429-4d1c-b1c7-6184fcf64812
encap               : []                                                                              
external_ids        : {}                                                                              
gateway_chassis     : []                                                                              
ha_chassis_group    : []                                                                              
logical_port        : "ls2-vir"                                                                       
mac                 : ["00:00:00:01:01:10 172.16.1.10"]
nat_addresses       : []                                                                              
options             : {virtual-ip="172.16.1.10", virtual-parents="ls2p1,ls2p2"}
parent_port         : []                                                                              
tag                 : []                                                                              
tunnel_key          : 3                                                                               
type                : virtual                                                                         
virtual_parent      : "ls2p1"                                                                         
[root@hp-dl380pg8-12 bz1762341]# ip netns exec server2 python garp2.py
.                                                                                                     
Sent 1 packets.                                                                                       
[root@hp-dl380pg8-12 bz1762341]# ovn-sbctl find port_binding logical_port=ls2-vir
_uuid               : 01053304-6f8f-4bc1-abe6-63a256a1fc4b
chassis             : 01580c4a-383e-4525-9261-21b7bc28dee3
datapath            : 62a7353c-e429-4d1c-b1c7-6184fcf64812
encap               : []                                                                              
external_ids        : {}                                                                              
gateway_chassis     : []                                                                              
ha_chassis_group    : []                                                                              
logical_port        : "ls2-vir"                                                                       
mac                 : ["00:00:00:01:01:10 172.16.1.10"]
nat_addresses       : []                                                                              
options             : {virtual-ip="172.16.1.10", virtual-parents="ls2p1,ls2p2"}
parent_port         : []                                                                              
tag                 : []                                                                              
tunnel_key          : 3                                                                               
type                : virtual                                                                         
virtual_parent      : "ls2p2"                                                                         
[root@hp-dl380pg8-12 bz1762341]# rpm -qa | grep -E "openvswitch|ovn"
kernel-kernel-networking-openvswitch-ovn-basic-1.0-14.noarch
openvswitch-selinux-extra-policy-1.0-19.el8fdp.noarch
ovn2.11-host-2.11.1-24.el8fdp.x86_64
ovn2.11-2.11.1-24.el8fdp.x86_64
kernel-kernel-networking-openvswitch-ovn-common-1.0-6.noarch
ovn2.11-central-2.11.1-24.el8fdp.x86_64
openvswitch2.11-2.11.0-35.el8fdp.x86_6

Comment 6 errata-xmlrpc 2020-01-21 17:02:44 UTC
Since the problem described in this bug report should be
resolved in a recent advisory, it has been closed with a
resolution of ERRATA.

For information on the advisory, and where to find the updated
files, follow the link below.

If the solution does not work for you, open a new bug report.

https://access.redhat.com/errata/RHBA-2020:0190