Bug 1841526

Summary: [OVN][RFE] Add support for VXLAN tunnels for interchassis communication
Product: Red Hat Enterprise Linux Fast Datapath Reporter: Ihar Hrachyshka <ihrachys>
Component: ovn2.13Assignee: Ihar Hrachyshka <ihrachys>
Status: CLOSED ERRATA QA Contact: Jianlin Shi <jishi>
Severity: unspecified Docs Contact:
Priority: unspecified    
Version: FDP 20.ACC: avishnoi, ctrautma, jishi, nusiddiq, ralongi
Target Milestone: ---Keywords: FutureFeature
Target Release: ---   
Hardware: Unspecified   
OS: Unspecified   
Whiteboard:
Fixed In Version: Doc Type: If docs needed, set a value
Doc Text:
Story Points: ---
Clone Of:
: 1881697 (view as bug list) Environment:
Last Closed: 2020-10-27 09:49:12 UTC Type: Bug
Regression: --- Mount Type: ---
Documentation: --- CRM:
Verified Versions: Category: ---
oVirt Team: --- RHEL 7.3 requirements from Atomic Host:
Cloudforms Team: --- Target Upstream Version:
Embargoed:
Bug Depends On:    
Bug Blocks: 1881697, 1881704, 1916091, 1916092, 2128735    

Description Ihar Hrachyshka 2020-05-29 11:42:50 UTC 
Description of problem: right now VXLAN is not supported by OVN for regular inter-chassis communication. There is VXLAN support for "vtep" ovsdb schema compliant external switches (aka ramp switches). This RFE is to extend core OVN to support VXLAN for regular in-cluster traffic.

Initial upstream discussion: https://www.mail-archive.com/ovs-discuss@openvswitch.org/msg06771.html
Demo implementation: https://patchwork.ozlabs.org/project/openvswitch/patch/20200320050711.247351-1-ihrachys@redhat.com/

Related links:
https://blog.russellbryant.net/2017/05/30/ovn-geneve-vs-vxlan-does-it-matter/
Comment 2 Ihar Hrachyshka 2020-09-22 20:08:25 UTC 
It's to be included in 20.09 upstream OVN release. (Now waiting for final tarballs.)
Comment 6 Jianlin Shi 2020-10-10 06:22:29 UTC 
tested with following script:

server:
systemctl start openvswitch                                              
systemctl start ovn-northd
ovn-nbctl set-connection ptcp:6641                                         
ovn-sbctl set-connection ptcp:6642
ovs-vsctl set open . external_ids:system-id=hv1 external_ids:ovn-remote=tcp:20.0.149.25:6642 external_ids:ovn-encap-type=vxlan external_ids:ovn-encap-ip=20.0.149.25
systemctl restart ovn-controller     
                                                     
ip netns add server0
ip link add veth0_s0 netns server0 type veth peer name veth0_s0_p          
ip netns exec server0 ip link set lo up
ip netns exec server0 ip link set veth0_s0 up
ip netns exec server0 ip link set veth0_s0 address 00:00:00:01:01:02
ip netns exec server0 ip addr add 192.168.1.1/24 dev veth0_s0                
ip netns exec server0 ip -6 addr add 2001::1/64 dev veth0_s0
ip netns exec server0 ip route add default via 192.168.1.254 dev veth0_s0
ip netns exec server0 ip -6 route add default via 2001::a dev veth0_s0
ovs-vsctl add-port br-int veth0_s0_p
ip link set veth0_s0_p up                                                
ovs-vsctl set interface veth0_s0_p external_ids:iface-id=ls1p1
ovn-nbctl ls-add ls1       
ovn-nbctl lsp-add ls1 ls1p1                                              
#ovn-nbctl lsp-set-addresses ls1p1 "00:00:00:01:01:02 2001::1 192.168.1.1"
ovn-nbctl lsp-set-addresses ls1p1 "00:00:00:01:01:02 192.168.1.1 2001::1"
ovn-nbctl lsp-add ls1 ls1p2                                      
ovn-nbctl lsp-set-addresses ls1p2 "00:00:00:01:02:02 192.168.1.2 2001::2"
ovn-nbctl lr-add lr1                         
ovn-nbctl lrp-add lr1 lr1-ls1 00:00:00:00:00:01 192.168.1.254/24 2001::a/64
ovn-nbctl lsp-add ls1 ls1-lr1                                
ovn-nbctl lsp-set-addresses ls1-lr1 "00:00:00:00:00:01 192.168.1.254 2001::a"
ovn-nbctl lsp-set-type ls1-lr1 router                                    
ovn-nbctl lsp-set-options ls1-lr1 router-port=lr1-ls1                 

ovn-nbctl lrp-add lr1 lr1-ls2 00:00:00:00:00:02 192.168.2.254/24 2002::a/64
                         
ovn-nbctl ls-add ls2                                          
ovn-nbctl lsp-add ls2 ls2-lr1
ovn-nbctl lsp-set-addresses ls2-lr1 "00:00:00:00:00:02 192.168.2.254 2002::a"
ovn-nbctl lsp-set-type ls2-lr1 router
ovn-nbctl lsp-set-options ls2-lr1 router-port=lr1-ls2

ovn-nbctl lsp-add ls2 ls2p1
ovn-nbctl lsp-set-addresses ls2p1 "00:00:00:02:01:02 192.168.2.1 2002::1"

ovn-nbctl lsp-add ls1 ls1p3
ovn-nbctl lsp-set-addresses ls1p3 "00:00:00:01:03:02 192.168.1.3 2001::3"

ip netns add server2
ip link add veth0_s2 netns server2 type veth peer name veth0_s2_p
ip netns exec server2 ip link set lo up
ip netns exec server2 ip link set veth0_s2 up
ip netns exec server2 ip link set veth0_s2 address 00:00:00:01:03:02
ip netns exec server2 ip addr add 192.168.1.3/24 dev veth0_s2
ip netns exec server2 ip -6 addr add 2001::3/64 dev veth0_s2
ip netns exec server2 ip route add default via 192.168.1.254 dev veth0_s2
ip netns exec server2 ip -6 route add default via 2001::a dev veth0_s2

ovs-vsctl add-port br-int veth0_s2_p
ip link set veth0_s2_p up
ovs-vsctl set interface veth0_s2_p external_ids:iface-id=ls1p3

ovn-nbctl lb-add lb0 192.168.1.100 192.168.1.1,192.168.1.2
ovn-nbctl ls-lb-add ls2 lb0

client:

systemctl start openvswitch                                                        
ovs-vsctl set open . external_ids:system-id=hv0 external_ids:ovn-remote=tcp:20.0.149.25:6642 external_ids:ovn-encap-type=vxlan external_ids:ovn-encap-ip=20.0.149.26
systemctl restart ovn-controller

ip netns add server1                                        
ip link add veth0_s1 netns server1 type veth peer name veth0_s1_p        
ip netns exec server1 ip link set lo up                                                                                                                            
ip netns exec server1 ip link set veth0_s1 up
ip netns exec server1 ip link set veth0_s1 address 00:00:00:01:02:02
ip netns exec server1 ip addr add 192.168.1.2/24 dev veth0_s1
ip netns exec server1 ip -6 addr add 2001::2/64 dev veth0_s1  
ip netns exec server1 ip route add default via 192.168.1.254 dev veth0_s1
ip netns exec server1 ip -6 route add default via 2001::a dev veth0_s1
                                             
ovs-vsctl add-port br-int veth0_s1_p                                
ip link set veth0_s1_p up                                    
ovs-vsctl set interface veth0_s1_p external_ids:iface-id=ls1p2
                                                                         
ip netns add client0                                                  
ip link add veth0_c0 netns client0 type veth peer name veth0_c0_p
ip netns exec client0 ip link set lo up                                                                                                                                                      
ip netns exec client0 ip link set veth0_c0 up                                                                                                                                                
ip netns exec client0 ip link set veth0_c0 address 00:00:00:02:01:02               
ip netns exec client0 ip addr add 192.168.2.1/24 dev veth0_c0                      
ip netns exec client0 ip -6 addr add 2002::1/64 dev veth0_c0
ip netns exec client0 ip route add default via 192.168.2.254 dev veth0_c0
ip netns exec client0 ip -6 route add default via 2002::a dev veth0_c0
                                             
ovs-vsctl add-port br-int veth0_c0_p                                
ip link set veth0_c0_p up                                    
ovs-vsctl set interface veth0_c0_p external_ids:iface-id=ls2p1


verified on ovn2.13-20.09.0-1.el7fdp.x86_64:

[root@wsfd-advnetlab16 bz1841526]# rpm -qa | grep -E "openvswitch|ovn"
kernel-kernel-networking-openvswitch-ovn-common-1.0-9.noarch
ovn2.13-20.09.0-1.el7fdp.x86_64
kernel-kernel-networking-openvswitch-ovn-basic-1.0-30.noarch
openvswitch-selinux-extra-policy-1.0-15.el7fdp.noarch
openvswitch2.13-2.13.0-51.el7fdp.x86_64
ovn2.13-central-20.09.0-1.el7fdp.x86_64
ovn2.13-host-20.09.0-1.el7fdp.x86_64

[root@wsfd-advnetlab16 bz1841526]# ip netns exec server0 ping 192.168.2.1 -c 1
PING 192.168.2.1 (192.168.2.1) 56(84) bytes of data.
64 bytes from 192.168.2.1: icmp_seq=1 ttl=63 time=2.19 ms

--- 192.168.2.1 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 2.198/2.198/2.198/0.000 ms
[root@wsfd-advnetlab16 bz1841526]# ip netns exec server0 ping 192.168.1.2 -c 1
PING 192.168.1.2 (192.168.1.2) 56(84) bytes of data.
64 bytes from 192.168.1.2: icmp_seq=1 ttl=64 time=1.54 ms

--- 192.168.1.2 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 1.548/1.548/1.548/0.000 ms
[root@wsfd-advnetlab16 bz1841526]# ovn-sbctl show
Chassis hv1
    hostname: wsfd-advnetlab16.anl.lab.eng.bos.redhat.com
    Encap vxlan
        ip: "20.0.149.25"
        options: {csum="true"}
    Port_Binding ls1p3
    Port_Binding ls1p1
Chassis hv0
    hostname: wsfd-advnetlab17.anl.lab.eng.bos.redhat.com
    Encap vxlan
        ip: "20.0.149.26"
        options: {csum="true"}
    Port_Binding ls1p2
    Port_Binding ls2p1
[root@wsfd-advnetlab16 bz1841526]# ovs-vsctl show
a5de8d5a-2d27-47d6-85e4-60f0c21240ce
    Bridge br-int
        fail_mode: secure
        Port br-int
            Interface br-int
                type: internal
        Port veth0_s0_p
            Interface veth0_s0_p
        Port veth0_s2_p
            Interface veth0_s2_p
        Port ovn-hv0-0
            Interface ovn-hv0-0
                type: vxlan
                options: {csum="true", key=flow, remote_ip="20.0.149.26"}
    ovs_version: "2.13.2"

[root@wsfd-advnetlab17 bz1841526]# ip netns exec client0 ping 192.168.1.100 -c 1
PING 192.168.1.100 (192.168.1.100) 56(84) bytes of data.
64 bytes from 192.168.1.100: icmp_seq=1 ttl=63 time=4.67 ms

--- 192.168.1.100 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 4.672/4.672/4.672/0.000 ms
Comment 7 Jianlin Shi 2020-10-10 06:33:26 UTC 
also verified on rhel8 version:

[root@wsfd-advnetlab18 bz1841526]# rpm -qa | grep -E "openvswitch|ovn"
openvswitch-selinux-extra-policy-1.0-23.el8fdp.noarch
ovn2.13-20.09.0-1.el8fdp.x86_64
openvswitch2.13-2.13.0-60.el8fdp.x86_64
ovn2.13-central-20.09.0-1.el8fdp.x86_64
ovn2.13-host-20.09.0-1.el8fdp.x86_64

[root@wsfd-advnetlab18 bz1841526]# ovs-vsctl show
88ba2366-4e8e-46e9-aec3-9e5475197491
    Bridge br-int
        fail_mode: secure
        Port br-int
            Interface br-int
                type: internal
        Port ovn-hv0-0
            Interface ovn-hv0-0
                type: vxlan
                options: {csum="true", key=flow, remote_ip="1.1.23.26"}
        Port veth0_s2_p
            Interface veth0_s2_p
        Port veth0_s0_p
            Interface veth0_s0_p
    ovs_version: "2.13.2"

[root@wsfd-advnetlab18 bz1841526]# ip netns exec server0 ping 192.168.1.2 -c 1
PING 192.168.1.2 (192.168.1.2) 56(84) bytes of data.
64 bytes from 192.168.1.2: icmp_seq=1 ttl=64 time=2.52 ms

--- 192.168.1.2 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 2.522/2.522/2.522/0.000 ms
[root@wsfd-advnetlab18 bz1841526]# ip netns exec server0 ping 192.168.2.1 -c 1
PING 192.168.2.1 (192.168.2.1) 56(84) bytes of data.
64 bytes from 192.168.2.1: icmp_seq=1 ttl=63 time=2.94 ms

--- 192.168.2.1 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 2.941/2.941/2.941/0.000 ms

[root@wsfd-advnetlab18 bz1841526]# ovn-sbctl show
Chassis hv0
    hostname: wsfd-advnetlab19.anl.lab.eng.bos.redhat.com
    Encap vxlan
        ip: "1.1.23.26"
        options: {csum="true"}
    Port_Binding ls2p1
    Port_Binding ls1p2
Chassis hv1
    hostname: wsfd-advnetlab18.anl.lab.eng.bos.redhat.com
    Encap vxlan
        ip: "1.1.23.25"
        options: {csum="true"}
    Port_Binding ls1p3
    Port_Binding ls1p1

client:

[root@wsfd-advnetlab19 bz1841526]# ip netns exec client0 ping 192.168.1.100 -c 1
PING 192.168.1.100 (192.168.1.100) 56(84) bytes of data.
64 bytes from 192.168.1.100: icmp_seq=1 ttl=63 time=2.27 ms

--- 192.168.1.100 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 2.273/2.273/2.273/0.000 ms
[root@wsfd-advnetlab19 bz1841526]# ovs-vsctl show
6e2432d6-57e5-4743-9afd-0f2d15b00827
    Bridge br-int
        fail_mode: secure
        Port veth0_s1_p
            Interface veth0_s1_p
        Port veth0_c0_p
            Interface veth0_c0_p
        Port ovn-hv1-0
            Interface ovn-hv1-0
                type: vxlan
                options: {csum="true", key=flow, remote_ip="1.1.23.25"}
        Port br-int
            Interface br-int
                type: internal
    ovs_version: "2.13.2"
Comment 9 errata-xmlrpc 2020-10-27 09:49:12 UTC 
Since the problem described in this bug report should be
resolved in a recent advisory, it has been closed with a
resolution of ERRATA.

For information on the advisory (ovn2.13 bug fix and enhancement update), and where to find the updated
files, follow the link below.

If the solution does not work for you, open a new bug report.

https://access.redhat.com/errata/RHBA-2020:4356