Bug 1857106

Summary: the first packet after mac_binding is destroyed won't be marked
Product: Red Hat Enterprise Linux Fast Datapath Reporter: Jianlin Shi <jishi>
Component: ovn2.13Assignee: lorenzo bianconi <lorenzo.bianconi>
Status: CLOSED ERRATA QA Contact: Jianlin Shi <jishi>
Severity: medium Docs Contact:
Priority: medium    
Version: FDP 20.ECC: ctrautma, fiezzi, jishi, lorenzo.bianconi, ralongi
Target Milestone: ---   
Target Release: ---   
Hardware: Unspecified   
OS: Linux   
Whiteboard:
Fixed In Version: ovn2.13-20.12.0-10.el7fdn Doc Type: If docs needed, set a value
Doc Text:
Story Points: ---
Clone Of: Environment:
Last Closed: 2021-03-15 14:36:02 UTC Type: Bug
Regression: --- Mount Type: ---
Documentation: --- CRM:
Verified Versions: Category: ---
oVirt Team: --- RHEL 7.3 requirements from Atomic Host:
Cloudforms Team: --- Target Upstream Version:
Embargoed:

Description Jianlin Shi 2020-07-15 07:14:29 UTC
Description of problem:
the first packet after mac_binding is destroyed won't be marked

Version-Release number of selected component (if applicable):
ovn2.13-20.06.1-2.el8fdp.x86_64

How reproducible:
Always

Steps to Reproduce:
systemctl start openvswitch                                                
systemctl start ovn-northd    
ovn-nbctl set-connection ptcp:6641   
ovn-sbctl set-connection ptcp:6642                   
ovs-vsctl set open . external_ids:system-id=hv1 external_ids:ovn-remote=tcp:20.0.111.25:6642 external_ids:ovn-encap-type=geneve external_ids:ovn-encap-ip=20.0.111.25
systemctl restart ovn-controller
ip netns add server0                                                       
ip link add veth0_s0 netns server0 type veth peer name veth0_s0_p
ip netns exec server0 ip link set lo up      
ip netns exec server0 ip link set veth0_s0 up          
ip netns exec server0 ip link set veth0_s0 address 00:00:00:01:01:02
ip netns exec server0 ip addr add 192.168.1.1/24 dev veth0_s0
ip netns exec server0 ip -6 addr add 3001::1/64 dev veth0_s0     
ip netns exec server0 ip route add default via 192.168.1.254 dev veth0_s0
ip netns exec server0 ip -6 route add default via 3001::a dev veth0_s0
ovs-vsctl add-port br-int veth0_s0_p                                
ip link set veth0_s0_p up                                    
                                                            
ovs-vsctl set interface veth0_s0_p external_ids:iface-id=ls1p1           
                                                                      
ovn-nbctl ls-add ls1                                               
ovn-nbctl lsp-add ls1 ls1p1         
ovn-nbctl lsp-set-addresses ls1p1 00:00:00:01:01:02          
ovn-nbctl lr-add lr1                                     
ovn-nbctl lrp-add lr1 lr1-ls1 00:00:00:00:00:01 192.168.1.254/24 3001::a/64
ovn-nbctl lsp-add ls1 ls1-lr1                                                    
ovn-nbctl lsp-set-type ls1-lr1 router                                            
ovn-nbctl lsp-set-options ls1-lr1 router-port=lr1-ls1               
ovn-nbctl lsp-set-addresses ls1-lr1 "00:00:00:00:00:01 192.168.1.254/24 3001::a/64"
                                                                                                     
ovn-nbctl ls-add ls2                                                       
ovn-nbctl lsp-add ls2 ls2-lr1 
ovn-nbctl lsp-set-type ls2-lr1 router      
ovn-nbctl lsp-set-options ls2-lr1 router-port=lr1-ls2
ovn-nbctl lsp-set-addresses ls2-lr1 "00:00:00:00:00:02 192.168.0.254/24 3000::a/64"

ovn-nbctl lrp-add lr1 lr1-ls2 00:00:00:00:00:02 192.168.0.254/24 3000::a/64
                                         
ovn-nbctl lsp-add ls2 ls2p1                  
ovn-nbctl lsp-set-addresses ls2p1 00:00:00:02:01:01    
                                                            
ip netns add server1                                                                                
ip link add veth0_s1 netns server1 type veth peer name veth0_s1_p
ip netns exec server1 ip link set lo up 
ip netns exec server1 ip link set veth0_s1 up
ip netns exec server1 ip link set veth0_s1 address 00:00:00:02:01:01
ip netns exec server1 ip addr add 192.168.0.1/24 dev veth0_s1
ip netns exec server1 ip -6 addr add 3000::1/64 dev veth0_s1
ip netns exec server1 ip route add default via 192.168.0.254 dev veth0_s1
ip netns exec server1 ip -6 route add default via 3000::a dev veth0_s1

ovs-vsctl add-port br-int veth0_s1_p                                               
ip link set veth0_s1_p up                                                                            
                                                                           
ovs-vsctl set interface veth0_s1_p external_ids:iface-id=ls2p1 
                                                                                 
ovs-vsctl add-br br-phys                                                         
ovs-vsctl set open . external-ids:ovn-bridge-mappings=public:br-phys               
                                                              
ovn-nbctl ls-add public                                                                              
ovn-nbctl lrp-add lr1 lr1_p 00:00:20:20:12:13 172.168.0.100/24 1111::100/64
ovn-nbctl lsp-add public p_lr1                
ovn-nbctl lsp-set-type p_lr1 router                    
ovn-nbctl lsp-set-addresses p_lr1 router                    
ovn-nbctl lsp-set-options p_lr1 router-port=lr1_p                                                   
                                                                 
ovn-nbctl lsp-add public ln_public         
ovn-nbctl lsp-set-type ln_public localnet    
ovn-nbctl lsp-set-addresses ln_public unknown                       
ovn-nbctl lsp-set-options ln_public network_name=public      
                                                            
ip netns add ext                                                                                    
ip link add veth0_e netns ext type veth peer name veth0_e_p           
ovs-vsctl add-port br-phys veth0_e_p                               
ip link set veth0_e_p up            
                                                             
ip netns exec ext ip link set veth0_e up                 
ip netns exec ext ip addr add 172.168.0.1/24 dev veth0_e      
ip netns exec ext ip -6 addr add 1111::1/64 dev veth0_e                          
ip netns exec ext ip route add default via 172.168.0.100 dev veth0_e             
ip netns exec ext ip -6 route add default via 1111::100 dev veth0_e 

ovn-nbctl lr-policy-add lr1 2000 "ip4.src==192.168.0.1" allow
ovn-nbctl lr-policy-add lr1 1000 "ip6.src==3001::1" allow

pol1=$(ovn-nbctl --bare --columns _uuid find logical_router_policy priority=2000)
pol2=$(ovn-nbctl --bare --columns _uuid find logical_router_policy priority=1000)

ovn-nbctl set logical_router_policy $pol1 options:pkt_mark=100
ovs-ofctl --protocols=OpenFlow13 add-flow br-phys "table=0, priority=100, pkt_mark=0x64 actions=drop"
ovn-nbctl --wait=hv sync

ip netns exec server1 ping 172.168.0.1 -c 1
ovs-ofctl dump-flows br-int table=19
ovs-ofctl dump-flows br-phys table=0

ip netns exec server1 ping 172.168.0.1 -c 1
ovs-ofctl dump-flows br-int table=19
ovs-ofctl dump-flows br-phys table=0

ovn-nbctl set logical_router_policy $pol2 options:pkt_mark=5
ovs-ofctl --protocols=OpenFlow13 add-flow br-phys "table=0, priority=100, pkt_mark=0x5 actions=drop"
ovn-nbctl --wait=hv sync
ip netns exec server0 ping6 1111::1 -c 1
ovs-ofctl dump-flows br-int table=19
ovs-ofctl dump-flows br-phys table=0

ip netns exec server0 ping6 1111::1 -c 1
ovs-ofctl dump-flows br-int table=19
ovs-ofctl dump-flows br-phys table=0

Actual results:
the first ping would pass

Expected results:


Additional info:
the first and following ping should fail

+ ip netns exec server1 ping 172.168.0.1 -c 1
PING 172.168.0.1 (172.168.0.1) 56(84) bytes of data.
64 bytes from 172.168.0.1: icmp_seq=1 ttl=63 time=4.92 ms
 
--- 172.168.0.1 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 4.915/4.915/4.915/0.000 ms
+ ovs-ofctl dump-flows br-int table=19
 cookie=0x4d648ee2, duration=0.114s, table=19, n_packets=1, n_bytes=98, priority=2000,ip,metadata=0x2,nw_src=192.168.0.1 actions=load:0x64->NXM_NX_PKT_MARK[],resubmit(,20)
 cookie=0x4cf0e735, duration=0.147s, table=19, n_packets=0, n_bytes=0, priority=1000,ipv6,metadata=0x2,ipv6_src=3001::1 actions=resubmit(,20)
 cookie=0x6e12013e, duration=0.846s, table=19, n_packets=1, n_bytes=98, priority=0,metadata=0x2 actions=resubmit(,20)
 cookie=0x2562052e, duration=0.843s, table=19, n_packets=7, n_bytes=494, priority=0,metadata=0x3 actions=resubmit(,20)
 cookie=0x7d636781, duration=0.842s, table=19, n_packets=1, n_bytes=86, priority=0,metadata=0x1 actions=resubmit(,20)
 cookie=0x3b96cdee, duration=0.471s, table=19, n_packets=7, n_bytes=570, priority=0,metadata=0x4 actions=resubmit(,20)
+ ovs-ofctl dump-flows br-phys table=0
 cookie=0x0, duration=0.080s, table=0, n_packets=0, n_bytes=0, priority=100,pkt_mark=0x64 actions=drop

<=== not take effect, packet not marked
 
 cookie=0x0, duration=0.584s, table=0, n_packets=7, n_bytes=570, priority=0 actions=NORMAL
+ ip netns exec server1 ping 172.168.0.1 -c 1
PING 172.168.0.1 (172.168.0.1) 56(84) bytes of data.
 
--- 172.168.0.1 ping statistics ---
1 packets transmitted, 0 received, 100% packet loss, time 0ms
 
+ ovs-ofctl dump-flows br-int table=19
 cookie=0x4d648ee2, duration=10.161s, table=19, n_packets=2, n_bytes=196, priority=2000,ip,metadata=0x2,nw_src=192.168.0.1 actions=load:0x64->NXM_NX_PKT_MARK[],resubmit(,20)
 cookie=0x4cf0e735, duration=10.194s, table=19, n_packets=0, n_bytes=0, priority=1000,ipv6,metadata=0x2,ipv6_src=3001::1 actions=resubmit(,20)
 cookie=0x6e12013e, duration=10.893s, table=19, n_packets=1, n_bytes=98, priority=0,metadata=0x2 actions=resubmit(,20)
 cookie=0x2562052e, duration=10.890s, table=19, n_packets=13, n_bytes=1062, priority=0,metadata=0x3 actions=resubmit(,20)
 cookie=0x7d636781, duration=10.889s, table=19, n_packets=5, n_bytes=446, priority=0,metadata=0x1 actions=resubmit(,20)
 cookie=0x3b96cdee, duration=10.518s, table=19, n_packets=17, n_bytes=1394, priority=0,metadata=0x4 actions=resubmit(,20)
+ ovs-ofctl dump-flows br-phys table=0
 cookie=0x0, duration=10.126s, table=0, n_packets=1, n_bytes=98, priority=100,pkt_mark=0x64 actions=drop
 
<=== take effect, packet marked
 
 cookie=0x0, duration=10.630s, table=0, n_packets=16, n_bytes=1296, priority=0 actions=NORMAL
+ ovn-nbctl set logical_router_policy 16cf38cf-8fba-47ea-9991-2cf5833a6e52 options:pkt_mark=5
+ ovs-ofctl --protocols=OpenFlow13 add-flow br-phys 'table=0, priority=100, pkt_mark=0x5 actions=drop'
+ ovn-nbctl --wait=hv sync
+ ip netns exec server0 ping6 1111::1 -c 1
PING 1111::1(1111::1) 56 data bytes
64 bytes from 1111::1: icmp_seq=1 ttl=63 time=7.50 ms
 
--- 1111::1 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 7.500/7.500/7.500/0.000 ms
+ ovs-ofctl dump-flows br-int table=19
 cookie=0x4d648ee2, duration=10.301s, table=19, n_packets=2, n_bytes=196, priority=2000,ip,metadata=0x2,nw_src=192.168.0.1 actions=load:0x64->NXM_NX_PKT_MARK[],resubmit(,20)
 cookie=0xf6f121d1, duration=0.112s, table=19, n_packets=1, n_bytes=118, priority=1000,ipv6,metadata=0x2,ipv6_src=3001::1 actions=load:0x5->NXM_NX_PKT_MARK[],resubmit(,20)
 cookie=0x6e12013e, duration=11.033s, table=19, n_packets=2, n_bytes=216, priority=0,metadata=0x2 actions=resubmit(,20)
 cookie=0x2562052e, duration=11.030s, table=19, n_packets=13, n_bytes=1062, priority=0,metadata=0x3 actions=resubmit(,20)
 cookie=0x7d636781, duration=11.030s, table=19, n_packets=11, n_bytes=1026, priority=0,metadata=0x1 actions=resubmit(,20)
 cookie=0x3b96cdee, duration=10.659s, table=19, n_packets=25, n_bytes=2146, priority=0,metadata=0x4 actions=resubmit(,20)
+ ovs-ofctl dump-flows br-phys table=0
 cookie=0x0, duration=10.266s, table=0, n_packets=1, n_bytes=98, priority=100,pkt_mark=0x64 actions=drop
 cookie=0x0, duration=0.077s, table=0, n_packets=0, n_bytes=0, priority=100,pkt_mark=0x5 actions=drop
 
<=== not take effect, packet not marked
 
 cookie=0x0, duration=10.770s, table=0, n_packets=24, n_bytes=2048, priority=0 actions=NORMAL
+ ip netns exec server0 ping6 1111::1 -c 1
PING 1111::1(1111::1) 56 data bytes
 
--- 1111::1 ping statistics ---
1 packets transmitted, 0 received, 100% packet loss, time 0ms
 
+ ovs-ofctl dump-flows br-int table=19
 cookie=0x4d648ee2, duration=20.348s, table=19, n_packets=2, n_bytes=196, priority=2000,ip,metadata=0x2,nw_src=192.168.0.1 actions=load:0x64->NXM_NX_PKT_MARK[],resubmit(,20)
 cookie=0xf6f121d1, duration=10.159s, table=19, n_packets=2, n_bytes=236, priority=1000,ipv6,metadata=0x2,ipv6_src=3001::1 actions=load:0x5->NXM_NX_PKT_MARK[],resubmit(,20)
 cookie=0x6e12013e, duration=21.080s, table=19, n_packets=2, n_bytes=216, priority=0,metadata=0x2 actions=resubmit(,20)
 cookie=0x2562052e, duration=21.077s, table=19, n_packets=14, n_bytes=1132, priority=0,metadata=0x3 actions=resubmit(,20)
 cookie=0x7d636781, duration=21.076s, table=19, n_packets=15, n_bytes=1386, priority=0,metadata=0x1 actions=resubmit(,20)
 cookie=0x3b96cdee, duration=20.705s, table=19, n_packets=29, n_bytes=2506, priority=0,metadata=0x4 actions=resubmit(,20)
+ ovs-ofctl dump-flows br-phys table=0
 cookie=0x0, duration=20.313s, table=0, n_packets=1, n_bytes=98, priority=100,pkt_mark=0x64 actions=drop
 cookie=0x0, duration=10.124s, table=0, n_packets=1, n_bytes=118, priority=100,pkt_mark=0x5 actions=drop
 
<==== take effect, packet marked
 cookie=0x0, duration=20.817s, table=0, n_packets=27, n_bytes=2290, priority=0 actions=NORMAL
 
[root@hp-dl380pg8-12 bz1828933]# rpm -qa | grep -E "openvswitch|ovn"
openvswitch2.13-2.13.0-41.el8fdb.x86_64
ovn2.13-central-20.06.1-2.el8fdp.x86_64
openvswitch-selinux-extra-policy-1.0-23.el8fdp.noarch
ovn2.13-host-20.06.1-2.el8fdp.x86_64
ovn2.13-20.06.1-2.el8fdp.x86_64


if destroy mac_binding for 172.168.0.1 with ovn-nbctl destroy mac_binding $uuid, the first ping after destroy would also pass, which means the packet is not marked

Comment 2 Jianlin Shi 2021-01-22 09:45:50 UTC
(In reply to lorenzo bianconi from comment #1)
> - upstream fix:
> http://patchwork.ozlabs.org/project/ovn/patch/
> 9bc8046bd39414fc2125372091e69bb9fa051c27.1611156958.git.lorenzo.
> bianconi/

the test passed on the build compiled by lorenzo on this commit.

Comment 5 Jianlin Shi 2021-02-09 02:34:12 UTC
Verified on ovn2.13-20.12.0-15:

+ ovn-nbctl --wait=hv sync                                                                                                                                                                                 
+ ip netns exec server1 ping 172.168.0.1 -c 1                                                
PING 172.168.0.1 (172.168.0.1) 56(84) bytes of data.                  
                                   
--- 172.168.0.1 ping statistics ---                                          
1 packets transmitted, 0 received, 100% packet loss, time 0ms

<=== the first packet dropped

+ ovs-ofctl dump-flows br-phys table=0                                                                                                                                                                     
 cookie=0x0, duration=10.074s, table=0, n_packets=1, n_bytes=98, priority=100,pkt_mark=0x64 actions=drop
 cookie=0x0, duration=10.402s, table=0, n_packets=11, n_bytes=926, priority=0 actions=NORMAL   

<=== marked
                                                                                                            
+ ip netns exec server1 ping 172.168.0.1 -c 1                                                                                                                                                              
PING 172.168.0.1 (172.168.0.1) 56(84) bytes of data.                                         
                                                                              
--- 172.168.0.1 ping statistics ---     
1 packets transmitted, 0 received, 100% packet loss, time 0ms
+ ovs-ofctl dump-flows br-phys table=0
 cookie=0x0, duration=20.118s, table=0, n_packets=2, n_bytes=196, priority=100,pkt_mark=0x64 actions=drop
 cookie=0x0, duration=20.446s, table=0, n_packets=12, n_bytes=996, priority=0 actions=NORMAL 
+ ovn-nbctl set logical_router_policy ce1225ad-fb94-469b-a4a1-8788c04e1e4d options:pkt_mark=5
+ ovs-ofctl --protocols=OpenFlow13 add-flow br-phys 'table=0, priority=100, pkt_mark=0x5 actions=drop'
+ ovn-nbctl --wait=hv sync
+ ip netns exec server0 ping6 1111::1 -c 1
PING 1111::1(1111::1) 56 data bytes

--- 1111::1 ping statistics ---
1 packets transmitted, 0 received, 100% packet loss, time 0ms
+ ovs-ofctl dump-flows br-phys table=0                                                      
 cookie=0x0, duration=30.222s, table=0, n_packets=2, n_bytes=196, priority=100,pkt_mark=0x64 actions=drop
 cookie=0x0, duration=10.064s, table=0, n_packets=1, n_bytes=118, priority=100,pkt_mark=0x5 actions=drop
 cookie=0x0, duration=30.550s, table=0, n_packets=17, n_bytes=1410, priority=0 actions=NORMAL
+ ip netns exec server0 ping6 1111::1 -c 1
PING 1111::1(1111::1) 56 data bytes

--- 1111::1 ping statistics ---
1 packets transmitted, 0 received, 100% packet loss, time 0ms

+ ovs-ofctl dump-flows br-phys table=0
 cookie=0x0, duration=40.268s, table=0, n_packets=2, n_bytes=196, priority=100,pkt_mark=0x64 actions=drop
 cookie=0x0, duration=20.110s, table=0, n_packets=2, n_bytes=236, priority=100,pkt_mark=0x5 actions=drop
 cookie=0x0, duration=40.596s, table=0, n_packets=17, n_bytes=1410, priority=0 actions=NORMAL
[root@wsfd-advnetlab21 bz1857106]# rpm -qa | grep -E "openvswitch2.13|ovn2.13" 
ovn2.13-central-20.12.0-15.el8fdp.x86_64
openvswitch2.13-2.13.0-93.el8fdp.x86_64
ovn2.13-20.12.0-15.el8fdp.x86_64
ovn2.13-host-20.12.0-15.el8fdp.x86_64

Comment 7 errata-xmlrpc 2021-03-15 14:36:02 UTC
Since the problem described in this bug report should be
resolved in a recent advisory, it has been closed with a
resolution of ERRATA.

For information on the advisory (ovn2.13 bug fix and enhancement update), and where to find the updated
files, follow the link below.

If the solution does not work for you, open a new bug report.

https://access.redhat.com/errata/RHBA-2021:0836