The FDP team is no longer accepting new bugs in Bugzilla. Please report your issues under FDP project in Jira. Thanks.
Bug 2062580 - ping to dnat failed intermittently in gateway router mode
Summary: ping to dnat failed intermittently in gateway router mode
Keywords:
Status: CLOSED CURRENTRELEASE
Alias: None
Product: Red Hat Enterprise Linux Fast Datapath
Classification: Red Hat
Component: ovn-2021
Version: FDP 22.B
Hardware: Unspecified
OS: Unspecified
medium
medium
Target Milestone: ---
: ---
Assignee: Ales Musil
QA Contact: Jianlin Shi
URL:
Whiteboard:
Depends On:
Blocks:
TreeView+ depends on / blocked
 
Reported: 2022-03-10 08:00 UTC by Jianlin Shi
Modified: 2023-03-13 07:23 UTC (History)
5 users (show)

Fixed In Version:
Doc Type: If docs needed, set a value
Doc Text:
Clone Of:
Environment:
Last Closed: 2023-03-13 07:23:31 UTC
Target Upstream Version:
Embargoed:


Attachments (Terms of Use)


Links
System ID Private Priority Status Summary Last Updated
Red Hat Issue Tracker FD-1827 0 None None None 2022-03-10 08:07:06 UTC

Description Jianlin Shi 2022-03-10 08:00:46 UTC
Description of problem:
ping to dnat failed intermittently in gateway router mode 

Version-Release number of selected component (if applicable):
ovn-2021-21.12.0-30
openvswitch2.15-2.15.0-81.el8fdp.x86_64

How reproducible:
Always

Steps to Reproduce:
1. we need 2 machines, and machine1 can ssh to machine2 without password
2. machine1 has address 1.1.5.25, machine2 has 1.1.5.26
3. machine1 has another nic which is connected to another nic on machine2
4. run following script on machine1

ip_subnet=5

mac_veth2_hv0="00:00:00:01:00:02"                                                       
mac_veth2_hv1="00:00:00:01:01:02"                                                       
ip4_veth2_hv0="172.16.0.1"                                                              
ip6_veth2_hv0="2000::1"                                                                 
ip4_veth2_hv1="172.16.1.1"                                                              
ip6_veth2_hv1="2001::1"                                                                 
                                                                                        
ip4_external1_hv1="192.$ip_subnet.1.1"                                                  
ip6_external1_hv1="3000:$ip_subnet::1"                                                  
ip4_external2_hv1="192.$ip_subnet.1.2"                                                  
ip6_external2_hv1="3000:$ip_subnet::2"

# second card on machine 1
nic_test2_ser=enp5s0f1
# second card on machine 2
nic_test2_cli=ens3f1
# name of machine 2
client1="dell-per740-42.rhts.eng.pek2.redhat.com"

systemctl start openvswitch
systemctl start ovn-northd
ovn-nbctl set-connection ptcp:6641
ovn-sbctl set-connection ptcp:6642
ovs-vsctl set open . external_ids:system-id=hv1 external_ids:ovn-remote=tcp:1.1.5.25:6642 external_ids:ovn-encap-type=geneve external_ids:ovn-encap-ip=1.1.5.25
systemctl restart ovn-controller

ssh -q $client1 systemctl start openvswitch
ssh -q $client1 "ovs-vsctl set open . external_ids:system-id=hv0 external_ids:ovn-remote=tcp:1.1.5.25:6642 external_ids:ovn-encap-type=geneve external_ids:ovn-encap-ip=1.1.5.26"
ssh -q $client1 systemctl restart ovn-controller
ssh -q $client1 "ovs-vsctl add-br br-nat"
ssh -q $client1 "ovs-vsctl set open . external_ids:ovn-bridge-mappings=nattest:br-nat"
ssh -q $client1 "ip link set br-nat up"
ssh -q $client1 "ovs-vsctl add-port br-nat $nic_test2_cli"
ssh -q $client1 "ip link set $nic_test2_cli up"

ip netns add server0
ip link add veth0_s0 netns server0 type veth peer name veth0_s0_p
ip netns exec server0 ip link set veth0_s0 up
ip netns exec server0 ip link set lo up
ip netns exec server0 ip link set veth0_s0 address $mac_veth2_hv1
ip netns exec server0 ip addr add ${ip4_veth2_hv1}/24 dev veth0_s0
ip netns exec server0 ip -6 addr add ${ip6_veth2_hv1}/64 dev veth0_s0
ip netns exec server0 ip route add default via 172.16.1.254 dev veth0_s0
ip netns exec server0 ip -6 route add default via 2001::a dev veth0_s0

ovs-vsctl add-port br-int veth0_s0_p
ip link set veth0_s0_p up
ovs-vsctl set interface veth0_s0_p external_ids:iface-id=ls2p1

ovn-nbctl ls-add ls1
ovn-nbctl lsp-add ls1 ls1p1
ovn-nbctl lsp-set-addresses ls1p1 $mac_veth2_hv0

ovn-nbctl ls-add ls2
ovn-nbctl lsp-add ls2 ls2p1
ovn-nbctl lsp-set-addresses ls2p1 $mac_veth2_hv1

ovn-nbctl lr-add lr1
ovn-nbctl lrp-add lr1 lr1-ls1 00:00:09:09:00:01 172.16.0.254/24 2000::a/64
ovn-nbctl lrp-add lr1 lr1-ls2 00:00:09:09:00:02 172.16.1.254/24 2001::a/64
ovn-nbctl lrp-add lr1 lr1-ls0 00:00:09:09:00:03 192.$ip_subnet.1.254/24 3000:$ip_subnet::a/64

ovn-nbctl lsp-add ls1 ls1-lr1
ovn-nbctl lsp-set-type ls1-lr1 router
ovn-nbctl lsp-set-addresses ls1-lr1 00:00:09:09:00:01
ovn-nbctl lsp-set-options ls1-lr1 router-port=lr1-ls1

ovn-nbctl lsp-add ls2 ls2-lr1
ovn-nbctl lsp-set-type ls2-lr1 router
ovn-nbctl lsp-set-addresses ls2-lr1 00:00:09:09:00:02
ovn-nbctl lsp-set-options ls2-lr1 router-port=lr1-ls2

ovn-nbctl ls-add ls0
ovn-nbctl lsp-add ls0 ls0-lr1
ovn-nbctl lsp-set-addresses ls0-lr1 00:00:09:09:00:03
ovn-nbctl lsp-set-type ls0-lr1 router
ovn-nbctl lsp-set-options ls0-lr1 router-port=lr1-ls0

ovn-nbctl lsp-add ls0 ln_port
ovn-nbctl lsp-set-addresses ln_port unknown
ovn-nbctl lsp-set-type ln_port localnet
ovn-nbctl lsp-set-options ln_port network_name=nattest

ovs-vsctl add-br br-nat
ovs-vsctl set open . external_ids:ovn-bridge-mappings=nattest:br-nat
ip link set br-nat up
ovs-vsctl add-port br-nat $nic_test2_ser
ip link set $nic_test2_ser up

ip netns add server1
ip link add veth0_s1 netns server1 type veth peer name veth0_s1_p
ip netns exec server1 ip link set veth0_s1 up
ip netns exec server1 ip addr add ${ip4_external1_hv1}/24 dev veth0_s1
ip netns exec server1 ip -6 addr add ${ip6_external1_hv1}/24 dev veth0_s1
ip netns exec server1 ip route add default via 192.$ip_subnet.1.254 dev veth0_s1
ip netns exec server1 ip -6 route add default via 3000:$ip_subnet::a dev veth0_s1

ovs-vsctl add-port br-nat veth0_s1_p
ip link set veth0_s1_p up

ssh -q $client1 ip netns add client0
ssh -q $client1 ip link add veth0_c0 netns client0 type veth peer name veth0_c0_p
ssh -q $client1 ip netns exec client0 ip link set lo up
ssh -q $client1 ip netns exec client0 ip link set veth0_c0 up
ssh -q $client1 ip netns exec client0 ip addr add ${ip4_veth2_hv0}/24 dev veth0_c0
ssh -q $client1 ip netns exec client0 ip -6 addr add ${ip6_veth2_hv0}/64 dev veth0_c0
ssh -q $client1 ip netns exec client0 ip link set veth0_c0 address $mac_veth2_hv0
ssh -q $client1 ip netns exec client0 ip route add default via 172.16.0.254 dev veth0_c0
ssh -q $client1 ip netns exec client0 ip -6 route add default via 2000::a dev veth0_c0
ssh -q $client1 ovs-vsctl add-port br-int veth0_c0_p
ssh -q $client1 ip link set veth0_c0_p up
ssh -q $client1 ovs-vsctl set interface veth0_c0_p external_ids:iface-id=ls1p1

ip netns exec server0 ping6 ${ip6_veth2_hv0} -c 3

ovn-nbctl set logical_router lr1 options:chassis=hv1
ovn-nbctl --may-exist lr-nat-add lr1 dnat_and_snat 3001::1 ${ip6_veth2_hv1}
ovn-nbctl --may-exist lr-nat-add lr1 dnat_and_snat 3001::1 ${ip6_veth2_hv1}

ip netns exec server0 ping6 $ip6_external1_hv1 -c 5
ovs-ofctl show br-nat
ovs-appctl fdb/show br-nat
ovn-nbctl --if-exists lr-nat-del lr1 dnat_and_snat 3001::1
ovn-nbctl --may-exist lr-nat-add lr1 dnat 3001::41 ${ip6_veth2_hv1}
ovn-nbctl --may-exist lr-nat-add lr1 dnat 3001::41 ${ip6_veth2_hv1}

ip netns exec server1 ping6 3001::41 -c 1
ovs-appctl fdb/show br-nat
ip netns exec server1 ping6 3001::41 -c 5
ovs-appctl fdb/show br-nat

Actual results:
+ ovs-ofctl show br-nat                                                                               
OFPT_FEATURES_REPLY (xid=0x2): dpid:0000a0369fab36b6                                                  
n_tables:254, n_buffers:0                                                                             
capabilities: FLOW_STATS TABLE_STATS PORT_STATS QUEUE_STATS ARP_MATCH_IP                              
actions: output enqueue set_vlan_vid set_vlan_pcp strip_vlan mod_dl_src mod_dl_dst mod_nw_src mod_nw_dst mod_nw_tos mod_tp_src mod_tp_dst
 1(patch-ln_port-t): addr:8e:99:60:29:66:d0                                                           
     config:     0                                                                                    
     state:      0                                                                                    
     speed: 0 Mbps now, 0 Mbps max                                                                    
 2(enp5s0f1): addr:a0:36:9f:ab:36:b6                                                                  
     config:     0                                                                                    
     state:      0                                                                                    
     current:    10GB-FD                                                                              
     advertised: 10GB-FD FIBER AUTO_PAUSE                                                             
     supported:  10GB-FD FIBER AUTO_PAUSE                                                             
     speed: 10000 Mbps now, 10000 Mbps max                                                            
 3(veth0_s1_p): addr:16:61:f4:5e:d6:e8                                                                
     config:     0                                                                                    
     state:      0                                                                                    
     current:    10GB-FD COPPER                                                                       
     speed: 10000 Mbps now, 0 Mbps max                                                                
 LOCAL(br-nat): addr:a0:36:9f:ab:36:b6                                                                
     config:     0                                                                                    
     state:      0                                                                                    
     speed: 0 Mbps now, 0 Mbps max                                                                    
OFPT_GET_CONFIG_REPLY (xid=0x4): frags=normal miss_send_len=0 
......
+ ip netns exec server1 ping6 3001::41 -c 1
PING 3001::41(3001::41) 56 data bytes
64 bytes from 3001::41: icmp_seq=1 ttl=63 time=1.05 ms

--- 3001::41 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 1.049/1.049/1.049/0.000 ms                                                     
+ ovs-appctl fdb/show br-nat
 port  VLAN  MAC                Age
LOCAL     0  a6:87:c9:55:41:49   10
    2    12  fe:5c:0d:b6:c9:63   10
    2    11  fe:5c:0d:b6:c9:63   10
    2     0  b8:59:9f:ea:92:1b    8                                                                   
    2    14  68:05:ca:2a:3a:a2    7
    2    12  a6:07:e1:b3:ab:97    6
    2     3  34:73:5a:99:55:2e    3
LOCAL     0  a0:36:9f:ab:36:b6    3
    2     0  34:73:5a:99:55:2e    3                                                                   
    2    12  68:05:ca:2a:3a:a2    3
    2    14  fe:5c:0d:b6:c9:63    2
    2     0  40:f0:78:11:69:ea    0
    3     0  6a:06:51:58:44:15    0
    1     0  00:00:09:09:00:03    0

<=== mac of router is learned on port 1

+ ip netns exec server1 ping6 3001::41 -c 5                                                           
PING 3001::41(3001::41) 56 data bytes                                                                 
64 bytes from 3001::41: icmp_seq=1 ttl=63 time=0.078 ms
64 bytes from 3001::41: icmp_seq=2 ttl=63 time=0.280 ms

--- 3001::41 ping statistics ---
5 packets transmitted, 2 received, 60% packet loss, time 4094ms
rtt min/avg/max/mdev = 0.078/0.179/0.280/0.101 ms     

<=== packet loss after 2 ping
                                                
+ ovs-appctl fdb/show br-nat
 port  VLAN  MAC                Age
LOCAL     0  a6:87:c9:55:41:49   15
    2    12  fe:5c:0d:b6:c9:63   15                                                                   
    2    11  fe:5c:0d:b6:c9:63   15
    2     0  b8:59:9f:ea:92:1b   13
    2    14  68:05:ca:2a:3a:a2   12
    2    12  a6:07:e1:b3:ab:97   11
LOCAL     0  a0:36:9f:ab:36:b6    8                                                                   
    2    12  68:05:ca:2a:3a:a2    8
    2    14  fe:5c:0d:b6:c9:63    7                                                                   
    2    13  68:05:ca:2a:3a:a2    4                                                                   
    2     0  00:00:09:09:00:03    3  

<=== mac of router is learned on port 2
                                                                 
    2     0  40:f0:78:11:69:ea    1                                                                   
    2     0  08:94:ef:04:78:5a    0                                                                   
    3     0  6a:06:51:58:44:15    0                                                                   
    2     0  00:15:17:de:0c:87    0                                                                   
    2     0  08:94:ef:04:78:5b    0                                                                   
    2     0  34:73:5a:99:55:2e    0                                                                   
    2     3  34:73:5a:99:55:2e    0 

Expected results:
there is no loss for ping

Additional info:

it seems that the bind for mac of router on br-nat changed from port 1 to port 2. port 1 is the patch port, port 2 is another nic. the right bind should be port 1.

[root@dell-per730-03 test]# ovs-vsctl show                                                            
42e94eab-1950-4816-875b-7b99beade360                                                                  
    Bridge br-nat                                                                                     
        Port veth0_s1_p                                                                               
            Interface veth0_s1_p                                                                      
        Port enp5s0f1                                                                                 
            Interface enp5s0f1                                                                        
        Port patch-ln_port-to-br-int                                                                  
            Interface patch-ln_port-to-br-int                                                         
                type: patch                                                                           
                options: {peer=patch-br-int-to-ln_port}                                               
        Port br-nat                                                                                   
            Interface br-nat                                                                          
                type: internal                                                                        
    Bridge br-int                                                                                     
        fail_mode: secure                                                                             
        datapath_type: system                                                                         
        Port ovn-hv0-0                                                                                
            Interface ovn-hv0-0                                                                       
                type: geneve                                                                          
                options: {csum="true", key=flow, remote_ip="1.1.5.26"}                                
        Port veth0_s0_p                                                                               
            Interface veth0_s0_p                                                                      
        Port patch-br-int-to-ln_port                                                                  
            Interface patch-br-int-to-ln_port                                                         
                type: patch                                                                           
                options: {peer=patch-ln_port-to-br-int}                                               
        Port br-int                                                                                   
            Interface br-int                                                                          
                type: internal                                                                        
    ovs_version: "2.15.4"                                                                             
[root@dell-per730-03 test]# rpm -qa | grep -E "openvswitch2.15|ovn-2021"                              
ovn-2021-central-21.12.0-30.el8fdp.x86_64                                                             
openvswitch2.15-2.15.0-81.el8fdp.x86_64                                                               
ovn-2021-21.12.0-30.el8fdp.x86_64                                                                     
ovn-2021-host-21.12.0-30.el8fdp.x86_64

[root@dell-per740-42 ~]# ovs-vsctl show                                                               
849c1d02-adab-46ca-a5fe-a767cee7c0ef
    Bridge br-nat
        Port br-nat
            Interface br-nat                                                                          
                type: internal                                                                        
        Port ens3f1
            Interface ens3f1                                                                          
        Port patch-ln_port-to-br-int                                                                  
            Interface patch-ln_port-to-br-int                                                         
                type: patch                                                                           
                options: {peer=patch-br-int-to-ln_port}                                               
    Bridge br-int
        fail_mode: secure
        datapath_type: system                                                                         
        Port ovn-hv1-0
            Interface ovn-hv1-0
                type: geneve
                options: {csum="true", key=flow, remote_ip="1.1.5.25"}                                
        Port veth0_c0_p
            Interface veth0_c0_p                                                                      
        Port br-int
            Interface br-int
                type: internal                                                                        
        Port patch-br-int-to-ln_port
            Interface patch-br-int-to-ln_port                                                         
                type: patch
                options: {peer=patch-ln_port-to-br-int}                                               
    ovs_version: "2.15.4"

Comment 1 Jianlin Shi 2022-03-10 08:41:18 UTC
the issue doesn't occur on ovn-2021-21.06.0-29. and appear from ovn-2021-21.09.0-14.

Comment 2 Mark Michelson 2022-04-04 13:09:05 UTC
Hi, does https://bugzilla.redhat.com/show_bug.cgi?id=2062580#c1 mean that the issue is fixed now?

Comment 3 Jianlin Shi 2022-04-06 00:49:51 UTC
(In reply to Mark Michelson from comment #2)
> Hi, does https://bugzilla.redhat.com/show_bug.cgi?id=2062580#c1 mean that
> the issue is fixed now?

no, ovn-2021-21.09.0-14 is newer than ovn-2021-21.06.0-29, which means that the issue doesn't exist on the old version, but still exist on the new version.

Comment 4 Ales Musil 2022-04-12 14:29:13 UTC
I am still not able to reproduce it locally however on the env I have noticed few things: 

The ping will catch up after few seconds and since then it looks ok. 

The logical flows are ok the whole time: 

# ovn-trace --minimal ls0 'inport == "ln_port" && eth.src == f6:26:8f:88:f7:78 && ip6.src == 3000:54::1 && eth.dst == 00:00:09:09:00:03 && ip6.dst == 3001::41 && ip.ttl == 32'
# ipv6,reg14=0x2,vlan_tci=0x0000,dl_src=f6:26:8f:88:f7:78,dl_dst=00:00:09:09:00:03,ipv6_src=3000:54::1,ipv6_dst=3001::41,ipv6_label=0x00000,nw_proto=0,nw_tos=0,nw_ecn=0,nw_ttl=32
ct_dnat(ip6.dst=2001::1) /* default (use --ct to customize) */ {
    ip.ttl--;
    eth.src = 00:00:09:09:00:02;
    eth.dst = 00:00:00:01:01:02;
    ct_dnat /* assuming no un-dnat entry, so no change */ {
        output("ls2p1");
    };
};

The physical flows are disturbed when the ping is not going though.

------- Working flow -------:
# ovs-appctl ofproto/trace br-nat in_port=3,icmp6,dl_src=f6:26:8f:88:f7:78,dl_dst=00:00:09:09:00:03,ipv6_src=3000:54::1,ipv6_dst=3001::41
Flow: icmp6,in_port=3,vlan_tci=0x0000,dl_src=f6:26:8f:88:f7:78,dl_dst=00:00:09:09:00:03,ipv6_src=3000:54::1,ipv6_dst=3001::41,ipv6_label=0x00000,nw_tos=0,nw_ecn=0,nw_ttl=0,icmp_type=0,icmp_code=0

bridge("br-nat")
----------------
 0. priority 0
    NORMAL
     -> forwarding to learned port

bridge("br-int")
----------------
 0. in_port=3,vlan_tci=0x0000/0x1000, priority 100, cookie 0x3fd3b927
    set_field:0x8->reg11
    set_field:0x9->reg12
    set_field:0x4->metadata
    set_field:0x2->reg14
    resubmit(,8)
 8. reg14=0x2,metadata=0x4, priority 50, cookie 0x4afe314c
    resubmit(,9)
 9. metadata=0x4, priority 0, cookie 0xb617ba56
    resubmit(,10)
10. metadata=0x4, priority 0, cookie 0xbab7df93
    resubmit(,11)
11. metadata=0x4, priority 0, cookie 0x7ef87950
    resubmit(,12)
12. metadata=0x4, priority 0, cookie 0x5c491ce7
    resubmit(,13)
13. metadata=0x4, priority 0, cookie 0x7f4270a2
    resubmit(,14)
14. ipv6,reg14=0x2,metadata=0x4, priority 110, cookie 0x8bf356ea
    resubmit(,15)
15. metadata=0x4, priority 0, cookie 0x5cc9d6f
    resubmit(,16)
16. metadata=0x4, priority 65535, cookie 0xba18576c
    resubmit(,17)
17. metadata=0x4, priority 65535, cookie 0x7b126187
    resubmit(,18)
18. metadata=0x4, priority 0, cookie 0xa3179698
    resubmit(,19)
19. metadata=0x4, priority 0, cookie 0xc0dbb190
    resubmit(,20)
20. metadata=0x4, priority 0, cookie 0x8d569033
    resubmit(,21)
21. metadata=0x4, priority 0, cookie 0x23785310
    resubmit(,22)
22. metadata=0x4, priority 0, cookie 0x84b0be42
    resubmit(,23)
23. metadata=0x4, priority 0, cookie 0xc1f6777
    resubmit(,24)
24. metadata=0x4, priority 0, cookie 0x94ecbbd9
    resubmit(,25)
25. metadata=0x4, priority 0, cookie 0xaad469ed
    resubmit(,26)
26. reg14=0x2,metadata=0x4, priority 100, cookie 0x1dabebda
    resubmit(,27)
27. metadata=0x4, priority 0, cookie 0xde823af8
    resubmit(,28)
28. metadata=0x4, priority 0, cookie 0x6bdd0572
    resubmit(,29)
29. metadata=0x4, priority 0, cookie 0xad47b4a4
    resubmit(,30)
30. metadata=0x4, priority 0, cookie 0x7e3d8de8
    resubmit(,31)
31. metadata=0x4, priority 0, cookie 0x203ddc5e
    resubmit(,32)
32. metadata=0x4,dl_dst=00:00:09:09:00:03, priority 50, cookie 0x9e0691c5
    set_field:0x1->reg15
    resubmit(,37)
37. priority 0
    resubmit(,38)
38. reg15=0x1,metadata=0x4, priority 100, cookie 0xec729a34
    set_field:0x8->reg11
    set_field:0x9->reg12
    resubmit(,39)
39. priority 0
    set_field:0->reg0
    set_field:0->reg1
    set_field:0->reg2
    set_field:0->reg3
    set_field:0->reg4
    set_field:0->reg5
    set_field:0->reg6
    set_field:0->reg7
    set_field:0->reg8
    set_field:0->reg9
    resubmit(,40)
40. ipv6,reg15=0x1,metadata=0x4, priority 110, cookie 0x454f22c6
    resubmit(,41)
41. metadata=0x4, priority 0, cookie 0xdfaccffe
    resubmit(,42)
42. metadata=0x4, priority 0, cookie 0x1d2b3b7d
    resubmit(,43)
43. metadata=0x4, priority 65535, cookie 0xa7f28712
    resubmit(,44)
44. metadata=0x4, priority 65535, cookie 0x5b64a9f2
    resubmit(,45)
45. metadata=0x4, priority 0, cookie 0x6ed73e66
    resubmit(,46)
46. metadata=0x4, priority 0, cookie 0x1b88aa07
    resubmit(,47)
47. metadata=0x4, priority 0, cookie 0x6fc13b06
    resubmit(,48)
48. metadata=0x4, priority 0, cookie 0xfc31e2b4
    resubmit(,49)
49. reg15=0x1,metadata=0x4, priority 50, cookie 0xfbe73cbd
    resubmit(,64)
64. priority 0
    resubmit(,65)
65. reg15=0x1,metadata=0x4, priority 100, cookie 0xec729a34
    clone(ct_clear,set_field:0->reg11,set_field:0->reg12,set_field:0->reg13,set_field:0x5->reg11,set_field:0x4->reg12,set_field:0x3->metadata,set_field:0x3->reg14,set_field:0->reg10,set_field:0->reg15,set_field:0->reg0,set_field:0->reg1,set_field:0->reg2,set_field:0->reg3,set_field:0->reg4,set_field:0->reg5,set_field:0->reg6,set_field:0->reg7,set_field:0->reg8,set_field:0->reg9,resubmit(,8))
    ct_clear
    set_field:0->reg11
    set_field:0->reg12
    set_field:0->reg13
    set_field:0x5->reg11
    set_field:0x4->reg12
    set_field:0x3->metadata
    set_field:0x3->reg14
    set_field:0->reg10
    set_field:0->reg15
    set_field:0->reg0
    set_field:0->reg1
    set_field:0->reg2
    set_field:0->reg3
    set_field:0->reg4
    set_field:0->reg5
    set_field:0->reg6
    set_field:0->reg7
    set_field:0->reg8
    set_field:0->reg9
    resubmit(,8)
 8. reg14=0x3,metadata=0x3,dl_dst=00:00:09:09:00:03, priority 50, cookie 0x2a64a866
    set_field:0x90900030000000000000000/0xffffffffffff0000000000000000->xxreg0
    resubmit(,9)
 9. metadata=0x3, priority 0, cookie 0xf6342cab
    set_field:0x4/0x4->xreg4
    resubmit(,10)
10. reg9=0x4/0x4,metadata=0x3, priority 100, cookie 0x380c6ee4
    resubmit(,11)
11. ipv6,reg14=0x3,metadata=0x3,ipv6_src=3000:54::/64,nw_ttl=0,nw_frag=not_later, priority 100, cookie 0xc42be348
    controller(userdata=00.00.00.0a.00.00.00.00.ff.ff.00.18.00.00.23.20.00.1b.00.00.00.00.04.06.00.30.00.00.00.00.00.00.ff.ff.00.18.00.00.23.20.00.1b.00.00.00.00.02.06.00.30.00.00.00.00.00.00.ff.ff.00.18.00.00.23.20.00.1c.00.00.00.00.04.06.00.30.00.00.00.00.00.00.ff.ff.00.18.00.00.23.20.00.1c.00.00.00.00.02.06.00.30.00.00.00.00.00.00.00.1c.00.18.00.80.00.00.00.00.00.00.80.00.34.10.80.00.36.10.00.00.00.00.00.19.00.18.80.00.34.10.30.00.00.54.00.00.00.00.00.00.00.00.00.00.00.0a.00.19.00.10.00.01.3a.01.fe.00.00.00.00.00.00.00.00.19.00.10.80.00.3a.01.03.00.00.00.00.00.00.00.00.19.00.10.80.00.3c.01.00.00.00.00.00.00.00.00.00.19.00.10.00.01.1e.04.00.00.00.03.00.00.00.00.00.19.00.10.00.01.15.08.00.00.00.01.00.00.00.01.ff.ff.00.10.00.00.23.20.00.0e.ff.f8.25.00.00.00)

Final flow: unchanged
Megaflow: recirc_id=0,ct_state=-new-trk,eth,icmp6,in_port=3,dl_src=f6:26:8f:88:f7:78,dl_dst=00:00:09:09:00:03,ipv6_src=3000:54::/64,ipv6_dst=3001::41,nw_ttl=0,nw_frag=no,nd_target=::
Datapath actions: userspace(pid=4015846741,controller(reason=1,dont_send=1,continuation=0,recirc_id=60,rule_cookie=0xc42be348,controller_id=0,max_len=65535))


------- Disturbed flow -------: 
# ovs-appctl ofproto/trace br-nat in_port=3,icmp6,dl_src=52:ad:dd:51:5b:77,dl_dst=00:00:09:09:00:03,ipv6_src=3000:54::1,ipv6_dst=3001::41
Flow: icmp6,in_port=3,vlan_tci=0x0000,dl_src=52:ad:dd:51:5b:77,dl_dst=00:00:09:09:00:03,ipv6_src=3000:54::1,ipv6_dst=3001::41,ipv6_label=0x00000,nw_tos=0,nw_ecn=0,nw_ttl=0,icmp_type=0,icmp_code=0

bridge("br-nat")
----------------
 0. priority 0
    NORMAL
     -> forwarding to learned port

Final flow: unchanged
Megaflow: recirc_id=0,eth,ipv6,in_port=3,dl_src=52:ad:dd:51:5b:77,dl_dst=00:00:09:09:00:03,nw_frag=no
Datapath actions: 5

Comment 5 Ales Musil 2022-04-14 11:42:44 UTC
Patch posted: https://patchwork.ozlabs.org/project/ovn/list/?series=295147

Comment 6 Jianlin Shi 2022-09-13 08:50:48 UTC
confirmed that the issue disappered on ovn22.06-22.06.0-27.el8:

[root@wsfd-advnetlab16 bz2062580]# rpm -qa | grep  -E "openvswitch2.15|ovn22.06"                      
openvswitch2.15-2.15.0-81.el8fdp.x86_64                                                               
ovn22.06-central-22.06.0-27.el8fdp.x86_64                                                             
ovn22.06-22.06.0-27.el8fdp.x86_64                                                                     
ovn22.06-host-22.06.0-27.el8fdp.x86_64

+ ovs-ofctl show br-nat                            
OFPT_FEATURES_REPLY (xid=0x2): dpid:00000c42a1080b1b
n_tables:254, n_buffers:0          
capabilities: FLOW_STATS TABLE_STATS PORT_STATS QUEUE_STATS ARP_MATCH_IP
actions: output enqueue set_vlan_vid set_vlan_pcp strip_vlan mod_dl_src mod_dl_dst mod_nw_src mod_nw_dst mod_nw_tos mod_tp_src mod_tp_dst
 1(patch-ln_port-t): addr:be:a0:d5:c7:9e:52                                                           
     config:     0                                 
     state:      0                                 
     speed: 0 Mbps now, 0 Mbps max                                                                    
 2(ens1f1): addr:0c:42:a1:08:0b:1b
     config:     0                                 
     state:      0                                                                                    
     current:    AUTO_NEG                        
     advertised: 1GB-FD 10GB-FD AUTO_NEG AUTO_PAUSE 
     supported:  1GB-FD 10GB-FD AUTO_NEG AUTO_PAUSE 
     speed: 0 Mbps now, 10000 Mbps max
 3(veth0_s1_p): addr:3a:28:7e:5d:65:20
     config:     0                                 
     state:      0                                 
     current:    10GB-FD COPPER    
     speed: 10000 Mbps now, 0 Mbps max     
 LOCAL(br-nat): addr:0c:42:a1:08:0b:1b
     config:     0                                                                                    
     state:      0                                                                                    
     speed: 0 Mbps now, 0 Mbps max                                                                    
OFPT_GET_CONFIG_REPLY (xid=0x4): frags=normal miss_send_len=0
+ ovs-appctl fdb/show br-nat                                                                          
 port  VLAN  MAC                Age
LOCAL     0  8e:cf:14:4b:be:47   10
    2     0  0c:42:a1:08:0a:d3    5                                                                   
LOCAL     0  0c:42:a1:08:0b:1b    4              
    1     0  00:00:09:09:00:03    0
    3     0  e6:25:09:82:5e:60    0
+ ovn-nbctl --if-exists lr-nat-del lr1 dnat_and_snat 3001::1
+ ovn-nbctl --may-exist lr-nat-add lr1 dnat 3001::41 2001::1
+ ovn-nbctl --may-exist lr-nat-add lr1 dnat 3001::41 2001::1                                          
+ ip netns exec server1 ping6 3001::41 -c 1
PING 3001::41(3001::41) 56 data bytes
64 bytes from 3001::41: icmp_seq=1 ttl=63 time=1.54 ms                      
                                                   
--- 3001::41 ping statistics ---                                                                      
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 1.542/1.542/1.542/0.000 ms                                                     
+ ovs-appctl fdb/show br-nat                                                                          
 port  VLAN  MAC                Age                                                                   
LOCAL     0  8e:cf:14:4b:be:47   10
    2     0  0c:42:a1:08:0a:d3    5
LOCAL     0  0c:42:a1:08:0b:1b    4                                                                   
    1     0  00:00:09:09:00:03    0              
    3     0  e6:25:09:82:5e:60    0
+ ip netns exec server1 ping6 3001::41 -c 5
PING 3001::41(3001::41) 56 data bytes    
64 bytes from 3001::41: icmp_seq=1 ttl=63 time=0.140 ms
64 bytes from 3001::41: icmp_seq=2 ttl=63 time=0.487 ms
64 bytes from 3001::41: icmp_seq=3 ttl=63 time=0.085 ms
64 bytes from 3001::41: icmp_seq=4 ttl=63 time=0.079 ms
64 bytes from 3001::41: icmp_seq=5 ttl=63 time=0.080 ms

--- 3001::41 ping statistics ---
5 packets transmitted, 5 received, 0% packet loss, time 4089ms
rtt min/avg/max/mdev = 0.079/0.174/0.487/0.158 ms
+ ovs-appctl fdb/show br-nat
 port  VLAN  MAC                Age
LOCAL     0  8e:cf:14:4b:be:47   14
LOCAL     0  0c:42:a1:08:0b:1b    8
    1     0  00:00:09:09:00:03    0
    3     0  e6:25:09:82:5e:60    0
    2     0  0c:42:a1:08:0a:d3    0

Comment 7 OVN Bot 2023-01-13 14:18:35 UTC
ovn22.03 fast-datapath-rhel-8 clone created at https://bugzilla.redhat.com/show_bug.cgi?id=2160737
ovn22.03 fast-datapath-rhel-9 clone created at https://bugzilla.redhat.com/show_bug.cgi?id=2160738


Note You need to log in before you can comment on or make changes to this bug.