Description of problem: packet replied for load balance is not snated if both lb and nat are added Version-Release number of selected component (if applicable): ovn-2021-20.12.0-20 How reproducible: Always Steps to Reproduce: 1. setup on hv1: systemctl start openvswitch systemctl start ovn-northd ovn-nbctl set-connection ptcp:6641 ovn-sbctl set-connection ptcp:6642 ovs-vsctl set open . external_ids:system-id=hv1 external_ids:ovn-remote=tcp:20.0.180.25:6642 external_ids:ovn-encap-type=geneve external_ids:ovn-encap-ip=20.0.180.25 systemctl restart ovn-controller ovs-vsctl add-br br-ex ovs-vsctl set open . external_ids:ovn-bridge-mappings=provider:br-ex ovs-vsctl add-port br-ex ens1f1 ip link set ens1f1 up ip link set br-ex up ovn-nbctl ls-add ls1 ovn-nbctl lsp-add ls1 ls1p1 ovn-nbctl lsp-set-addresses ls1p1 "00:00:00:01:01:11 192.168.1.11 2001::11" ovn-nbctl lsp-add ls1 ls1p2 ovn-nbctl lsp-set-addresses ls1p2 "00:00:00:01:01:12 192.168.1.12 2001::12" ovn-nbctl lsp-add ls1 ls1p3 ovn-nbctl lsp-set-addresses ls1p3 "00:00:00:01:01:13 192.168.1.13 2001::13" ovn-nbctl lr-add lr1 ovn-nbctl lrp-add lr1 lr1-ls1 00:00:00:00:00:01 192.168.1.254/24 2001::a/64 ovn-nbctl lsp-add ls1 ls1-lr1 ovn-nbctl lsp-set-addresses ls1-lr1 router ovn-nbctl lsp-set-type ls1-lr1 router ovn-nbctl lsp-set-options ls1-lr1 router-port=lr1-ls1 ovn-nbctl ls-add ls_pub ovn-nbctl lsp-add ls_pub ls_pub_ln ovn-nbctl lsp-set-options ls_pub_ln network_name=provider ovn-nbctl lsp-set-type ls_pub_ln localnet ovn-nbctl lsp-set-addresses ls_pub_ln unknown ovn-nbctl lrp-add lr1 r1-ln 00:00:00:00:00:03 172.18.1.254/24 7011:18::1/64 ovn-nbctl lrp-set-gateway-chassis r1-ln hv0 ovn-nbctl lsp-add ls_pub ls_pub-r1 ovn-nbctl lsp-set-type ls_pub-r1 router ovn-nbctl lsp-set-addresses ls_pub-r1 router ovn-nbctl lsp-set-options ls_pub-r1 router-port=r1-ln ovn-nbctl lr-nat-add lr1 dnat_and_snat 172.18.1.11 192.168.1.11 ls1p1 00:00:00:00:01:11 ovn-nbctl lr-nat-add lr1 dnat_and_snat 172.18.1.12 192.168.1.12 ls1p2 00:00:00:00:01:12 ovn-nbctl lb-add lb_tcp 172.18.1.50:50001 192.168.1.11:50001,192.168.1.12:50001,192.168.1.13:50001 tcp ovn-nbctl lr-lb-add lr1 lb_tcp ovn-nbctl set load_balancer lb_tcp selection_fields="ip_src,ip_dst,tp_src,tp_dst" ip netns add ls1p1 ovs-vsctl add-port br-int ls1p1 -- set interface ls1p1 type=internal external_ids:iface-id=ls1p1 ip link set ls1p1 netns ls1p1 ip netns exec ls1p1 ip link set ls1p1 address 00:00:00:01:01:11 ip netns exec ls1p1 ip link set ls1p1 up ip netns exec ls1p1 ip addr add 192.168.1.11/24 dev ls1p1 ip netns exec ls1p1 ip route add default via 192.168.1.254 dev ls1p1 ip netns exec ls1p1 nc -k -l 50001 & ip netns add ls1p2 ovs-vsctl add-port br-int ls1p2 -- set interface ls1p2 type=internal external_ids:iface-id=ls1p2 ip link set ls1p2 netns ls1p2 ip netns exec ls1p2 ip link set ls1p2 address 00:00:00:01:01:12 ip netns exec ls1p2 ip link set ls1p2 up ip netns exec ls1p2 ip addr add 192.168.1.12/24 dev ls1p2 ip netns exec ls1p2 ip route add default via 192.168.1.254 dev ls1p2 ip netns exec ls1p2 nc -k -l 50001 & ovs-vsctl add-port br-ex ext1 -- set interface ext1 type=internal ip netns add ext1 ip link set ext1 netns ext1 ip netns exec ext1 ip link set ext1 up ip netns exec ext1 ip addr add 172.18.1.1/24 dev ext1 2. setup on hv0 systemctl start openvswitch ovs-vsctl set open . external_ids:system-id=hv0 external_ids:ovn-remote=tcp:20.0.180.25:6642 external_ids:ovn-encap-type=geneve external_ids:ovn-encap-ip=20.0.180.26 systemctl restart ovn-controller ovs-vsctl add-br br-ex ovs-vsctl set open . external_ids:ovn-bridge-mappings=provider:br-ex ovs-vsctl add-port br-ex ens1f1 ip link set ens1f1 up ip link set br-ex up ip netns add ls1p3 ovs-vsctl add-port br-int ls1p3 -- set interface ls1p3 type=internal external_ids:iface-id=ls1p3 ip link set ls1p3 netns ls1p3 ip netns exec ls1p3 ip link set ls1p3 address 00:00:00:01:01:13 ip netns exec ls1p3 ip link set ls1p3 up ip netns exec ls1p3 ip addr add 192.168.1.13/24 dev ls1p3 ip netns exec ls1p3 ip route add default via 192.168.1.254 dev ls1p3 ip netns exec ls1p3 nc -k -l 50001 & ip netns add ext2 ovs-vsctl add-port br-ex ext2 -- set interface ext2 type=internal ip link set ext2 netns ext2 ip netns exec ext2 ip link set ext2 up ip netns exec ext2 ip addr add 172.18.1.2/24 dev ext2 3. run nc on hv1: for i in {1..10}; do ip netns exec ext1 nc 172.18.1.50 50001 <<< h; done Actual results: part of the nc would fail with time out Ncat: Connection timed out. Expected results: all nc passed Additional info: from the tcpdump on ext1, I see that if the lb ip is load balanced to 192.168.1.11 or 192.168.1.12 which locate on hv1, the replied packet to ext1 is not snated: 02:13:42.275182 4a:cf:0c:a6:27:d0 > 00:00:00:00:00:03, ethertype IPv4 (0x0800), length 74: (tos 0x0, ttl 64, id 48738, offset 0, flags [DF], proto TCP (6), length 60) 172.18.1.1.46948 > 172.18.1.50.50001: Flags [S], cksum 0x5a86 (incorrect -> 0x45e2), seq 3994669128, win 29200, options [mss 1460,sackOK,TS val 2174974713 ecr 0,nop,wscale 7], length 0 02:13:42.276894 00:00:00:00:01:12 > 4a:cf:0c:a6:27:d0, ethertype IPv4 (0x0800), length 74: (tos 0x0, ttl 63, id 0, offset 0, flags [DF], proto TCP (6), length 60) 192.168.1.12.50001 > 172.18.1.1.46948: Flags [S.], cksum 0x6ef6 (incorrect -> 0x136c), seq 407500889, ack 3994669129, win 28960, options [mss 1460,sackOK,TS val 1810276955 ecr 2174974713,nop,wscale 7] , length 0 02:13:52.322551 4a:cf:0c:a6:27:d0 > 00:00:00:00:00:03, ethertype IPv4 (0x0800), length 74: (tos 0x0, ttl 64, id 44848, offset 0, flags [DF], proto TCP (6), length 60) 172.18.1.1.46950 > 172.18.1.50.50001: Flags [S], cksum 0x5a86 (incorrect -> 0x5802), seq 3894254818, win 29200, options [mss 1460,sackOK,TS val 2174984761 ecr 0,nop,wscale 7], length 0 02:13:52.324299 00:00:00:00:01:11 > 4a:cf:0c:a6:27:d0, ethertype IPv4 (0x0800), length 74: (tos 0x0, ttl 63, id 0, offset 0, flags [DF], proto TCP (6), length 60) 192.168.1.11.50001 > 172.18.1.1.46950: Flags [S.], cksum 0x6ef5 (incorrect -> 0xdbfd), seq 1741590943, ack 3894254819, win 28960, options [mss 1460,sackOK,TS val 2400837612 ecr 2174984761,nop,wscale 7 ], length 0
the problem still exist on ovn22.03-host-22.03.0-52.el8fdp.x86_64. following is another simple reproducer: server: systemctl start openvswitch systemctl start ovn-northd ovn-nbctl set-connection ptcp:6641 ovn-sbctl set-connection ptcp:6642 ovs-vsctl set open . external_ids:system-id=hv1 external_ids:ovn-remote=tcp:20.0.186.25:6642 external_ids:ovn-encap-type=geneve external_ids:ovn-encap-ip=20.0.186.25 systemctl restart ovn-controller ovn-nbctl lr-add lr1 ovn-nbctl ls-add ls1 ovn-nbctl ls-add public ovn-nbctl lrp-add lr1 lr1-ls1 00:00:01:01:02:03 192.168.1.254/24 2001::a/64 ovn-nbctl lrp-add lr1 lr1-pub 00:00:02:01:02:03 172.16.1.254/24 1000::a/64 ovs-vsctl add-br br-ext ovs-vsctl add-port br-ext ens1f1 ovn-nbctl lsp-add ls1 ls1-lr1 -- set Logical_Switch_Port ls1-lr1 \ type=router options:router-port=lr1-ls1 \ -- lsp-set-addresses ls1-lr1 router ovn-nbctl lsp-add public pub-lr1 -- set Logical_Switch_Port pub-lr1 \ type=router options:router-port=lr1-pub \ -- lsp-set-addresses pub-lr1 router \ ovn-nbctl lsp-add ls1 ls1p1 ovn-nbctl lsp-add ls1 ls1p1.11 ls1p1 11 -- lsp-set-addresses ls1p1.11 "f0:00:00:01:02:11 192.168.1.11 2001::11/64" ovs-vsctl set Open_vSwitch . external-ids:ovn-bridge-mappings=phynet:br-ext ovn-nbctl lsp-add public public1 \ -- lsp-set-addresses public1 unknown \ -- lsp-set-type public1 localnet \ -- lsp-set-options public1 network_name=phynet ovn-nbctl lr-nat-add lr1 dnat_and_snat 172.16.1.11 192.168.1.11 ls1p1.11 00:00:02:01:02:11 ovn-nbctl ha-chassis-group-add hagrp1 ovn-nbctl ha-chassis-group-add-chassis hagrp1 hv0 100 group1_id=$(ovn-nbctl get ha_chassis_group hagrp1 _uuid) ovn-nbctl set logical_router_port lr1-pub ha_chassis_group=$group1_id ovn-nbctl lb-add lb_r1_tcp 172.16.1.101:50001 192.168.1.11:50001 tcp ovn-nbctl lb-add lb_r1_tcp [1000::101]:50001 [2001::11]:50001 tcp ovn-nbctl set load_balancer lb_r1_tcp selection_fields="ip_src,ip_dst,tp_src,tp_dst" lb_r1_tcp_uuid=$(ovn-nbctl find load_balancer name=lb_r1_tcp | awk '/_uuid/{print $3}') lb_r1_grp_id=$(ovn-nbctl create load_balancer_group name=lb_r1_grp \ -- add load_balancer_group lb_r1_grp load_balancer $lb_r1_tcp_uuid) ovn-nbctl add logical_router lr1 load_balancer_group $lb_r1_grp_id ovs-vsctl add-port br-int ls1p1 -- set interface ls1p1 type=internal external:iface-id=ls1p1 ip link add link ls1p1 name ls1p1.11 type vlan id 11 ip link set ls1p1 up ip netns add ls1p1.11 ip link set ls1p1.11 netns ls1p1.11 ip netns exec ls1p1.11 ip link set ls1p1.11 address f0:00:00:01:02:11 ip netns exec ls1p1.11 ip link set ls1p1.11 up ip netns exec ls1p1.11 ip addr add 192.168.1.11/24 dev ls1p1.11 ip netns exec ls1p1.11 ip route add default via 192.168.1.254 ip netns exec ls1p1.11 ip addr add 2001::11/64 dev ls1p1.11 ip netns exec ls1p1.11 ip -6 route add default via 2001::a ip netns exec ls1p1.11 iperf3 -s -D -p 50001 & ovs-vsctl add-port br-ext server -- set interface server type=internal ip netns add server ip netns exec server ip link set lo up ip link set server netns server ip netns exec server ip link set server up ip netns exec server ip addr add 172.16.1.50/24 dev server ip netns exec server ip addr add 1000::50/64 dev server client: systemctl start openvswitch ovs-vsctl set open . external_ids:system-id=hv0 external_ids:ovn-remote=tcp:20.0.186.25:6642 external_ids:ovn-encap-type=geneve external_ids:ovn-encap-ip=20.0.186.26 systemctl restart ovn-controller ovs-vsctl add-br br-ext ovs-vsctl set Open_vSwitch . external-ids:ovn-bridge-mappings=phynet:br-ext ovs-vsctl add-port br-ext ens1f1
> ovs-vsctl add-port br-int ls1p1 -- set interface ls1p1 type=internal external:iface-id=ls1p1 <=== should be external_ids:iface-id=ls1p1
upstream fix: https://patchwork.ozlabs.org/project/ovn/patch/62e4e1ffc0ada5ce7eaf3e7c3c6e09d627eee2ac.1684441005.git.lorenzo.bianconi@redhat.com/
ovn23.09 fast-datapath-rhel-9 clone created at https://bugzilla.redhat.com/show_bug.cgi?id=2224399