Description of problem: OVN trace doesn't respect register value matches when tracing. Example: Trace doesn't recognize that in table 6, we have reg9[16..31] == 32398 (which is the tcp.dstport): ovn-sbctl --no-leader-only lflow-list GR_ovn-worker2 | grep lr_in_dnat table=5 (lr_in_defrag ), priority=110 , match=(ip && ip4.dst == 10.96.0.1 && tcp), action=(reg0 = 10.96.0.1; reg9[16..31] = tcp.dst; ct_dnat;) table=5 (lr_in_defrag ), priority=110 , match=(ip && ip4.dst == 10.96.0.10 && tcp), action=(reg0 = 10.96.0.10; reg9[16..31] = tcp.dst; ct_dnat;) table=5 (lr_in_defrag ), priority=110 , match=(ip && ip4.dst == 10.96.0.10 && udp), action=(reg0 = 10.96.0.10; reg9[16..31] = udp.dst; ct_dnat;) table=5 (lr_in_defrag ), priority=110 , match=(ip && ip4.dst == 10.96.120.53 && tcp), action=(reg0 = 10.96.120.53; reg9[16..31] = tcp.dst; ct_dnat;) table=5 (lr_in_defrag ), priority=110 , match=(ip && ip4.dst == 172.18.0.3 && tcp), action=(reg0 = 172.18.0.3; reg9[16..31] = tcp.dst; ct_dnat;) table=5 (lr_in_defrag ), priority=0 , match=(1), action=(next;) table=6 (lr_in_dnat ), priority=120 , match=(ct.est && ip4 && reg0 == 10.96.0.1 && tcp && reg9[16..31] == 443 && ct_label.natted == 1), action=(flags.force_snat_for_lb = 1; next;) table=6 (lr_in_dnat ), priority=120 , match=(ct.est && ip4 && reg0 == 10.96.0.10 && tcp && reg9[16..31] == 53 && ct_label.natted == 1), action=(flags.force_snat_for_lb = 1; next;) table=6 (lr_in_dnat ), priority=120 , match=(ct.est && ip4 && reg0 == 10.96.0.10 && tcp && reg9[16..31] == 9153 && ct_label.natted == 1), action=(flags.force_snat_for_lb = 1; next;) table=6 (lr_in_dnat ), priority=120 , match=(ct.est && ip4 && reg0 == 10.96.0.10 && udp && reg9[16..31] == 53 && ct_label.natted == 1), action=(flags.force_snat_for_lb = 1; next;) table=6 (lr_in_dnat ), priority=120 , match=(ct.est && ip4 && reg0 == 10.96.120.53 && tcp && reg9[16..31] == 8080 && ct_label.natted == 1), action=(flags.force_snat_for_lb = 1; next;) table=6 (lr_in_dnat ), priority=120 , match=(ct.est && ip4 && reg0 == 172.18.0.3 && tcp && reg9[16..31] == 32398 && ct_label.natted == 1), action=(flags.force_snat_for_lb = 1; next;) table=6 (lr_in_dnat ), priority=120 , match=(ct.new && ip4 && reg0 == 10.96.0.1 && tcp && reg9[16..31] == 443), action=(flags.force_snat_for_lb = 1; ct_lb(backends=172.18.0.4:6443);) table=6 (lr_in_dnat ), priority=120 , match=(ct.new && ip4 && reg0 == 10.96.0.10 && tcp && reg9[16..31] == 53), action=(flags.force_snat_for_lb = 1; ct_lb(backends=10.244.0.3:53,10.244.2.4:53);) table=6 (lr_in_dnat ), priority=120 , match=(ct.new && ip4 && reg0 == 10.96.0.10 && tcp && reg9[16..31] == 9153), action=(flags.force_snat_for_lb = 1; ct_lb(backends=10.244.0.3:9153,10.244.2.4:9153);) table=6 (lr_in_dnat ), priority=120 , match=(ct.new && ip4 && reg0 == 10.96.0.10 && udp && reg9[16..31] == 53), action=(flags.force_snat_for_lb = 1; ct_lb(backends=10.244.0.3:53,10.244.2.4:53);) table=6 (lr_in_dnat ), priority=120 , match=(ct.new && ip4 && reg0 == 10.96.120.53 && tcp && reg9[16..31] == 8080), action=(flags.force_snat_for_lb = 1; ct_lb(backends=10.244.0.4:8080,10.244.2.5:8080);) table=6 (lr_in_dnat ), priority=120 , match=(ct.new && ip4 && reg0 == 172.18.0.3 && tcp && reg9[16..31] == 32398), action=(flags.force_snat_for_lb = 1; ct_lb(backends=10.244.0.4:8080,10.244.2.5:8080);) table=6 (lr_in_dnat ), priority=0 , match=(1), action=(next;) Trace command and output snippet: sh-5.1# ovn-trace --ct new 'eth.dst == 0a:58:64:40:00:03 && inport == "rtoj-GR_ovn-worker2" && ip4.src==8.8.8.8 && ip4.dst==172.18.0.3 && ip.ttl==64 && tcp && tcp.src==41314 && tcp.dst==32398' --lb-dst 10.244.2.5:8080 # tcp,reg14=0x1,vlan_tci=0x0000,dl_src=00:00:00:00:00:00,dl_dst=0a:58:64:40:00:03,nw_src=8.8.8.8,nw_dst=172.18.0.3,nw_tos=0,nw_ecn=0,nw_ttl=64,tp_src=41314,tp_dst=32398,tcp_flags=0 ingress(dp="GR_ovn-worker2", inport="rtoj-GR_ovn-worker2") ---------------------------------------------------------- 0. lr_in_admission (northd.c:10374): eth.dst == 0a:58:64:40:00:03 && inport == "rtoj-GR_ovn-worker2", priority 50, uuid 26beac16 xreg0[0..47] = 0a:58:64:40:00:03; next; 1. lr_in_lookup_neighbor (northd.c:10455): 1, priority 0, uuid cda3ef89 reg9[2] = 1; next; 2. lr_in_learn_neighbor (northd.c:10464): reg9[2] == 1 || reg9[3] == 0, priority 100, uuid 47cabf81 next; 4. lr_in_unsnat (northd.c:9429): ip4 && ip4.dst == 172.18.0.3 && tcp && tcp.dst == 32398, priority 120, uuid bae81a29 next; 5. lr_in_defrag (northd.c:9623): ip && ip4.dst == 172.18.0.3 && tcp, priority 110, uuid e3512bce reg0 = 172.18.0.3; reg9[16..31] = tcp.dst; ct_dnat; ct_dnat /* assuming no un-dnat entry, so no change */ ----------------------------------------------------- 10. lr_in_ip_routing (northd.c:9203): ip4.dst == 172.18.0.0/16, priority 33, uuid f3f06084 ip.ttl--; reg8[0..15] = 0; reg0 = ip4.dst; reg1 = 172.18.0.3; eth.src = 02:42:ac:12:00:03; outport = "rtoe-GR_ovn-worker2"; flags.loopback = 1; next; 11. lr_in_ip_routing_ecmp (northd.c:10760): reg8[0..15] == 0, priority 150, uuid bde3d1a8 next; 12. lr_in_policy (northd.c:10885): 1, priority 0, uuid 3bc938fb reg8[0..15] = 0; next; 13. lr_in_policy_ecmp (northd.c:10887): reg8[0..15] == 0, priority 150, uuid 5b040850 next; 14. lr_in_arp_resolve (northd.c:10125): ip4.dst == {172.18.0.3}, priority 1, uuid 4e24ca9f drop; Version-Release number of selected component (if applicable): ARG ovnver=ovn-21.09.0-3.fc33 How reproducible: Always Expected results: It should match on table=6 (lr_in_dnat ), priority=120 , match=(ct.new && ip4 && reg0 == 172.18.0.3 && tcp && reg9[16..31] == 32398), action=(flags.force_snat_for_lb = 1; ct_lb(backends=10.244.0.4:8080,10.244.2.5:8080);)
Hi Surya, I dug more into this, the problem is actually that tracing 'ct_dnat' that's not preceded by a 'ct_next' in the pipeline will incorrectly skip setting the ct_state in the microflow. This happens since ct()/ct(nat) have been merged in https://github.com/ovn-org/ovn/commit/0038579d192802fff03c3594e4f85dab4f7af2bd I posted a patch to fix this and updated the summary of this BZ: http://patchwork.ozlabs.org/project/ovn/list/?series=269816&state=* Thanks, Dumitru
Reproduced in [root@bz_2017540 ~]# rpm -qa |grep -E "ovn|openvswitch" openvswitch-selinux-extra-policy-1.0-28.el8fdp.noarch ovn-2021-21.09.0-5.el8fdp.x86_64 ovn-2021-central-21.09.0-5.el8fdp.x86_64 ovn-2021-host-21.09.0-5.el8fdp.x86_64 openvswitch2.15-2.15.0-50.el8fdp.x86_64 #### Steps to reproduce as below #### systemctl start ovn-northd ovn-nbctl set-connection ptcp:6641 ovn-sbctl set-connection ptcp:6642 systemctl start openvswitch ovs-vsctl set open . external_ids:system-id=hv1 ovs-vsctl set open . external_ids:ovn-remote=tcp:42.42.42.1:6642 ovs-vsctl set open . external_ids:ovn-encap-type=geneve ovs-vsctl set open . external_ids:ovn-encap-ip=42.42.42.1 systemctl start ovn-controller ovn-nbctl lr-add rtr ovn-nbctl set logical_router rtr options:chassis=hv1 ovn-nbctl lrp-add rtr rtr-ls 00:00:00:00:01:01 10.0.0.1/24 ovn-nbctl lb-add lb-test 11.0.0.11:1111 10.0.0.2:1010 tcp ovn-nbctl lr-lb-add rtr lb-test ovn-nbctl --wait=sb sync [root@bz_2017540 ~]# ovn-sbctl --no-leader-only lflow-list rtr | grep lr_in_dnat table=6 (lr_in_dnat ), priority=120 , match=(ct.est && ip4 && reg0 == 11.0.0.11 && tcp && reg9[16..31] == 1111 && ct_label.natted == 1), action=(next;) table=6 (lr_in_dnat ), priority=120 , match=(ct.new && ip4 && reg0 == 11.0.0.11 && tcp && reg9[16..31] == 1111), action=(ct_lb(backends=10.0.0.2:1010);) table=6 (lr_in_dnat ), priority=0 , match=(1), action=(next;) [root@bz_2017540 ~]# ovn-trace --ct new 'eth.dst == 00:00:00:00:01:01 && inport == "rtr-ls" && ip4.src == 10.0.0.10 && ip4.dst == 11.0.0.11 && ip.ttl == 64 && tcp && tcp.dst == 1111' --lb-dst 10.0.0.10:1010 # tcp,reg14=0x1,vlan_tci=0x0000,dl_src=00:00:00:00:00:00,dl_dst=00:00:00:00:01:01,nw_src=10.0.0.10,nw_dst=11.0.0.11,nw_tos=0,nw_ecn=0,nw_ttl=64,tp_src=0,tp_dst=1111,tcp_flags=0 ingress(dp="rtr", inport="rtr-ls") ---------------------------------- 0. lr_in_admission (northd.c:10294): eth.dst == 00:00:00:00:01:01 && inport == "rtr-ls", priority 50, uuid 1b22c640 xreg0[0..47] = 00:00:00:00:01:01; next; 1. lr_in_lookup_neighbor (northd.c:10374): 1, priority 0, uuid bd05decb reg9[2] = 1; next; 2. lr_in_learn_neighbor (northd.c:10383): reg9[2] == 1, priority 100, uuid 4ebf265a next; 5. lr_in_defrag (northd.c:9541): ip && ip4.dst == 11.0.0.11 && tcp, priority 110, uuid 8091b92c reg0 = 11.0.0.11; reg9[16..31] = tcp.dst; ct_dnat; ct_dnat /* assuming no un-dnat entry, so no change */ ----------------------------------------------------- 10. lr_in_ip_routing: no match (implicit drop) [root@bz_2017540 ~]# ovn-trace --ct new --minimal 'eth.dst == 00:00:00:00:01:01 && inport == "rtr-ls" && ip4.src == 10.0.0.10 && ip4.dst == 11.0.0.11 && ip.ttl == 64 && tcp && tcp.dst == 1111' --lb-dst 10.0.0.10:1010 # tcp,reg14=0x1,vlan_tci=0x0000,dl_src=00:00:00:00:00:00,dl_dst=00:00:00:00:01:01,nw_src=10.0.0.10,nw_dst=11.0.0.11,nw_tos=0,nw_ecn=0,nw_ttl=64,tp_src=0,tp_dst=1111,tcp_flags=0 ct_dnat /* assuming no un-dnat entry, so no change */; <========== OVN trace doesn't following register value matches when tracing ########## Verified on ######## [root@bz_2017540 ~]# rpm -qa |grep -E "ovn|openvswitch" openvswitch2.15-2.15.0-53.el8fdp.x86_64 ovn-2021-central-21.09.1-23.el8fdp.x86_64 openvswitch-selinux-extra-policy-1.0-28.el8fdp.noarch ovn-2021-host-21.09.1-23.el8fdp.x86_64 ovn-2021-21.09.1-23.el8fdp.x86_64 [root@bz_2017540 ~]# ovn-trace --ct new 'eth.dst == 00:00:00:00:01:01 && inport == "rtr-ls" && ip4.src == 10.0.0.10 && ip4.dst == 11.0.0.11 && ip.ttl == 64 && tcp && tcp.dst == 1111' --lb-dst 10.0.0.10:1010 # tcp,reg14=0x1,vlan_tci=0x0000,dl_src=00:00:00:00:00:00,dl_dst=00:00:00:00:01:01,nw_src=10.0.0.10,nw_dst=11.0.0.11,nw_tos=0,nw_ecn=0,nw_ttl=64,tp_src=0,tp_dst=1111,tcp_flags=0 ingress(dp="rtr", inport="rtr-ls") ---------------------------------- 0. lr_in_admission (northd.c:10285): eth.dst == 00:00:00:00:01:01 && inport == "rtr-ls", priority 50, uuid 91748199 xreg0[0..47] = 00:00:00:00:01:01; next; 1. lr_in_lookup_neighbor (northd.c:10365): 1, priority 0, uuid 44529e18 reg9[2] = 1; next; 2. lr_in_learn_neighbor (northd.c:10374): reg9[2] == 1, priority 100, uuid 7f725256 next; 5. lr_in_defrag (northd.c:9597): ip && ip4.dst == 11.0.0.11 && tcp, priority 110, uuid cb39f60c reg0 = 11.0.0.11; reg9[16..31] = tcp.dst; ct_dnat; ct_dnat /* assuming no un-dnat entry, so no change */ ----------------------------------------------------- 6. lr_in_dnat (northd.c:9447): ct.new && ip4 && reg0 == 11.0.0.11 && tcp && reg9[16..31] == 1111, priority 120, uuid 6115420f ct_lb(backends=10.0.0.2:1010); ct_lb ----- 10. lr_in_ip_routing (northd.c:9179): ip4.dst == 10.0.0.0/24, priority 49, uuid f6b8c0c6 ip.ttl--; reg8[0..15] = 0; reg0 = ip4.dst; reg1 = 10.0.0.1; eth.src = 00:00:00:00:01:01; outport = "rtr-ls"; flags.loopback = 1; next; 11. lr_in_ip_routing_ecmp (northd.c:10670): reg8[0..15] == 0, priority 150, uuid 1da5c95d next; 12. lr_in_policy (northd.c:10795): 1, priority 0, uuid 35bedcdf reg8[0..15] = 0; next; 13. lr_in_policy_ecmp (northd.c:10797): reg8[0..15] == 0, priority 150, uuid 3af5585c next; 14. lr_in_arp_resolve (northd.c:10831): ip4, priority 0, uuid c90fe64f get_arp(outport, reg0); /* No MAC binding. */ next; 18. lr_in_arp_request (northd.c:11479): eth.dst == 00:00:00:00:00:00 && ip4, priority 100, uuid bf8b5949 arp { eth.dst = ff:ff:ff:ff:ff:ff; arp.spa = reg1; arp.tpa = reg0; arp.op = 1; output; }; arp --- eth.dst = ff:ff:ff:ff:ff:ff; arp.spa = reg1; arp.tpa = reg0; arp.op = 1; output; egress(dp="rtr", inport="rtr-ls", outport="rtr-ls") --------------------------------------------------- 4. lr_out_delivery (northd.c:11536): outport == "rtr-ls", priority 100, uuid dbb5895d output; /* output to "rtr-ls", type "l3gateway" */ [root@bz_2017540 ~]# ovn-trace --ct new --minimal 'eth.dst == 00:00:00:00:01:01 && inport == "rtr-ls" && ip4.src == 10.0.0.10 && ip4.dst == 11.0.0.11 && ip.ttl == 64 && tcp && tcp.dst == 1111' --lb-dst 10.0.0.10:1010 # tcp,reg14=0x1,vlan_tci=0x0000,dl_src=00:00:00:00:00:00,dl_dst=00:00:00:00:01:01,nw_src=10.0.0.10,nw_dst=11.0.0.11,nw_tos=0,nw_ecn=0,nw_ttl=64,tp_src=0,tp_dst=1111,tcp_flags=0 ct_dnat /* assuming no un-dnat entry, so no change */ { ct_lb { ip.ttl--; eth.src = 00:00:00:00:01:01; eth.dst = 00:00:00:00:00:00; arp { eth.dst = ff:ff:ff:ff:ff:ff; arp.spa = 0xa000001; arp.tpa = 0xa000002; arp.op = 1; output("rtr-ls"); }; }; }; <========== OVN trace following register value matches when tracing on table=6 (lr_in_dnat ), priority=120 , match=(ct.new && ip4 && reg0 == 11.0.0.11 && tcp && reg9[16..31] == 1111), action=(ct_lb(backends=10.0.0.2:1010);)
Since the problem described in this bug report should be resolved in a recent advisory, it has been closed with a resolution of ERRATA. For information on the advisory (ovn bug fix and enhancement update), and where to find the updated files, follow the link below. If the solution does not work for you, open a new bug report. https://access.redhat.com/errata/RHBA-2022:0049