Note: This bug is displayed in read-only format because the product is no longer active in Red Hat Bugzilla.
The FDP team is no longer accepting new bugs in Bugzilla. Please report your issues under FDP project in Jira. Thanks.

Bug 2017540

Summary: [ovn-trace] OVN trace doesn't set ct_state for ct_nat not preceded by ct_next
Product: Red Hat Enterprise Linux Fast Datapath Reporter: Surya Seetharaman <surya>
Component: OVNAssignee: Dumitru Ceara <dceara>
Status: CLOSED ERRATA QA Contact: Ehsan Elahi <eelahi>
Severity: unspecified Docs Contact:
Priority: unspecified    
Version: FDP 21.CCC: ctrautma, dceara, fleitner, jiji
Target Milestone: ---   
Target Release: ---   
Hardware: Unspecified   
OS: Unspecified   
Whiteboard:
Fixed In Version: ovn-2021-21.09.1-20.el8fdp Doc Type: If docs needed, set a value
Doc Text:
Story Points: ---
Clone Of: Environment:
Last Closed: 2022-01-10 16:49:01 UTC Type: Bug
Regression: --- Mount Type: ---
Documentation: --- CRM:
Verified Versions: Category: ---
oVirt Team: --- RHEL 7.3 requirements from Atomic Host:
Cloudforms Team: --- Target Upstream Version:
Embargoed:

Description Surya Seetharaman 2021-10-26 18:44:26 UTC 
Description of problem:

OVN trace doesn't respect register value matches when tracing. Example: Trace doesn't recognize that in table 6, we have  reg9[16..31] == 32398 (which is the tcp.dstport): 

ovn-sbctl --no-leader-only lflow-list GR_ovn-worker2 | grep lr_in_dnat
table=5 (lr_in_defrag       ), priority=110  , match=(ip && ip4.dst == 10.96.0.1 && tcp), action=(reg0 = 10.96.0.1; reg9[16..31] = tcp.dst; ct_dnat;)
  table=5 (lr_in_defrag       ), priority=110  , match=(ip && ip4.dst == 10.96.0.10 && tcp), action=(reg0 = 10.96.0.10; reg9[16..31] = tcp.dst; ct_dnat;)
  table=5 (lr_in_defrag       ), priority=110  , match=(ip && ip4.dst == 10.96.0.10 && udp), action=(reg0 = 10.96.0.10; reg9[16..31] = udp.dst; ct_dnat;)
  table=5 (lr_in_defrag       ), priority=110  , match=(ip && ip4.dst == 10.96.120.53 && tcp), action=(reg0 = 10.96.120.53; reg9[16..31] = tcp.dst; ct_dnat;)
  table=5 (lr_in_defrag       ), priority=110  , match=(ip && ip4.dst == 172.18.0.3 && tcp), action=(reg0 = 172.18.0.3; reg9[16..31] = tcp.dst; ct_dnat;)
  table=5 (lr_in_defrag       ), priority=0    , match=(1), action=(next;)
  table=6 (lr_in_dnat         ), priority=120  , match=(ct.est && ip4 && reg0 == 10.96.0.1 && tcp && reg9[16..31] == 443 && ct_label.natted == 1), action=(flags.force_snat_for_lb = 1; next;)
  table=6 (lr_in_dnat         ), priority=120  , match=(ct.est && ip4 && reg0 == 10.96.0.10 && tcp && reg9[16..31] == 53 && ct_label.natted == 1), action=(flags.force_snat_for_lb = 1; next;)
  table=6 (lr_in_dnat         ), priority=120  , match=(ct.est && ip4 && reg0 == 10.96.0.10 && tcp && reg9[16..31] == 9153 && ct_label.natted == 1), action=(flags.force_snat_for_lb = 1; next;)
  table=6 (lr_in_dnat         ), priority=120  , match=(ct.est && ip4 && reg0 == 10.96.0.10 && udp && reg9[16..31] == 53 && ct_label.natted == 1), action=(flags.force_snat_for_lb = 1; next;)
  table=6 (lr_in_dnat         ), priority=120  , match=(ct.est && ip4 && reg0 == 10.96.120.53 && tcp && reg9[16..31] == 8080 && ct_label.natted == 1), action=(flags.force_snat_for_lb = 1; next;)
  table=6 (lr_in_dnat         ), priority=120  , match=(ct.est && ip4 && reg0 == 172.18.0.3 && tcp && reg9[16..31] == 32398 && ct_label.natted == 1), action=(flags.force_snat_for_lb = 1; next;)
  table=6 (lr_in_dnat         ), priority=120  , match=(ct.new && ip4 && reg0 == 10.96.0.1 && tcp && reg9[16..31] == 443), action=(flags.force_snat_for_lb = 1; ct_lb(backends=172.18.0.4:6443);)
  table=6 (lr_in_dnat         ), priority=120  , match=(ct.new && ip4 && reg0 == 10.96.0.10 && tcp && reg9[16..31] == 53), action=(flags.force_snat_for_lb = 1; ct_lb(backends=10.244.0.3:53,10.244.2.4:53);)
  table=6 (lr_in_dnat         ), priority=120  , match=(ct.new && ip4 && reg0 == 10.96.0.10 && tcp && reg9[16..31] == 9153), action=(flags.force_snat_for_lb = 1; ct_lb(backends=10.244.0.3:9153,10.244.2.4:9153);)
  table=6 (lr_in_dnat         ), priority=120  , match=(ct.new && ip4 && reg0 == 10.96.0.10 && udp && reg9[16..31] == 53), action=(flags.force_snat_for_lb = 1; ct_lb(backends=10.244.0.3:53,10.244.2.4:53);)
  table=6 (lr_in_dnat         ), priority=120  , match=(ct.new && ip4 && reg0 == 10.96.120.53 && tcp && reg9[16..31] == 8080), action=(flags.force_snat_for_lb = 1; ct_lb(backends=10.244.0.4:8080,10.244.2.5:8080);)
  table=6 (lr_in_dnat         ), priority=120  , match=(ct.new && ip4 && reg0 == 172.18.0.3 && tcp && reg9[16..31] == 32398), action=(flags.force_snat_for_lb = 1; ct_lb(backends=10.244.0.4:8080,10.244.2.5:8080);)
  table=6 (lr_in_dnat         ), priority=0    , match=(1), action=(next;)


Trace command and output snippet:

sh-5.1# ovn-trace --ct new 'eth.dst == 0a:58:64:40:00:03 && inport == "rtoj-GR_ovn-worker2" && ip4.src==8.8.8.8 && ip4.dst==172.18.0.3 && ip.ttl==64 && tcp && tcp.src==41314 && tcp.dst==32398' --lb-dst 10.244.2.5:8080
# tcp,reg14=0x1,vlan_tci=0x0000,dl_src=00:00:00:00:00:00,dl_dst=0a:58:64:40:00:03,nw_src=8.8.8.8,nw_dst=172.18.0.3,nw_tos=0,nw_ecn=0,nw_ttl=64,tp_src=41314,tp_dst=32398,tcp_flags=0

ingress(dp="GR_ovn-worker2", inport="rtoj-GR_ovn-worker2")
----------------------------------------------------------
 0. lr_in_admission (northd.c:10374): eth.dst == 0a:58:64:40:00:03 && inport == "rtoj-GR_ovn-worker2", priority 50, uuid 26beac16
    xreg0[0..47] = 0a:58:64:40:00:03;
    next;
 1. lr_in_lookup_neighbor (northd.c:10455): 1, priority 0, uuid cda3ef89
    reg9[2] = 1;
    next;
 2. lr_in_learn_neighbor (northd.c:10464): reg9[2] == 1 || reg9[3] == 0, priority 100, uuid 47cabf81
    next;
 4. lr_in_unsnat (northd.c:9429): ip4 && ip4.dst == 172.18.0.3 && tcp && tcp.dst == 32398, priority 120, uuid bae81a29
    next;
 5. lr_in_defrag (northd.c:9623): ip && ip4.dst == 172.18.0.3 && tcp, priority 110, uuid e3512bce
    reg0 = 172.18.0.3;
    reg9[16..31] = tcp.dst;
    ct_dnat;

ct_dnat /* assuming no un-dnat entry, so no change */
-----------------------------------------------------
10. lr_in_ip_routing (northd.c:9203): ip4.dst == 172.18.0.0/16, priority 33, uuid f3f06084
    ip.ttl--;
    reg8[0..15] = 0;
    reg0 = ip4.dst;
    reg1 = 172.18.0.3;
    eth.src = 02:42:ac:12:00:03;
    outport = "rtoe-GR_ovn-worker2";
    flags.loopback = 1;
    next;
11. lr_in_ip_routing_ecmp (northd.c:10760): reg8[0..15] == 0, priority 150, uuid bde3d1a8
    next;
12. lr_in_policy (northd.c:10885): 1, priority 0, uuid 3bc938fb
    reg8[0..15] = 0;
    next;
13. lr_in_policy_ecmp (northd.c:10887): reg8[0..15] == 0, priority 150, uuid 5b040850
    next;
14. lr_in_arp_resolve (northd.c:10125): ip4.dst == {172.18.0.3}, priority 1, uuid 4e24ca9f
    drop;

Version-Release number of selected component (if applicable):
ARG ovnver=ovn-21.09.0-3.fc33

How reproducible: Always


Expected results:

It should match on

 table=6 (lr_in_dnat         ), priority=120  , match=(ct.new && ip4 && reg0 == 172.18.0.3 && tcp && reg9[16..31] == 32398), action=(flags.force_snat_for_lb = 1; ct_lb(backends=10.244.0.4:8080,10.244.2.5:8080);)
Comment 1 Dumitru Ceara 2021-11-01 13:38:43 UTC 
Hi Surya,

I dug more into this, the problem is actually that tracing 'ct_dnat' that's not preceded by a 'ct_next' in the pipeline will incorrectly skip setting the ct_state in the microflow.  This happens since ct()/ct(nat) have been merged in https://github.com/ovn-org/ovn/commit/0038579d192802fff03c3594e4f85dab4f7af2bd

I posted a patch to fix this and updated the summary of this BZ:

http://patchwork.ozlabs.org/project/ovn/list/?series=269816&state=*

Thanks,
Dumitru
Comment 4 Ehsan Elahi 2021-12-16 12:59:48 UTC 
Reproduced in
[root@bz_2017540 ~]# rpm -qa |grep -E "ovn|openvswitch"
openvswitch-selinux-extra-policy-1.0-28.el8fdp.noarch
ovn-2021-21.09.0-5.el8fdp.x86_64
ovn-2021-central-21.09.0-5.el8fdp.x86_64
ovn-2021-host-21.09.0-5.el8fdp.x86_64
openvswitch2.15-2.15.0-50.el8fdp.x86_64

#### Steps to reproduce as below ####
systemctl start ovn-northd
ovn-nbctl set-connection ptcp:6641
ovn-sbctl set-connection ptcp:6642
systemctl start openvswitch
ovs-vsctl set open . external_ids:system-id=hv1
ovs-vsctl set open . external_ids:ovn-remote=tcp:42.42.42.1:6642
ovs-vsctl set open . external_ids:ovn-encap-type=geneve
ovs-vsctl set open . external_ids:ovn-encap-ip=42.42.42.1
systemctl start ovn-controller

ovn-nbctl lr-add rtr
ovn-nbctl set logical_router rtr options:chassis=hv1
ovn-nbctl lrp-add rtr rtr-ls 00:00:00:00:01:01 10.0.0.1/24
ovn-nbctl lb-add lb-test 11.0.0.11:1111 10.0.0.2:1010 tcp
ovn-nbctl lr-lb-add rtr lb-test

ovn-nbctl --wait=sb sync

[root@bz_2017540 ~]# ovn-sbctl --no-leader-only lflow-list rtr | grep lr_in_dnat
  table=6 (lr_in_dnat         ), priority=120  , match=(ct.est && ip4 && reg0 == 11.0.0.11 && tcp && reg9[16..31] == 1111 && ct_label.natted == 1), action=(next;)
  table=6 (lr_in_dnat         ), priority=120  , match=(ct.new && ip4 && reg0 == 11.0.0.11 && tcp && reg9[16..31] == 1111), action=(ct_lb(backends=10.0.0.2:1010);)
  table=6 (lr_in_dnat         ), priority=0    , match=(1), action=(next;)

[root@bz_2017540 ~]# ovn-trace --ct new 'eth.dst == 00:00:00:00:01:01 && inport == "rtr-ls" && ip4.src == 10.0.0.10 && ip4.dst == 11.0.0.11 && ip.ttl == 64 && tcp && tcp.dst == 1111' --lb-dst 10.0.0.10:1010

# tcp,reg14=0x1,vlan_tci=0x0000,dl_src=00:00:00:00:00:00,dl_dst=00:00:00:00:01:01,nw_src=10.0.0.10,nw_dst=11.0.0.11,nw_tos=0,nw_ecn=0,nw_ttl=64,tp_src=0,tp_dst=1111,tcp_flags=0

ingress(dp="rtr", inport="rtr-ls")
----------------------------------
 0. lr_in_admission (northd.c:10294): eth.dst == 00:00:00:00:01:01 && inport == "rtr-ls", priority 50, uuid 1b22c640
    xreg0[0..47] = 00:00:00:00:01:01;
    next;
 1. lr_in_lookup_neighbor (northd.c:10374): 1, priority 0, uuid bd05decb
    reg9[2] = 1;
    next;
 2. lr_in_learn_neighbor (northd.c:10383): reg9[2] == 1, priority 100, uuid 4ebf265a
    next;
 5. lr_in_defrag (northd.c:9541): ip && ip4.dst == 11.0.0.11 && tcp, priority 110, uuid 8091b92c
    reg0 = 11.0.0.11;
    reg9[16..31] = tcp.dst;
    ct_dnat;

ct_dnat /* assuming no un-dnat entry, so no change */
-----------------------------------------------------
10. lr_in_ip_routing: no match (implicit drop)

[root@bz_2017540 ~]# ovn-trace --ct new --minimal 'eth.dst == 00:00:00:00:01:01 && inport == "rtr-ls" && ip4.src == 10.0.0.10 && ip4.dst == 11.0.0.11 && ip.ttl == 64 && tcp && tcp.dst == 1111' --lb-dst 10.0.0.10:1010
# tcp,reg14=0x1,vlan_tci=0x0000,dl_src=00:00:00:00:00:00,dl_dst=00:00:00:00:01:01,nw_src=10.0.0.10,nw_dst=11.0.0.11,nw_tos=0,nw_ecn=0,nw_ttl=64,tp_src=0,tp_dst=1111,tcp_flags=0
ct_dnat /* assuming no un-dnat entry, so no change */;

<========== OVN trace doesn't following register value matches when tracing

########## Verified on ########
[root@bz_2017540 ~]# rpm -qa |grep -E "ovn|openvswitch"
openvswitch2.15-2.15.0-53.el8fdp.x86_64
ovn-2021-central-21.09.1-23.el8fdp.x86_64
openvswitch-selinux-extra-policy-1.0-28.el8fdp.noarch
ovn-2021-host-21.09.1-23.el8fdp.x86_64
ovn-2021-21.09.1-23.el8fdp.x86_64

[root@bz_2017540 ~]# ovn-trace --ct new 'eth.dst == 00:00:00:00:01:01 && inport == "rtr-ls" && ip4.src == 10.0.0.10 && ip4.dst == 11.0.0.11 && ip.ttl == 64 && tcp && tcp.dst == 1111' --lb-dst 10.0.0.10:1010

# tcp,reg14=0x1,vlan_tci=0x0000,dl_src=00:00:00:00:00:00,dl_dst=00:00:00:00:01:01,nw_src=10.0.0.10,nw_dst=11.0.0.11,nw_tos=0,nw_ecn=0,nw_ttl=64,tp_src=0,tp_dst=1111,tcp_flags=0

ingress(dp="rtr", inport="rtr-ls")
----------------------------------
 0. lr_in_admission (northd.c:10285): eth.dst == 00:00:00:00:01:01 && inport == "rtr-ls", priority 50, uuid 91748199
    xreg0[0..47] = 00:00:00:00:01:01;
    next;
 1. lr_in_lookup_neighbor (northd.c:10365): 1, priority 0, uuid 44529e18
    reg9[2] = 1;
    next;
 2. lr_in_learn_neighbor (northd.c:10374): reg9[2] == 1, priority 100, uuid 7f725256
    next;
 5. lr_in_defrag (northd.c:9597): ip && ip4.dst == 11.0.0.11 && tcp, priority 110, uuid cb39f60c
    reg0 = 11.0.0.11;
    reg9[16..31] = tcp.dst;
    ct_dnat;

ct_dnat /* assuming no un-dnat entry, so no change */
-----------------------------------------------------
 6. lr_in_dnat (northd.c:9447): ct.new && ip4 && reg0 == 11.0.0.11 && tcp && reg9[16..31] == 1111, priority 120, uuid 6115420f
    ct_lb(backends=10.0.0.2:1010);

ct_lb
-----
10. lr_in_ip_routing (northd.c:9179): ip4.dst == 10.0.0.0/24, priority 49, uuid f6b8c0c6
    ip.ttl--;
    reg8[0..15] = 0;
    reg0 = ip4.dst;
    reg1 = 10.0.0.1;
    eth.src = 00:00:00:00:01:01;
    outport = "rtr-ls";
    flags.loopback = 1;
    next;
11. lr_in_ip_routing_ecmp (northd.c:10670): reg8[0..15] == 0, priority 150, uuid 1da5c95d
    next;
12. lr_in_policy (northd.c:10795): 1, priority 0, uuid 35bedcdf
    reg8[0..15] = 0;
    next;
13. lr_in_policy_ecmp (northd.c:10797): reg8[0..15] == 0, priority 150, uuid 3af5585c
    next;
14. lr_in_arp_resolve (northd.c:10831): ip4, priority 0, uuid c90fe64f
    get_arp(outport, reg0);
    /* No MAC binding. */
    next;
18. lr_in_arp_request (northd.c:11479): eth.dst == 00:00:00:00:00:00 && ip4, priority 100, uuid bf8b5949
    arp { eth.dst = ff:ff:ff:ff:ff:ff; arp.spa = reg1; arp.tpa = reg0; arp.op = 1; output; };

arp
---
    eth.dst = ff:ff:ff:ff:ff:ff;
    arp.spa = reg1;
    arp.tpa = reg0;
    arp.op = 1;
    output;

egress(dp="rtr", inport="rtr-ls", outport="rtr-ls")
---------------------------------------------------
 4. lr_out_delivery (northd.c:11536): outport == "rtr-ls", priority 100, uuid dbb5895d
    output;
    /* output to "rtr-ls", type "l3gateway" */


[root@bz_2017540 ~]# ovn-trace --ct new --minimal 'eth.dst == 00:00:00:00:01:01 && inport == "rtr-ls" && ip4.src == 10.0.0.10 && ip4.dst == 11.0.0.11 && ip.ttl == 64 && tcp && tcp.dst == 1111' --lb-dst 10.0.0.10:1010

# tcp,reg14=0x1,vlan_tci=0x0000,dl_src=00:00:00:00:00:00,dl_dst=00:00:00:00:01:01,nw_src=10.0.0.10,nw_dst=11.0.0.11,nw_tos=0,nw_ecn=0,nw_ttl=64,tp_src=0,tp_dst=1111,tcp_flags=0
ct_dnat /* assuming no un-dnat entry, so no change */ {
    ct_lb {
        ip.ttl--;
        eth.src = 00:00:00:00:01:01;
        eth.dst = 00:00:00:00:00:00;
        arp {
            eth.dst = ff:ff:ff:ff:ff:ff;
            arp.spa = 0xa000001;
            arp.tpa = 0xa000002;
            arp.op = 1;
            output("rtr-ls");
        };
    };
};

<========== OVN trace following register value matches when tracing on 
table=6 (lr_in_dnat         ), priority=120  , match=(ct.new && ip4 && reg0 == 11.0.0.11 && tcp && reg9[16..31] == 1111), action=(ct_lb(backends=10.0.0.2:1010);)
Comment 6 errata-xmlrpc 2022-01-10 16:49:01 UTC 
Since the problem described in this bug report should be
resolved in a recent advisory, it has been closed with a
resolution of ERRATA.

For information on the advisory (ovn bug fix and enhancement update), and where to find the updated
files, follow the link below.

If the solution does not work for you, open a new bug report.

https://access.redhat.com/errata/RHBA-2022:0049