Note: This bug is displayed in read-only format because the product is no longer active in Red Hat Bugzilla.
The FDP team is no longer accepting new bugs in Bugzilla. Please report your issues under FDP project in Jira. Thanks.

Bug 1626080

Summary: ovn-trace does not trace past an l3gateway port type
Product: Red Hat Enterprise Linux Fast Datapath Reporter: Dan Williams <dcbw>
Component: ovn2.11Assignee: OVN Team <ovnteam>
Status: CLOSED UPSTREAM QA Contact: haidong li <haili>
Severity: medium Docs Contact:
Priority: medium    
Version: FDP 19.03CC: ctrautma, haili, jishi, ovs-qe, qding
Target Milestone: ---   
Target Release: FDP 19.03   
Hardware: Unspecified   
OS: Unspecified   
Whiteboard:
Fixed In Version: Doc Type: If docs needed, set a value
Doc Text:
Story Points: ---
Clone Of: Environment:
Last Closed: 2025-02-10 03:59:26 UTC Type: Bug
Regression: --- Mount Type: ---
Documentation: --- CRM:
Verified Versions: Category: ---
oVirt Team: --- RHEL 7.3 requirements from Atomic Host:
Cloudforms Team: --- Target Upstream Version:
Embargoed:

Description Dan Williams 2018-09-06 14:17:11 UTC 
It seems odd that ovn-trace doesn't trace the datapath past an l3gateway port, when there are often clearly datapaths beyond that port.  For example:

$ ovn-trace --db=tcp:172.17.0.2:6642 425f5827-5676-4e61-ba14-06151f656d8e 'inport=="default_nginx1" && eth.dst == 00:00:00:8C:5E:78 && eth.src == 0a:00:00:00:00:01 && ip4.dst == 4.2.2.1 && ip4.src == 10.128.2.3 && ip.ttl==64'

# ip,reg14=0x3,vlan_tci=0x0000,dl_src=0a:00:00:00:00:01,dl_dst=00:00:00:8c:5e:78,nw_src=10.128.2.3,nw_dst=4.2.2.1,nw_proto=0,nw_tos=0,nw_ecn=0,nw_ttl=64

ingress(dp="openshift-node-2", inport="default_nginx1")
-------------------------------------------------------
 0. ls_in_port_sec_l2 (ovn-northd.c:3556): inport == "default_nginx1", priority 50, uuid 2cce4bab
    next;
 3. ls_in_pre_acl (ovn-northd.c:2882): ip, priority 100, uuid bd88b866
    reg0[0] = 1;
    next;
 5. ls_in_pre_stateful (ovn-northd.c:3016): reg0[0] == 1, priority 100, uuid 714b5e3a
    ct_next;

ct_next(ct_state=est|trk /* default (use --ct to customize) */)
---------------------------------------------------------------
15. ls_in_l2_lkup (ovn-northd.c:3887): eth.dst == 00:00:00:8c:5e:78, priority 50, uuid f32229e3
    outport = "stor-openshift-node-2";
    output;

egress(dp="openshift-node-2", inport="default_nginx1", outport="stor-openshift-node-2")
---------------------------------------------------------------------------------------
 1. ls_out_pre_acl (ovn-northd.c:2845): ip && outport == "stor-openshift-node-2", priority 110, uuid 32d01467
    next;
 8. ls_out_port_sec_l2 (ovn-northd.c:4012): outport == "stor-openshift-node-2", priority 50, uuid 8b4b6f00
    output;
    /* output to "stor-openshift-node-2", type "patch" */

ingress(dp="openshift-master", inport="rtos-openshift-node-2")
--------------------------------------------------------------
 0. lr_in_admission (ovn-northd.c:4429): eth.dst == 00:00:00:8c:5e:78 && inport == "rtos-openshift-node-2", priority 50, uuid c72ac7e3
    next;
 5. lr_in_ip_routing (ovn-northd.c:4140): ip4.src == 10.128.2.0/24, priority 48, uuid 8db21e17
    ip.ttl--;
    reg0 = 100.64.1.2;
    reg1 = 100.64.1.1;
    eth.src = 00:00:00:b8:df:45;
    outport = "rtoj-openshift-master";
    flags.loopback = 1;
    next;
 6. lr_in_arp_resolve (ovn-northd.c:5528): outport == "rtoj-openshift-master" && reg0 == 100.64.1.2, priority 100, uuid bdff17f1
    eth.dst = 00:00:00:48:22:33;
    next;
 8. lr_in_arp_request (ovn-northd.c:5618): 1, priority 0, uuid 921ae0a6
    output;

egress(dp="openshift-master", inport="rtos-openshift-node-2", outport="rtoj-openshift-master")
----------------------------------------------------------------------------------------------
 3. lr_out_delivery (ovn-northd.c:5646): outport == "rtoj-openshift-master", priority 100, uuid b2da5b59
    output;
    /* output to "rtoj-openshift-master", type "patch" */

ingress(dp="join", inport="jtor-openshift-master")
--------------------------------------------------
 0. ls_in_port_sec_l2 (ovn-northd.c:3556): inport == "jtor-openshift-master", priority 50, uuid 72776597
    next;
15. ls_in_l2_lkup (ovn-northd.c:3887): eth.dst == 00:00:00:48:22:33, priority 50, uuid 0205e709
    outport = "jtor-GR_openshift-node-2";
    output;

egress(dp="join", inport="jtor-openshift-master", outport="jtor-GR_openshift-node-2")
-------------------------------------------------------------------------------------
 8. ls_out_port_sec_l2 (ovn-northd.c:4012): outport == "jtor-GR_openshift-node-2", priority 50, uuid db67e8ba
    output;
    /* output to "jtor-GR_openshift-node-2", type "l3gateway" */


jtor-GR_openshift-node-2 is an l3gateway port bound to a specific chassis, but in this logical network there are some datapaths after it.  If I do the following patch:

diff -up openvswitch-2.8.1/ovn/utilities/ovn-trace.c.foo openvswitch-2.8.1/ovn/utilities/ovn-trace.c
--- openvswitch-2.8.1/ovn/utilities/ovn-trace.c.foo	2018-09-06 08:59:27.045863050 -0500
+++ openvswitch-2.8.1/ovn/utilities/ovn-trace.c	2018-09-06 09:02:50.461278060 -0500
@@ -599,7 +599,7 @@ read_ports(void)
                                : shorten_uuid(port->name2
                                               ? port->name2 : port->name));
 
-        if (!strcmp(sbpb->type, "patch")) {
+        if (!strcmp(sbpb->type, "patch") || !strcmp(sbpb->type, "l3gateway")) {
             const char *peer_name = smap_get(&sbpb->options, "peer");
             if (peer_name) {
                 struct ovntrace_port *peer

Then I get much further, allowing me to debug my issue:

...
egress(dp="join", inport="jtor-openshift-master", outport="jtor-GR_openshift-node-2")
-------------------------------------------------------------------------------------
 8. ls_out_port_sec_l2 (ovn-northd.c:4012): outport == "jtor-GR_openshift-node-2", priority 50, uuid db67e8ba
    output;
    /* output to "jtor-GR_openshift-node-2", type "l3gateway" */

ingress(dp="GR_openshift-node-2", inport="rtoj-GR_openshift-node-2")
--------------------------------------------------------------------
 0. lr_in_admission (ovn-northd.c:4429): eth.dst == 00:00:00:48:22:33 && inport == "rtoj-GR_openshift-node-2", priority 50, uuid 2842ff80
    next;
 4. lr_in_dnat (ovn-northd.c:5211): ip, priority 50, uuid f30ee742
    flags.loopback = 1;
    ct_dnat;

ct_dnat /* assuming no un-dnat entry, so no change */
-----------------------------------------------------
 5. lr_in_ip_routing (ovn-northd.c:4140): ip4.dst == 0.0.0.0/0, priority 1, uuid 898062d1
    ip.ttl--;
    reg0 = 172.17.0.1;
    reg1 = 172.17.0.4;
    eth.src = 02:42:ac:11:00:04;
    outport = "rtoe-GR_openshift-node-2";
    flags.loopback = 1;
    next;
 6. lr_in_arp_resolve (ovn-northd.c:5553): ip4, priority 0, uuid 355779a8
    get_arp(outport, reg0);
    /* No MAC binding. */
    next;
 8. lr_in_arp_request (ovn-northd.c:5617): eth.dst == 00:00:00:00:00:00, priority 100, uuid 41e1dfb2
    arp { eth.dst = ff:ff:ff:ff:ff:ff; arp.spa = reg1; arp.tpa = reg0; arp.op = 1; output; };

arp
---
    eth.dst = ff:ff:ff:ff:ff:ff;
    arp.spa = reg1;
    arp.tpa = reg0;
    arp.op = 1;
    output;

egress(dp="GR_openshift-node-2", inport="rtoj-GR_openshift-node-2", outport="rtoe-GR_openshift-node-2")
-------------------------------------------------------------------------------------------------------
 3. lr_out_delivery (ovn-northd.c:5646): outport == "rtoe-GR_openshift-node-2", priority 100, uuid 965f84f9
    output;
    /* output to "rtoe-GR_openshift-node-2", type "l3gateway" */

ingress(dp="ext_openshift-node-2", inport="etor-GR_openshift-node-2")
---------------------------------------------------------------------
 0. ls_in_port_sec_l2 (ovn-northd.c:3556): inport == "etor-GR_openshift-node-2", priority 50, uuid 83b2854c
    next;
15. ls_in_l2_lkup (ovn-northd.c:3864): eth.mcast, priority 100, uuid e5943c49
    outport = "_MC_flood";
    output;

multicast(dp="ext_openshift-node-2", mcgroup="_MC_flood")
---------------------------------------------------------

    egress(dp="ext_openshift-node-2", inport="etor-GR_openshift-node-2", outport="etor-GR_openshift-node-2")
    --------------------------------------------------------------------------------------------------------
            /* omitting output because inport == outport && !flags.loopback */

    egress(dp="ext_openshift-node-2", inport="etor-GR_openshift-node-2", outport="ext_openshift-node-2")
    ----------------------------------------------------------------------------------------------------
         8. ls_out_port_sec_l2 (ovn-northd.c:3989): eth.mcast, priority 100, uuid 33f4ed7a
            output;
            /* output to "ext_openshift-node-2", type "" */
Comment 2 Numan Siddique 2018-10-09 13:12:55 UTC 
Submitted the patch to fix the issue - https://patchwork.ozlabs.org/patch/981211/
Comment 3 haidong li 2019-04-26 02:31:56 UTC 
verified on the latest version:
[root@hp-dl380pg8-04 ovn_ha]# rpm -qa | grep ovn
ovn2.11-host-2.11.0-9.el7fdp.1.x86_64
puppet-ovn-13.3.1-1.el7.noarch
ovn2.11-2.11.0-9.el7fdp.1.x86_64
kernel-kernel-networking-openvswitch-ovn-1.0-121.noarch
openstack-nova-novncproxy-18.2.0-1.el7.noarch
novnc-0.5.1-2.el7.noarch
kernel-kernel-networking-openvswitch-ovn_ha-1.0-34.noarch
ovn2.11-central-2.11.0-9.el7fdp.1.x86_64
[root@hp-dl380pg8-04 ovn_ha]# rpm -qa | grep openvswitch
kernel-kernel-networking-openvswitch-ovn-1.0-121.noarch
openvswitch2.11-2.11.0-5.el7fdp.x86_64
openvswitch-selinux-extra-policy-1.0-11.el7fdp.noarch
kernel-kernel-networking-openvswitch-ovn_ha-1.0-34.noarch

[root@hp-dl380pg8-04 ~]#  ovn-trace --all r1 'inport == "r1_s2" && eth.src == 00e:ad:01:00:01 && eth.dst == 00e:ad:ff:01:02 && ip4.src == 172.16.102.11 && ip4.dst == 172.16.103.11 && ip.ttl==64'
# ip,reg14=0x1,vlan_tci=0x0000,dl_src=00e:ad:01:00:01,dl_dst=00e:ad:ff:01:02,nw_src=172.16.102.11,nw_dst=172.16.103.11,nw_proto=0,nw_tos=0,nw_ecn=0,nw_ttl=64
# Detailed trace.

ingress(dp="r1", inport="r1_s2")
--------------------------------
 0. lr_in_admission (ovn-northd.c:5150): eth.dst == 00e:ad:ff:01:02 && inport == "r1_s2", priority 50, uuid 36fcfcef
    next;
 4. lr_in_dnat (ovn-northd.c:6133): ip, priority 50, uuid 54bdd2df
    flags.loopback = 1;
    ct_dnat;

ct_dnat /* assuming no un-dnat entry, so no change */
-----------------------------------------------------
 7. lr_in_ip_routing (ovn-northd.c:4732): ip4.dst == 172.16.103.0/24, priority 49, uuid d8c02f67
    ip.ttl--;
    reg0 = ip4.dst;
    reg1 = 172.16.103.1;
    eth.src = 00e:ad:ff:01:03;
    outport = "r1_s3";
    flags.loopback = 1;
    next;
 8. lr_in_arp_resolve (ovn-northd.c:6592): ip4, priority 0, uuid 7378bce3
    get_arp(outport, reg0);
    /* MAC binding to 00e:ad:00:00:01. */
    next;
10. lr_in_arp_request (ovn-northd.c:6699): 1, priority 0, uuid 872ea686
    output;

egress(dp="r1", inport="r1_s2", outport="r1_s3")
------------------------------------------------
 3. lr_out_delivery (ovn-northd.c:6727): outport == "r1_s3", priority 100, uuid bb034c92
    output;
    /* output to "r1_s3", type "l3gateway" */

ingress(dp="s3", inport="s3_r1")
--------------------------------
 0. ls_in_port_sec_l2 (ovn-northd.c:4127): inport == "s3_r1", priority 50, uuid 975d14f4
    next;
16. ls_in_l2_lkup (ovn-northd.c:4567): 1, priority 0, uuid 6c97afd6
    outport = "_MC_unknown";
    output;

multicast(dp="s3", mcgroup="_MC_unknown")
-----------------------------------------

    egress(dp="s3", inport="s3_r1", outport="ln-s3")
    ------------------------------------------------
         9. ls_out_port_sec_l2 (ovn-northd.c:4604): outport == "ln-s3", priority 50, uuid 7d83c568
            output;
            /* output to "ln-s3", type "localnet" */
# Summary trace.
ingress(dp="r1", inport="r1_s2") {
    next;
    flags.loopback = 1;
    ct_dnat;
    ct_dnat /* assuming no un-dnat entry, so no change */ {
        ip.ttl--;
        reg0 = ip4.dst;
        reg1 = 172.16.103.1;
        eth.src = 00e:ad:ff:01:03;
        outport = "r1_s3";
        flags.loopback = 1;
        next;
        get_arp(outport, reg0);
        /* MAC binding to 00e:ad:00:00:01. */
        next;
        output;
        egress(dp="r1", inport="r1_s2", outport="r1_s3") {
            output;
            /* output to "r1_s3", type "l3gateway" */;
            ingress(dp="s3", inport="s3_r1") {
                next;
                outport = "_MC_unknown";
                output;
                multicast(dp="s3", mcgroup="_MC_unknown") {
                    egress(dp="s3", inport="s3_r1", outport="ln-s3") {
                        output;
                        /* output to "ln-s3", type "localnet" */;
                    };
                };
            };
        };
    };
};
# Minimal trace.
ct_dnat /* assuming no un-dnat entry, so no change */ {
    ip.ttl--;
    eth.src = 00e:ad:ff:01:03;
    eth.dst = 00e:ad:00:00:01;
    output("ln-s3");
};

[root@hp-dl380pg8-04 ~]# ovn-nbctl show
switch ec4b963d-db24-4a56-9c2d-9be20b12bcc1 (s3)
    port s3_r1
        type: router
        addresses: ["00e:ad:ff:01:03 172.16.103.1"]
        router-port: r1_s3
    port ln-s3
        type: localnet
        addresses: ["unknown"]
switch aefc06ca-5f7a-4179-8c1b-1b78443c6103 (outside)
    port hv0_vm01_vnet1
        addresses: ["00e:ad:00:01:01 172.16.103.12"]
    port hv0_vm00_vnet1
        addresses: ["00e:ad:00:00:01 172.16.103.11"]
    port ln-outside
        type: localnet
        addresses: ["unknown"]
switch 570a1ff3-dafe-4879-8f08-d5c07d1079dc (s2)
    port hv1_vm01_vnet1
        addresses: ["00e:ad:01:01:01 172.16.102.12"]
    port hv1_vm00_vnet1
        addresses: ["00e:ad:01:00:01 172.16.102.11"]
    port s2_r1
        type: router
        addresses: ["00e:ad:ff:01:02 172.16.102.1"]
        router-port: r1_s2
router 5d9478ae-271a-4d9e-8aa4-8b4e1518c478 (r1)
    port r1_s3
        mac: "00e:ad:ff:01:03"
        networks: ["172.16.103.1/24"]
        gateway chassis: [gw1 gw2]
    port r1_s2
        mac: "00e:ad:ff:01:02"
        networks: ["172.16.102.1/24"]
    nat 90c010df-f4fd-45fe-8ed1-e9056ecaa351
        external ip: "172.16.102.200"
        logical ip: "172.16.103.11"
        type: "dnat_and_snat"
[root@hp-dl380pg8-04 ~]#
Comment 6 Red Hat Bugzilla 2025-02-10 03:59:26 UTC 
This product has been discontinued or is no longer tracked in Red Hat Bugzilla.