It seems odd that ovn-trace doesn't trace the datapath past an l3gateway port, when there are often clearly datapaths beyond that port. For example: $ ovn-trace --db=tcp:172.17.0.2:6642 425f5827-5676-4e61-ba14-06151f656d8e 'inport=="default_nginx1" && eth.dst == 00:00:00:8C:5E:78 && eth.src == 0a:00:00:00:00:01 && ip4.dst == 4.2.2.1 && ip4.src == 10.128.2.3 && ip.ttl==64' # ip,reg14=0x3,vlan_tci=0x0000,dl_src=0a:00:00:00:00:01,dl_dst=00:00:00:8c:5e:78,nw_src=10.128.2.3,nw_dst=4.2.2.1,nw_proto=0,nw_tos=0,nw_ecn=0,nw_ttl=64 ingress(dp="openshift-node-2", inport="default_nginx1") ------------------------------------------------------- 0. ls_in_port_sec_l2 (ovn-northd.c:3556): inport == "default_nginx1", priority 50, uuid 2cce4bab next; 3. ls_in_pre_acl (ovn-northd.c:2882): ip, priority 100, uuid bd88b866 reg0[0] = 1; next; 5. ls_in_pre_stateful (ovn-northd.c:3016): reg0[0] == 1, priority 100, uuid 714b5e3a ct_next; ct_next(ct_state=est|trk /* default (use --ct to customize) */) --------------------------------------------------------------- 15. ls_in_l2_lkup (ovn-northd.c:3887): eth.dst == 00:00:00:8c:5e:78, priority 50, uuid f32229e3 outport = "stor-openshift-node-2"; output; egress(dp="openshift-node-2", inport="default_nginx1", outport="stor-openshift-node-2") --------------------------------------------------------------------------------------- 1. ls_out_pre_acl (ovn-northd.c:2845): ip && outport == "stor-openshift-node-2", priority 110, uuid 32d01467 next; 8. ls_out_port_sec_l2 (ovn-northd.c:4012): outport == "stor-openshift-node-2", priority 50, uuid 8b4b6f00 output; /* output to "stor-openshift-node-2", type "patch" */ ingress(dp="openshift-master", inport="rtos-openshift-node-2") -------------------------------------------------------------- 0. lr_in_admission (ovn-northd.c:4429): eth.dst == 00:00:00:8c:5e:78 && inport == "rtos-openshift-node-2", priority 50, uuid c72ac7e3 next; 5. lr_in_ip_routing (ovn-northd.c:4140): ip4.src == 10.128.2.0/24, priority 48, uuid 8db21e17 ip.ttl--; reg0 = 100.64.1.2; reg1 = 100.64.1.1; eth.src = 00:00:00:b8:df:45; outport = "rtoj-openshift-master"; flags.loopback = 1; next; 6. lr_in_arp_resolve (ovn-northd.c:5528): outport == "rtoj-openshift-master" && reg0 == 100.64.1.2, priority 100, uuid bdff17f1 eth.dst = 00:00:00:48:22:33; next; 8. lr_in_arp_request (ovn-northd.c:5618): 1, priority 0, uuid 921ae0a6 output; egress(dp="openshift-master", inport="rtos-openshift-node-2", outport="rtoj-openshift-master") ---------------------------------------------------------------------------------------------- 3. lr_out_delivery (ovn-northd.c:5646): outport == "rtoj-openshift-master", priority 100, uuid b2da5b59 output; /* output to "rtoj-openshift-master", type "patch" */ ingress(dp="join", inport="jtor-openshift-master") -------------------------------------------------- 0. ls_in_port_sec_l2 (ovn-northd.c:3556): inport == "jtor-openshift-master", priority 50, uuid 72776597 next; 15. ls_in_l2_lkup (ovn-northd.c:3887): eth.dst == 00:00:00:48:22:33, priority 50, uuid 0205e709 outport = "jtor-GR_openshift-node-2"; output; egress(dp="join", inport="jtor-openshift-master", outport="jtor-GR_openshift-node-2") ------------------------------------------------------------------------------------- 8. ls_out_port_sec_l2 (ovn-northd.c:4012): outport == "jtor-GR_openshift-node-2", priority 50, uuid db67e8ba output; /* output to "jtor-GR_openshift-node-2", type "l3gateway" */ jtor-GR_openshift-node-2 is an l3gateway port bound to a specific chassis, but in this logical network there are some datapaths after it. If I do the following patch: diff -up openvswitch-2.8.1/ovn/utilities/ovn-trace.c.foo openvswitch-2.8.1/ovn/utilities/ovn-trace.c --- openvswitch-2.8.1/ovn/utilities/ovn-trace.c.foo 2018-09-06 08:59:27.045863050 -0500 +++ openvswitch-2.8.1/ovn/utilities/ovn-trace.c 2018-09-06 09:02:50.461278060 -0500 @@ -599,7 +599,7 @@ read_ports(void) : shorten_uuid(port->name2 ? port->name2 : port->name)); - if (!strcmp(sbpb->type, "patch")) { + if (!strcmp(sbpb->type, "patch") || !strcmp(sbpb->type, "l3gateway")) { const char *peer_name = smap_get(&sbpb->options, "peer"); if (peer_name) { struct ovntrace_port *peer Then I get much further, allowing me to debug my issue: ... egress(dp="join", inport="jtor-openshift-master", outport="jtor-GR_openshift-node-2") ------------------------------------------------------------------------------------- 8. ls_out_port_sec_l2 (ovn-northd.c:4012): outport == "jtor-GR_openshift-node-2", priority 50, uuid db67e8ba output; /* output to "jtor-GR_openshift-node-2", type "l3gateway" */ ingress(dp="GR_openshift-node-2", inport="rtoj-GR_openshift-node-2") -------------------------------------------------------------------- 0. lr_in_admission (ovn-northd.c:4429): eth.dst == 00:00:00:48:22:33 && inport == "rtoj-GR_openshift-node-2", priority 50, uuid 2842ff80 next; 4. lr_in_dnat (ovn-northd.c:5211): ip, priority 50, uuid f30ee742 flags.loopback = 1; ct_dnat; ct_dnat /* assuming no un-dnat entry, so no change */ ----------------------------------------------------- 5. lr_in_ip_routing (ovn-northd.c:4140): ip4.dst == 0.0.0.0/0, priority 1, uuid 898062d1 ip.ttl--; reg0 = 172.17.0.1; reg1 = 172.17.0.4; eth.src = 02:42:ac:11:00:04; outport = "rtoe-GR_openshift-node-2"; flags.loopback = 1; next; 6. lr_in_arp_resolve (ovn-northd.c:5553): ip4, priority 0, uuid 355779a8 get_arp(outport, reg0); /* No MAC binding. */ next; 8. lr_in_arp_request (ovn-northd.c:5617): eth.dst == 00:00:00:00:00:00, priority 100, uuid 41e1dfb2 arp { eth.dst = ff:ff:ff:ff:ff:ff; arp.spa = reg1; arp.tpa = reg0; arp.op = 1; output; }; arp --- eth.dst = ff:ff:ff:ff:ff:ff; arp.spa = reg1; arp.tpa = reg0; arp.op = 1; output; egress(dp="GR_openshift-node-2", inport="rtoj-GR_openshift-node-2", outport="rtoe-GR_openshift-node-2") ------------------------------------------------------------------------------------------------------- 3. lr_out_delivery (ovn-northd.c:5646): outport == "rtoe-GR_openshift-node-2", priority 100, uuid 965f84f9 output; /* output to "rtoe-GR_openshift-node-2", type "l3gateway" */ ingress(dp="ext_openshift-node-2", inport="etor-GR_openshift-node-2") --------------------------------------------------------------------- 0. ls_in_port_sec_l2 (ovn-northd.c:3556): inport == "etor-GR_openshift-node-2", priority 50, uuid 83b2854c next; 15. ls_in_l2_lkup (ovn-northd.c:3864): eth.mcast, priority 100, uuid e5943c49 outport = "_MC_flood"; output; multicast(dp="ext_openshift-node-2", mcgroup="_MC_flood") --------------------------------------------------------- egress(dp="ext_openshift-node-2", inport="etor-GR_openshift-node-2", outport="etor-GR_openshift-node-2") -------------------------------------------------------------------------------------------------------- /* omitting output because inport == outport && !flags.loopback */ egress(dp="ext_openshift-node-2", inport="etor-GR_openshift-node-2", outport="ext_openshift-node-2") ---------------------------------------------------------------------------------------------------- 8. ls_out_port_sec_l2 (ovn-northd.c:3989): eth.mcast, priority 100, uuid 33f4ed7a output; /* output to "ext_openshift-node-2", type "" */
Submitted the patch to fix the issue - https://patchwork.ozlabs.org/patch/981211/
verified on the latest version: [root@hp-dl380pg8-04 ovn_ha]# rpm -qa | grep ovn ovn2.11-host-2.11.0-9.el7fdp.1.x86_64 puppet-ovn-13.3.1-1.el7.noarch ovn2.11-2.11.0-9.el7fdp.1.x86_64 kernel-kernel-networking-openvswitch-ovn-1.0-121.noarch openstack-nova-novncproxy-18.2.0-1.el7.noarch novnc-0.5.1-2.el7.noarch kernel-kernel-networking-openvswitch-ovn_ha-1.0-34.noarch ovn2.11-central-2.11.0-9.el7fdp.1.x86_64 [root@hp-dl380pg8-04 ovn_ha]# rpm -qa | grep openvswitch kernel-kernel-networking-openvswitch-ovn-1.0-121.noarch openvswitch2.11-2.11.0-5.el7fdp.x86_64 openvswitch-selinux-extra-policy-1.0-11.el7fdp.noarch kernel-kernel-networking-openvswitch-ovn_ha-1.0-34.noarch [root@hp-dl380pg8-04 ~]# ovn-trace --all r1 'inport == "r1_s2" && eth.src == 00e:ad:01:00:01 && eth.dst == 00e:ad:ff:01:02 && ip4.src == 172.16.102.11 && ip4.dst == 172.16.103.11 && ip.ttl==64' # ip,reg14=0x1,vlan_tci=0x0000,dl_src=00e:ad:01:00:01,dl_dst=00e:ad:ff:01:02,nw_src=172.16.102.11,nw_dst=172.16.103.11,nw_proto=0,nw_tos=0,nw_ecn=0,nw_ttl=64 # Detailed trace. ingress(dp="r1", inport="r1_s2") -------------------------------- 0. lr_in_admission (ovn-northd.c:5150): eth.dst == 00e:ad:ff:01:02 && inport == "r1_s2", priority 50, uuid 36fcfcef next; 4. lr_in_dnat (ovn-northd.c:6133): ip, priority 50, uuid 54bdd2df flags.loopback = 1; ct_dnat; ct_dnat /* assuming no un-dnat entry, so no change */ ----------------------------------------------------- 7. lr_in_ip_routing (ovn-northd.c:4732): ip4.dst == 172.16.103.0/24, priority 49, uuid d8c02f67 ip.ttl--; reg0 = ip4.dst; reg1 = 172.16.103.1; eth.src = 00e:ad:ff:01:03; outport = "r1_s3"; flags.loopback = 1; next; 8. lr_in_arp_resolve (ovn-northd.c:6592): ip4, priority 0, uuid 7378bce3 get_arp(outport, reg0); /* MAC binding to 00e:ad:00:00:01. */ next; 10. lr_in_arp_request (ovn-northd.c:6699): 1, priority 0, uuid 872ea686 output; egress(dp="r1", inport="r1_s2", outport="r1_s3") ------------------------------------------------ 3. lr_out_delivery (ovn-northd.c:6727): outport == "r1_s3", priority 100, uuid bb034c92 output; /* output to "r1_s3", type "l3gateway" */ ingress(dp="s3", inport="s3_r1") -------------------------------- 0. ls_in_port_sec_l2 (ovn-northd.c:4127): inport == "s3_r1", priority 50, uuid 975d14f4 next; 16. ls_in_l2_lkup (ovn-northd.c:4567): 1, priority 0, uuid 6c97afd6 outport = "_MC_unknown"; output; multicast(dp="s3", mcgroup="_MC_unknown") ----------------------------------------- egress(dp="s3", inport="s3_r1", outport="ln-s3") ------------------------------------------------ 9. ls_out_port_sec_l2 (ovn-northd.c:4604): outport == "ln-s3", priority 50, uuid 7d83c568 output; /* output to "ln-s3", type "localnet" */ # Summary trace. ingress(dp="r1", inport="r1_s2") { next; flags.loopback = 1; ct_dnat; ct_dnat /* assuming no un-dnat entry, so no change */ { ip.ttl--; reg0 = ip4.dst; reg1 = 172.16.103.1; eth.src = 00e:ad:ff:01:03; outport = "r1_s3"; flags.loopback = 1; next; get_arp(outport, reg0); /* MAC binding to 00e:ad:00:00:01. */ next; output; egress(dp="r1", inport="r1_s2", outport="r1_s3") { output; /* output to "r1_s3", type "l3gateway" */; ingress(dp="s3", inport="s3_r1") { next; outport = "_MC_unknown"; output; multicast(dp="s3", mcgroup="_MC_unknown") { egress(dp="s3", inport="s3_r1", outport="ln-s3") { output; /* output to "ln-s3", type "localnet" */; }; }; }; }; }; }; # Minimal trace. ct_dnat /* assuming no un-dnat entry, so no change */ { ip.ttl--; eth.src = 00e:ad:ff:01:03; eth.dst = 00e:ad:00:00:01; output("ln-s3"); }; [root@hp-dl380pg8-04 ~]# ovn-nbctl show switch ec4b963d-db24-4a56-9c2d-9be20b12bcc1 (s3) port s3_r1 type: router addresses: ["00e:ad:ff:01:03 172.16.103.1"] router-port: r1_s3 port ln-s3 type: localnet addresses: ["unknown"] switch aefc06ca-5f7a-4179-8c1b-1b78443c6103 (outside) port hv0_vm01_vnet1 addresses: ["00e:ad:00:01:01 172.16.103.12"] port hv0_vm00_vnet1 addresses: ["00e:ad:00:00:01 172.16.103.11"] port ln-outside type: localnet addresses: ["unknown"] switch 570a1ff3-dafe-4879-8f08-d5c07d1079dc (s2) port hv1_vm01_vnet1 addresses: ["00e:ad:01:01:01 172.16.102.12"] port hv1_vm00_vnet1 addresses: ["00e:ad:01:00:01 172.16.102.11"] port s2_r1 type: router addresses: ["00e:ad:ff:01:02 172.16.102.1"] router-port: r1_s2 router 5d9478ae-271a-4d9e-8aa4-8b4e1518c478 (r1) port r1_s3 mac: "00e:ad:ff:01:03" networks: ["172.16.103.1/24"] gateway chassis: [gw1 gw2] port r1_s2 mac: "00e:ad:ff:01:02" networks: ["172.16.102.1/24"] nat 90c010df-f4fd-45fe-8ed1-e9056ecaa351 external ip: "172.16.102.200" logical ip: "172.16.103.11" type: "dnat_and_snat" [root@hp-dl380pg8-04 ~]#