Bug 1832176

Summary: tcp reset packet is not sent after acl reject is added in special environment
Product: Red Hat Enterprise Linux Fast Datapath Reporter: Jianlin Shi <jishi>
Component: ovn2.13Assignee: Numan Siddique <nusiddiq>
Status: CLOSED ERRATA QA Contact: Jianlin Shi <jishi>
Severity: medium Docs Contact:
Priority: medium    
Version: FDP 20.DCC: ctrautma, dcbw, jishi, nusiddiq, ralongi
Target Milestone: ---Keywords: Regression
Target Release: ---   
Hardware: Unspecified   
OS: Unspecified   
Whiteboard:
Fixed In Version: Doc Type: If docs needed, set a value
Doc Text:
Story Points: ---
Clone Of: Environment:
Last Closed: 2020-07-15 13:00:54 UTC Type: Bug
Regression: --- Mount Type: ---
Documentation: --- CRM:
Verified Versions: Category: ---
oVirt Team: --- RHEL 7.3 requirements from Atomic Host:
Cloudforms Team: --- Target Upstream Version:
Attachments:
Description Flags
ovnnb_db.db
none
ovnsb_db.db none

Description Jianlin Shi 2020-05-06 09:31:12 UTC
Description of problem:
tcp reset packet is not sent after acl reject is added in special environment

Version-Release number of selected component (if applicable):
ovn2.13.0-21

How reproducible:
always

Steps to Reproduce:
1. topo:   ls1p1  ---  ls1  --- lr1 --- ls2 --- ls2p1
                        |                |
                       ls1p2            ls2p2
2. add port_group pg1="ls1p1 ls2p1", pg2="ls1p2 ls2p2"
3. add acl rules:
ovn-nbctl --type=port-group acl-add pg2 from-lport 1004 "inport == @pg2 && tcp" allow-related
ovn-nbctl --type=port-group acl-add pg1 to-lport 1004 "outport == @pg1 && tcp" allow-related

ovn-nbctl --type=port-group acl-add pg1 to-lport 1006 "outport == @pg1 && tcp.dst==22" reject

4. send tcp packet whose dst port as 22 to ls1p1 on ls1p2

Actual results:
no tcp reset packet sent

Expected results:
tcp reset packet should be sent

Additional info:


server:

systemctl start openvswitch                                    
systemctl start ovn-northd                                          
ovn-nbctl set-connection ptcp:6641                                       
ovn-sbctl set-connection ptcp:6642                                         
ovs-vsctl set open . external_ids:system-id=hv1 external_ids:ovn-remote=tcp:20.0.30.25:6642 external_ids:ovn-encap-type=geneve external_ids:ovn-encap-ip=20.0.30.25
systemctl restart ovn-controller                                      
                                                          
ip netns add server0                                      
ip link add veth0_s0 netns server0 type veth peer name veth0_s0_p
ip netns exec server0 ip link set lo up                       
ip netns exec server0 ip link set veth0_s0 up
ip netns exec server0 ip link set veth0_s0 address 00:00:00:01:01:02
ip netns exec server0 ip addr add 192.168.1.1/24 dev veth0_s0       
ip netns exec server0 ip -6 addr add 2001::1/64 dev veth0_s0
ip netns exec server0 ip route add default via 192.168.1.254 dev veth0_s0
ip netns exec server0 ip -6 route add default via 2001::a dev veth0_s0                       
ovs-vsctl add-port br-int veth0_s0_p                                                        
ip link set veth0_s0_p up                                    
ovs-vsctl set interface veth0_s0_p external_ids:iface-id=ls1p1                               
                                                                       
ip netns add server2                 
ip link add veth0_s2 netns server2 type veth peer name veth0_s2_p
ip netns exec server2 ip link set lo up
ip netns exec server2 ip link set veth0_s2 up
ip netns exec server2 ip link set veth0_s2 address 00:00:00:01:03:02
ip netns exec server2 ip addr add 192.168.2.3/24 dev veth0_s2
ip netns exec server2 ip -6 addr add 2002::3/64 dev veth0_s2
ip netns exec server2 ip route add default via 192.168.2.254 dev veth0_s2
ip netns exec server2 ip -6 route add default via 2002::a dev veth0_s2

ovs-vsctl add-port br-int veth0_s2_p
ip link set veth0_s2_p up
ovs-vsctl set interface veth0_s2_p external_ids:iface-id=ls2p1

ovn-nbctl lr-add lr1
ovn-nbctl lrp-add lr1 lr1-ls1 00:00:00:00:00:01 192.168.1.254/24 2001::a/64
ovn-nbctl lrp-add lr1 lr1-ls2 00:00:00:00:00:02 192.168.2.254/24 2002::a/64


ovn-nbctl ls-add ls1
ovn-nbctl lsp-add ls1 ls1-lr1
ovn-nbctl lsp-set-type ls1-lr1 router
ovn-nbctl lsp-set-addresses ls1-lr1 router
ovn-nbctl lsp-set-options ls1-lr1 router-port=lr1-ls1

ovn-nbctl lsp-add ls1 ls1p1
ovn-nbctl lsp-set-addresses ls1p1 "00:00:00:01:01:02 192.168.1.1 2001::1"
ovn-nbctl lsp-add ls1 ls1p2
ovn-nbctl lsp-set-addresses ls1p2 "00:00:00:01:02:02 192.168.1.2 2001::2"

ovn-nbctl ls-add ls2
ovn-nbctl lsp-add ls2 ls2-lr1
ovn-nbctl lsp-set-type ls2-lr1 router
ovn-nbctl lsp-set-addresses ls2-lr1 router
ovn-nbctl lsp-set-options ls2-lr1 router-port=lr1-ls2

ovn-nbctl lsp-add ls2 ls2p1
ovn-nbctl lsp-set-addresses ls2p1 "00:00:00:01:03:02 192.168.2.3 2002::3"
ovn-nbctl lsp-add ls2 ls2p2
ovn-nbctl lsp-set-addresses ls2p2 "00:00:00:01:04:02 192.168.2.4 2002::4"

ls1p1_uuid=`ovn-nbctl get logical_switch_port ls1p1 _uuid`
ls1p2_uuid=`ovn-nbctl get logical_switch_port ls1p2 _uuid`
ls2p1_uuid=`ovn-nbctl get logical_switch_port ls2p1 _uuid`
ls2p2_uuid=`ovn-nbctl get logical_switch_port ls2p2 _uuid`

ovn-nbctl create Port_Group name=pg1 ports="$ls1p1_uuid $ls2p1_uuid"
ovn-nbctl create Port_Group name=pg2 ports="$ls1p2_uuid $ls2p2_uuid"
ovn-nbctl list Port_Group

ovn-nbctl --type=port-group acl-add pg2 from-lport 1004 "inport == @pg2 && tcp" allow-related
ovn-nbctl --type=port-group acl-add pg1 to-lport 1004 "outport == @pg1 && tcp" allow-related

ovn-nbctl --type=port-group acl-add pg1 to-lport 1006 "outport == @pg1 && tcp.dst==22" reject

client:


systemctl start openvswitch                          
ovs-vsctl set open . external_ids:system-id=hv0 external_ids:ovn-remote=tcp:20.0.30.25:6642 external_ids:ovn-encap-type=geneve external_ids:ovn-encap-ip=20.0.30.26
systemctl restart ovn-controller

ip netns add server1
ip link add veth0_s1 netns server1 type veth peer name veth0_s1_p
ip netns exec server1 ip link set lo up
ip netns exec server1 ip link set veth0_s1 up
ip netns exec server1 ip link set veth0_s1 address 00:00:00:01:02:02
ip netns exec server1 ip addr add 192.168.1.2/24 dev veth0_s1
ip netns exec server1 ip -6 addr add 2001::2/64 dev veth0_s1
ip netns exec server1 ip route add default via 192.168.1.254 dev veth0_s1
ip netns exec server1 ip -6 route add default via 2001::a dev veth0_s1

ovs-vsctl add-port br-int veth0_s1_p
ip link set veth0_s1_p up
ovs-vsctl set interface veth0_s1_p external_ids:iface-id=ls1p2

ip netns add server3
ip link add veth0_s3 netns server3 type veth peer name veth0_s3_p
ip netns exec server3 ip link set lo up
ip netns exec server3 ip link set veth0_s3 up
ip netns exec server3 ip link set veth0_s3 address 00:00:00:01:04:02
ip netns exec server3 ip addr add 192.168.2.4/24 dev veth0_s3
ip netns exec server3 ip -6 addr add 2002::4/64 dev veth0_s3
ip netns exec server3 ip route add default via 192.168.2.254 dev veth0_s3
ip netns exec server3 ip -6 route add default via 2002::a dev veth0_s3

ovs-vsctl add-port br-int veth0_s3_p
ip link set veth0_s3_p up
ovs-vsctl set interface veth0_s3_p external_ids:iface-id=ls2p2


after setup, run test on server:

[root@dell-per740-12 test]# ip netns exec server2  nc 192.168.1.1 22 <<< h
Ncat: Connection timed out.   

<=== no tcp reset sent, should be refused, (the same as 192.168.1.2)                                
[root@dell-per740-12 test]# ip netns exec server2  nc 192.168.1.2 22 <<< h
Ncat: Connection refused.

<=== tcp reset sent

[root@dell-per740-12 test]# rpm -qa | grep -E "openvswitch|ovn"
kernel-kernel-networking-openvswitch-ovn-basic-1.0-24.noarch
openvswitch-selinux-extra-policy-1.0-23.el8fdp.noarch
ovn2.13-central-2.13.0-21.el8fdp.x86_64
python3-openvswitch2.13-2.13.0-9.el8fdp.x86_64
ovn2.13-host-2.13.0-21.el8fdp.x86_64
kernel-kernel-networking-openvswitch-ovn-common-1.0-7.noarch
ovn2.13-2.13.0-21.el8fdp.x86_64
openvswitch2.13-2.13.0-9.el8fdp.x86_64

[root@dell-per740-12 test]# ovn-nbctl acl-list pg1
  to-lport  1006 (outport == @pg1 && tcp.dst==22) reject
  to-lport  1004 (outport == @pg1 && tcp) allow-related
[root@dell-per740-12 test]# ovn-nbctl acl-list pg2
from-lport  1004 (inport == @pg2 && tcp) allow-related

Comment 1 Jianlin Shi 2020-05-06 09:32:41 UTC
the issue doesn't exist on ovn2.13.0-11, set regression

Comment 2 Jianlin Shi 2020-05-06 09:53:43 UTC
Created attachment 1685619 [details]
ovnnb_db.db

Comment 3 Jianlin Shi 2020-05-06 09:54:12 UTC
Created attachment 1685620 [details]
ovnsb_db.db

Comment 4 Numan Siddique 2020-05-06 10:33:47 UTC
In this case the first tcp  pkt goes from ls2p1 -> ls2 -> lr1 -> ls1 -> ovn-controller (and it generates the tcp rst)
and this tcp rst goes from ovn-controller -> ls1 (ingress pipeline) > lr1 -> ls2 -> and the pkt gets dropped in table 42.

 table=42, priority=100,ip,reg0=0x1/0x1,metadata=0x3 actions=ct(table=43,zone=NXM_NX_REG13[0..15])

We see the below warning logs in ovs-vswitchd

2020-05-06T10:29:05.084Z|00025|ofproto_dpif_upcall(handler1)|INFO|received packet on unassociated datapath port 4294967295
2020-05-06T10:29:05.251Z|00080|ofproto_dpif_upcall(revalidator37)|WARN|Failed to acquire udpif_key corresponding to unexpected flow (Invalid argument): ufid:0daac824-bda7-44d8-ad38-cdd9c5f0fc97
2020-05-06T10:29:06.127Z|00001|ofproto_dpif_upcall(handler12)|INFO|received packet on unassociated datapath port 4294967295
2020-05-06T10:29:06.253Z|00081|ofproto_dpif_upcall(revalidator37)|WARN|Failed to acquire udpif_key corresponding to unexpected flow (Invalid argument): ufid:e8e88f04-c1bd-4f64-87e6-9698b78c195f


Earlier, it worked because we were by passing the tcp rst packets from conntrack.

I'll debug further to see why the pkt is getting dropped by ovs-vswitchd during upcall.

Comment 9 Jianlin Shi 2020-06-28 07:52:23 UTC
Verified on ovn2.13-2.13.0-37.el8fdp.x86_64:

[root@dell-per740-12 bz1832176]# rpm -qa | grep -E "openvswitch|ovn"
openvswitch2.13-2.13.0-40.el8fdb.x86_64                                                                         
ovn2.13-2.13.0-37.el8fdp.x86_64                                                                                 
openvswitch-selinux-extra-policy-1.0-23.el8fdp.noarch                                                           
ovn2.13-central-2.13.0-37.el8fdp.x86_64                                                                         
ovn2.13-host-2.13.0-37.el8fdp.x86_64                                                                            
[root@dell-per740-12 bz1832176]# ip netns exec server2  nc 192.168.1.1 22 <<< h                                 
Ncat: Connection refused.           

<=== tcp reset sent
                                                                            
[root@dell-per740-12 bz1832176]# ip netns exec server2  nc 192.168.1.2 22 <<< h                                 
Ncat: Connection refused.

Comment 10 Jianlin Shi 2020-06-28 07:54:08 UTC
Verified on ovn2.13-2.13.0-37.el7fdp.x86_64:

[root@dell-per740-42 bz1832176]# rpm -qa | grep -E "openvswitch|ovn"                                            
ovn2.13-central-2.13.0-37.el7fdp.x86_64
openvswitch2.13-2.13.0-30.el7fdp.x86_64                                                                         
ovn2.13-host-2.13.0-37.el7fdp.x86_64
ovn2.13-2.13.0-37.el7fdp.x86_64
openvswitch-selinux-extra-policy-1.0-15.el7fdp.noarch                                                           
[root@dell-per740-42 bz1832176]# ip netns exec server2  nc 192.168.1.1 22 <<< h                                 
Ncat: Connection refused.

<=== tcp reset sent

[root@dell-per740-42 bz1832176]# ip netns exec server2  nc 192.168.1.2 22 <<< h                                 
Ncat: Connection refused.

Comment 12 errata-xmlrpc 2020-07-15 13:00:54 UTC
Since the problem described in this bug report should be
resolved in a recent advisory, it has been closed with a
resolution of ERRATA.

For information on the advisory, and where to find the updated
files, follow the link below.

If the solution does not work for you, open a new bug report.

https://access.redhat.com/errata/RHBA-2020:2941