Bug 2063971

Summary: [4.8] Conntrack entry is not removed for LoadBalancer IP
Product: OpenShift Container Platform Reporter: Dan Winship <danw>
Component: NetworkingAssignee: Dan Winship <danw>
Networking sub component: openshift-sdn QA Contact: jechen <jechen>
Status: CLOSED ERRATA Docs Contact:
Severity: high    
Priority: high CC: anbhat, ddelcian, rravaiol, zzhao
Version: 4.8   
Target Milestone: ---   
Target Release: 4.8.z   
Hardware: x86_64   
OS: Linux   
Whiteboard:
Fixed In Version: Doc Type: If docs needed, set a value
Doc Text:
Story Points: ---
Clone Of: 2063970 Environment:
Last Closed: 2022-08-24 08:05:46 UTC Type: ---
Regression: --- Mount Type: ---
Documentation: --- CRM:
Verified Versions: Category: ---
oVirt Team: --- RHEL 7.3 requirements from Atomic Host:
Cloudforms Team: --- Target Upstream Version:
Embargoed:
Bug Depends On: 2063970    
Bug Blocks:    

Comment 7 jechen 2022-08-18 19:51:37 UTC
Verified in 4.8.0-0.nightly-2022-08-17-015803 

# oc get clusterversion
NAME      VERSION                             AVAILABLE   PROGRESSING   SINCE   STATUS
version   4.8.0-0.nightly-2022-08-17-015803   True        False         8m39s   Cluster version is 4.8.0-0.nightly-2022-08-17-015803


# created a metalLB load balance service on a BM machine (detailed steps omitted here)

# create a project j1 and service

# oc new-project j1
# cat list.yaml 
---
apiVersion: v1
kind: List
items:
- apiVersion: v1
  kind: ReplicationController
  metadata:
    labels:
      name: test-rc
    name: test-rc
  spec:
    replicas: 7
    template:
      metadata:
        labels:
          name: test-pods
      spec:
        containers:
        - command:
          - "/usr/bin/ncat"
          - "-u"
          - "-l"
          - '8080'
          - "--keep-open"
          - "--exec"
          - "/bin/cat"
          image: quay.io/openshifttest/hello-sdn@sha256:2af5b5ec480f05fda7e9b278023ba04724a3dd53a296afcd8c13f220dec52197
          name: test-pod
          imagePullPolicy: Always
          resources:
            limits:
              memory: 340Mi
- apiVersion: v1
  kind: Service
  metadata:
    labels:
      name: test-service
    name: test-service
  spec:
    ports:
    - name: http
      port: 8080
      protocol: UDP
      targetPort: 8080
    selector:
      name: test-pods
    type: LoadBalancer
[root@dell-per740-36 ~]# oc create -f list.yaml
replicationcontroller/test-rc created
service/test-service created


# oc -n j1 get all
NAME                READY   STATUS              RESTARTS   AGE
pod/test-rc-7htcq   0/1     ContainerCreating   0          5s
pod/test-rc-8nc6q   0/1     ContainerCreating   0          5s
pod/test-rc-khht9   0/1     ContainerCreating   0          5s
pod/test-rc-kj5qh   0/1     ContainerCreating   0          5s
pod/test-rc-rhvkq   0/1     ContainerCreating   0          5s
pod/test-rc-wfrt8   0/1     ContainerCreating   0          5s
pod/test-rc-wrczj   0/1     ContainerCreating   0          5s

NAME                            DESIRED   CURRENT   READY   AGE
replicationcontroller/test-rc   7         7         0       5s

NAME                   TYPE           CLUSTER-IP     EXTERNAL-IP    PORT(S)          AGE
service/test-service   LoadBalancer   172.30.16.76   10.73.116.58   8080:32671/UDP   5s

# in another namesapce j2, create a test pod, from the test pod
# oc new-project j2
# oc create -f list_for_pods.json 
replicationcontroller/test-rc created
service/test-service created


# oc rsh -n j2 test-rc-4hdpt 
~ $ (while true ; sleep 1;  do echo "hello"; done) | ncat -u 10.73.116.58 8080
hello
hello
hello
hello
hello
hello
hello
hello
hello
hello
hello
hello
hello
hello
hello
hello
hello



# check conntrack entry from node where pod resides


# oc -n j2 get pod -owide
NAME            READY   STATUS    RESTARTS   AGE     IP            NODE                                      NOMINATED NODE   READINESS GATES
test-rc-4hdpt   1/1     Running   0          2m23s   10.128.2.36   dell-per740-14.rhts.eng.pek2.redhat.com   <none>           <none>
test-rc-8lhpd   1/1     Running   0          2m23s   10.131.0.30   dell-per740-35.rhts.eng.pek2.redhat.com   <none>           <none>
test-rc-nqwj8   1/1     Running   0          2m23s   10.128.2.35   dell-per740-14.rhts.eng.pek2.redhat.com   <none>           <none>

# oc debug node/dell-per740-14.rhts.eng.pek2.redhat.com
Starting pod/dell-per740-14rhtsengpek2redhatcom-debug ...
To use host binaries, run `chroot /host`
Pod IP: 10.73.116.62
If you don't see a command prompt, try pressing enter.
sh-4.4# chroot /host
sh-4.4# conntrack -L | grep 8080 | grep 10.73.116.58
conntrack v1.4.4 (conntrack-tools): 1177 flow entries have been shown.
tcp      6 293 ESTABLISHED src=10.73.116.50 dst=10.73.116.58 sport=2379 dport=58080 [UNREPLIED] src=10.73.116.58 dst=10.73.116.50 sport=58080 dport=2379 mark=0 secctx=system_u:object_r:unlabeled_t:s0 use=1
udp      17 119 src=10.128.2.36 dst=10.73.116.58 sport=50703 dport=8080 src=10.128.2.34 dst=10.128.2.1 sport=8080 dport=62526 [ASSURED] mark=0 secctx=system_u:object_r:unlabeled_t:s0 use=2
sh-4.4# 
sh-4.4# 


# delete the test service in j1 namespace
# oc -n j1 delete service/test-service
service "test-service" deleted
# oc -n j1 delete replicationcontroller/test-rc
replicationcontroller "test-rc" deleted


# check conntrack entry again
# oc debug node/dell-per740-14.rhts.eng.pek2.redhat.com
Starting pod/dell-per740-14rhtsengpek2redhatcom-debug ...
To use host binaries, run `chroot /host`
Pod IP: 10.73.116.62
If you don't see a command prompt, try pressing enter.
sh-4.4# chroot /host
sh-4.4# 
sh-4.4# conntrack -L | grep 8080 | grep 10.73.116.58
conntrack v1.4.4 (conntrack-tools): 1172 flow entries have been shown.
tcp      6 298 ESTABLISHED src=10.73.116.50 dst=10.73.116.58 sport=2379 dport=58080 [UNREPLIED] src=10.73.116.58 dst=10.73.116.50 sport=58080 dport=2379 mark=0 secctx=system_u:object_r:unlabeled_t:s0 use=1
sh-4.4# 


==>. conntrack entry for this UDP test-service is removed correctly.

Comment 9 errata-xmlrpc 2022-08-24 08:05:46 UTC
Since the problem described in this bug report should be
resolved in a recent advisory, it has been closed with a
resolution of ERRATA.

For information on the advisory (OpenShift Container Platform 4.8.48 security update), and where to find the updated
files, follow the link below.

If the solution does not work for you, open a new bug report.

https://access.redhat.com/errata/RHBA-2022:6099