Bug 2063885

Summary: [4.10] Conntrack entry is not removed for LoadBalancer IP
Product: OpenShift Container Platform Reporter: Dan Winship <danw>
Component: NetworkingAssignee: Dan Winship <danw>
Networking sub component: openshift-sdn QA Contact: jechen <jechen>
Status: CLOSED ERRATA Docs Contact:
Severity: high    
Priority: high CC: anbhat, crairobi, rravaiol, zzhao
Version: 4.8   
Target Milestone: ---   
Target Release: 4.10.z   
Hardware: x86_64   
OS: Linux   
Whiteboard:
Fixed In Version: Doc Type: If docs needed, set a value
Doc Text:
Story Points: ---
Clone Of: 2061002
: 2063970 (view as bug list) Environment:
Last Closed: 2022-05-02 18:38:50 UTC Type: ---
Regression: --- Mount Type: ---
Documentation: --- CRM:
Verified Versions: Category: ---
oVirt Team: --- RHEL 7.3 requirements from Atomic Host:
Cloudforms Team: --- Target Upstream Version:
Embargoed:
Bug Depends On: 2061002    
Bug Blocks: 2063970    

Comment 2 jechen 2022-04-28 20:42:19 UTC
verified in 4.10.0-0.nightly-2022-04-28-074916

# oc get clusterversion
NAME      VERSION                              AVAILABLE   PROGRESSING   SINCE   STATUS
version   4.10.0-0.nightly-2022-04-28-074916   True        False         41m     Cluster version is 4.10.0-0.nightly-2022-04-28-074916
   

# setup metalLB on a BM machine (steps omitted here)

# create a namespace, create test service in it
# oc new-project j1
# cat list.yaml 
---
apiVersion: v1
kind: List
items:
- apiVersion: v1
  kind: ReplicationController
  metadata:
    labels:
      name: test-rc
    name: test-rc
  spec:
    replicas: 7
    template:
      metadata:
        labels:
          name: test-pods
      spec:
        containers:
        - command:
          - "/usr/bin/ncat"
          - "-u"
          - "-l"
          - '8080'
          - "--keep-open"
          - "--exec"
          - "/bin/cat"
          image: quay.io/openshifttest/hello-sdn@sha256:2af5b5ec480f05fda7e9b278023ba04724a3dd53a296afcd8c13f220dec52197
          name: test-pod
          imagePullPolicy: Always
          resources:
            limits:
              memory: 340Mi
- apiVersion: v1
  kind: Service
  metadata:
    labels:
      name: test-service
    name: test-service
  spec:
    ports:
    - name: http
      port: 8080
      protocol: UDP
      targetPort: 8080
    selector:
      name: test-pods
    type: LoadBalancer
[root@dell-per740-36 ~]# oc create -f list.yaml
replicationcontroller/test-rc created
service/test-service created


# oc get all -owide -n j1
NAME                READY   STATUS    RESTARTS   AGE   IP            NODE                                      NOMINATED NODE   READINESS GATES
pod/test-rc-6kddt   1/1     Running   0          60m   10.129.0.22   dell-per740-14.rhts.eng.pek2.redhat.com   <none>           <none>
pod/test-rc-6qr5l   1/1     Running   0          60m   10.128.2.30   dell-per740-35.rhts.eng.pek2.redhat.com   <none>           <none>
pod/test-rc-7bxdm   1/1     Running   0          60m   10.128.2.31   dell-per740-35.rhts.eng.pek2.redhat.com   <none>           <none>
pod/test-rc-hdm7d   1/1     Running   0          60m   10.129.0.23   dell-per740-14.rhts.eng.pek2.redhat.com   <none>           <none>
pod/test-rc-lwjcm   1/1     Running   0          60m   10.129.0.21   dell-per740-14.rhts.eng.pek2.redhat.com   <none>           <none>
pod/test-rc-n8vd7   1/1     Running   0          60m   10.129.0.20   dell-per740-14.rhts.eng.pek2.redhat.com   <none>           <none>
pod/test-rc-rjv5r   1/1     Running   0          60m   10.128.2.32   dell-per740-35.rhts.eng.pek2.redhat.com   <none>           <none>

NAME                            DESIRED   CURRENT   READY   AGE   CONTAINERS   IMAGES                                                                                                    SELECTOR
replicationcontroller/test-rc   7         7         7       60m   test-pod     quay.io/openshifttest/hello-sdn@sha256:2af5b5ec480f05fda7e9b278023ba04724a3dd53a296afcd8c13f220dec52197   name=test-pods

NAME                   TYPE           CLUSTER-IP      EXTERNAL-IP    PORT(S)          AGE   SELECTOR
service/test-service   LoadBalancer   172.30.63.217   10.73.116.58   8080:32509/UDP   60m   name=test-pods



# in another namesapce j2, create a test pod, from the test pod
# oc new-project j2
# oc create -f list_for_pods.json 
replicationcontroller/test-rc created
service/test-service created


# oc get pod -owide -n j2
NAME            READY   STATUS    RESTARTS   AGE   IP            NODE                                      NOMINATED NODE   READINESS GATES
test-rc-6zgz5   1/1     Running   0          26s   10.129.0.34   dell-per740-14.rhts.eng.pek2.redhat.com   <none>           <none>
test-rc-b92pc   1/1     Running   0          26s   10.129.0.35   dell-per740-14.rhts.eng.pek2.redhat.com   <none>           <none>
test-rc-fr45c   1/1     Running   0          26s   10.128.2.33   dell-per740-35.rhts.eng.pek2.redhat.com   <none>           <none>

# oc -n j2 rsh test-rc-6zgz5
~ $ (while true ; sleep 1;  do echo "hello"; done) | ncat -u 10.73.116.58 8080
hello
hello
hello
hello
hello
hello
hello
hello
hello
hello
hello
hello
hello
hello
hello
hello
hello


# check conntrack entry from node where pod resides
# oc debug node/dell-per740-14.rhts.eng.pek2.redhat.com
Starting pod/dell-per740-14rhtsengpek2redhatcom-debug ...
To use host binaries, run `chroot /host`
Pod IP: 10.73.116.62
If you don't see a command prompt, try pressing enter.
sh-4.4# chroot /host
sh-4.4# conntrack -L | grep 8080 | grep 10.73.116.58
conntrack v1.4.4 (conntrack-tools): 1287 flow entries have been shown.
sh-4.4# conntrack -L | grep 8080 | grep 10.73.116.58
conntrack v1.4.4 (conntrack-tools): 1261 flow entries have been shown.
udp      17 119 src=10.129.0.34 dst=10.73.116.58 sport=47824 dport=8080 src=10.128.2.32 dst=10.129.0.1 sport=8080 dport=53275 [ASSURED] mark=0 secctx=system_u:object_r:unlabeled_t:s0 use=1
sh-4.4# 
sh-4.4# 


# delete the test service in j1 namespace
# oc -n j1 delete service/test-service
service "test-service" deleted
# oc -n j1 delete replicationcontroller/test-rc
replicationcontroller "test-rc" deleted


# check conntrack entry again
# oc debug node/dell-per740-14.rhts.eng.pek2.redhat.com
Starting pod/dell-per740-14rhtsengpek2redhatcom-debug ...
To use host binaries, run `chroot /host`
Pod IP: 10.73.116.62
If you don't see a command prompt, try pressing enter.
sh-4.4# chroot /host
sh-4.4# conntrack -L | grep 8080 | grep 10.73.116.58
conntrack v1.4.4 (conntrack-tools): 1276 flow entries have been shown.


==>. conntrack entry for this UDP test-service is removed correctly.

Comment 4 errata-xmlrpc 2022-05-02 18:38:50 UTC
Since the problem described in this bug report should be
resolved in a recent advisory, it has been closed with a
resolution of ERRATA.

For information on the advisory (OpenShift Container Platform 4.10.12 bug fix update), and where to find the updated
files, follow the link below.

If the solution does not work for you, open a new bug report.

https://access.redhat.com/errata/RHBA-2022:1601