The bug should be fixed once the OpenShift is rebased on top of Kubernetes 1.23.

Not completed this sprint.

The rebase is completed and the bug should be fixed as a result of rebasing.

Versions:
========
oc version
Client Version: 4.10.0-0.nightly-2022-02-02-000921
Server Version: 4.10.0-0.nightly-2022-02-03-220350
Kubernetes Version: v1.23.3+b63be7f

PAO Version:

    "msg": {
        "architecture": "x86_64",
        "build-date": "2022-02-02T19:59:27.762163",
        "com.redhat.build-host": "cpt-1005.osbs.prod.upshift.rdu2.redhat.com",
        "com.redhat.component": "performance-addon-operator-container",
        "com.redhat.license_terms": "https://www.redhat.com/agreements",
        "description": "performance-addon-operator",
        "distribution-scope": "public",
        "io.k8s.description": "performance-addon-operator",
        "io.k8s.display-name": "performance-addon-operator",
        "io.openshift.expose-services": "",
        "io.openshift.maintainer.component": "Performance Addon Operator",
        "io.openshift.maintainer.product": "OpenShift Container Platform",
        "io.openshift.tags": "operator",
        "maintainer": "openshift-operators",
        "name": "openshift4/performance-addon-rhel8-operator",
        "release": "28",
        "summary": "performance-addon-operator",
        "upstream-vcs-ref": "7e40c978acca61ea540fb10b34e826474d6a93cf",
        "upstream-vcs-type": "git",
        "upstream-version": "0.0.41001-2-g7e40c978",
        "url": "https://access.redhat.com/containers/#/registry.access.redhat.com/openshift4/performance-addon-rhel8-operator/images/v4.10.0-28",
        "vcs-ref": "8473aa2255f73db5523c2a665256ed6297a99025",
        "vcs-type": "git",
        "vendor": "Red Hat, Inc.",
        "version": "v4.10.0"


Steps:

1. create a performance profile as show below:

apiVersion: performance.openshift.io/v2
kind: PerformanceProfile
metadata:
  name: performance
spec:
  cpu:
    isolated: 5-19,45-59,20-39,60-79
    reserved: 0-4,40-44
  hugepages:
    defaultHugepagesSize: 1G
    pages:
    - count: 20
      size: 2M
  net:
    userLevelNetworking: true
  nodeSelector:
    node-role.kubernetes.io/workercnf: ""
  numa:
    topologyPolicy: restricted
  realTimeKernel:
    enabled: true

2. Create a pod using below specs:

apiVersion: v1
kind: Pod
metadata:
  name: pod1
spec:
  containers:
  - name: example-pod1
    image: fedora:latest
    command:
    - sleep
    - inf
    volumeMounts:
    - mountPath: /hugepages-2Mi
      name: hugepage-2mi
    resources:
      limits:
        hugepages-2Mi: 24Mi
        memory: "24Mi"
        cpu: "2"
      requests:
        hugepages-2Mi: 24Mi
        memory: "24Mi"
        cpu: "2"
  nodeSelector:
    kubernetes.io/hostname: "worker-0"
  volumes:
  - name: hugepage-2mi
    emptyDir:
      medium: HugePages-2Mi

3. check Pod status:

[root@registry bz-1999603]# oc get pods -o wide
NAME   READY   STATUS    RESTARTS   AGE    IP            NODE       NOMINATED NODE   READINESS GATES
pod1   1/1     Running   0          3m2s   10.128.2.26   worker-0   <none>           <none>
[root@registry bz-1999603]# oc debug node/worker-0

4. Get Memory Manager state file:

sh-4.4# cat memory_manager_state
{"policyName":"Static","machineState":{"0":{"numberOfAssignments":2,"memoryMap":{"hugepages-1Gi":{"total":0,"systemReserved":0,"allocatable":0,"reserved":0,"free":0},"hugepages-2Mi":{"total":20971520,"systemReserved":0,"allocatable":20971520,"reserved":20971520,"free":0},"memory":{"total":270146011136,"systemReserved":1153433600,"allocatable":268971606016,"reserved":25165824,"free":268946440192}},"cells":[0,1]},"1":{"numberOfAssignments":2,"memoryMap":{"hugepages-1Gi":{"total":0,"systemReserved":0,"allocatable":0,"reserved":0,"free":0},"hugepages-2Mi":{"total":20971520,"systemReserved":0,"allocatable":20971520,"reserved":4194304,"free":16777216},"memory":{"total":270531715072,"systemReserved":0,"allocatable":270510743552,"reserved":0,"free":270510743552}},"cells":[0,1]}},"entries":{"a9feb7f2-a1d4-4f7f-ae5f-ba5c2b60b254":{"example-pod1":[{"numaAffinity":[0,1],"type":"hugepages-2Mi","size":25165824},{"numaAffinity":[0,1],"type":"memory","size":25165824}]}},"checksum":279997132}sh-4.4#

5. Get cpus and numa nodes used by pod1

[root@registry bz-1999603]# oc exec -ti pods/pod1 -- bash -c "cat /sys/fs/cgroup/cpuset/cpuset.cpus"
5,45
[root@registry bz-1999603]# oc exec -ti pods/pod1 -- bash -c "cat /sys/fs/cgroup/cpuset/cpuset.mems"
0-1

6. Create Pod2. with below spec:

[root@registry bz-1999603]# cat test2.yaml 
apiVersion: v1
kind: Pod
metadata:
  name: pod2
spec:
  containers:
  - name: example-pod2
    image: fedora:latest
    command:
    - sleep
    - inf
    volumeMounts:
    - mountPath: /hugepages-2Mi
      name: hugepage-2mi
    resources:
      limits:
        hugepages-2Mi: 16Mi
        memory: "16Mi"
        cpu: "2"
      requests:
        hugepages-2Mi: 16Mi
        memory: "16Mi"
        cpu: "2"
  nodeSelector:
    kubernetes.io/hostname: "worker-0"
  volumes:
  - name: hugepage-2mi
    emptyDir:
      medium: HugePages-2Mi

7. Create of pod2 should fail:

[root@registry bz-1999603]# oc get pods -o wide
NAME   READY   STATUS                   RESTARTS   AGE     IP            NODE       NOMINATED NODE   READINESS GATES
pod1   1/1     Running                  0          8m11s   10.128.2.26   worker-0   <none>           <none>
pod2   0/1     ContainerStatusUnknown   0          13s     10.128.2.27   worker-0   <none>           <none>

8. Check the pod2 status 

[root@registry bz-1999603]# oc describe pods/pod2
Name:         pod2
Namespace:    default
Priority:     0
Node:         worker-0/10.46.80.2
Start Time:   Fri, 04 Feb 2022 04:23:52 -0500
Labels:       <none>
Annotations:  k8s.ovn.org/pod-networks:
                {"default":{"ip_addresses":["10.128.2.27/23"],"mac_address":"0a:58:0a:80:02:1b","gateway_ips":["10.128.2.1"],"ip_address":"10.128.2.27/23"...
              k8s.v1.cni.cncf.io/network-status:
                [{
                    "name": "ovn-kubernetes",
                    "interface": "eth0",
                    "ips": [
                        "10.128.2.27"
                    ],
                    "mac": "0a:58:0a:80:02:1b",
                    "default": true,
                    "dns": {}
                }]
              k8s.v1.cni.cncf.io/networks-status:
                [{
                    "name": "ovn-kubernetes",
                    "interface": "eth0",
                    "ips": [
                        "10.128.2.27"
                    ],
                    "mac": "0a:58:0a:80:02:1b",
                    "default": true,
                    "dns": {}
                }]
Status:       Failed
Reason:       TopologyAffinityError
Message:      Pod Resources cannot be allocated with Topology locality
IP:           10.128.2.27   
IPs:
  IP:  10.128.2.27
Containers:
  example-pod2:
    Container ID:
    Image:         fedora:latest
    Image ID:
    Port:          <none>   
    Host Port:     <none>   
    Command:
      sleep
      inf
    State:          Terminated
      Reason:       ContainerStatusUnknown
      Message:      The container could not be located when the pod was terminated
      Exit Code:    137
      Started:      Mon, 01 Jan 0001 00:00:00 +0000
      Finished:     Mon, 01 Jan 0001 00:00:00 +0000
    Ready:          False   
    Restart Count:  0
    Limits:
      cpu:            2
      hugepages-2Mi:  16Mi  
      memory:         16Mi  
    Requests:
      cpu:            2
      hugepages-2Mi:  16Mi  
      memory:         16Mi  
    Environment:      <none>
    Mounts:
      /hugepages-2Mi from hugepage-2mi (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-5529z (ro)
Conditions:
  Type              Status  
  Initialized       True
  Ready             False   
  ContainersReady   False   
  PodScheduled      True
Volumes:
  hugepage-2mi:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     HugePages-2Mi
    SizeLimit:  <unset>
  kube-api-access-5529z:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
    ConfigMapName:           openshift-service-ca.crt
    ConfigMapOptional:       <nil>
QoS Class:                   Guaranteed
Node-Selectors:              kubernetes.io/hostname=worker-0
Tolerations:                 node.kubernetes.io/memory-pressure:NoSchedule op=Exists
                             node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
Events:
  Type     Reason                 Age   From               Message
  ----     ------                 ----  ----               -------
  Normal   Scheduled              22s   default-scheduler  Successfully assigned default/pod2 to worker-0
  Warning  TopologyAffinityError  23s   kubelet            Resources cannot be allocated with Topology locality
  Normal   AddedInterface         20s   multus             Add eth0 [10.128.2.27/23] from ovn-kubernetes
  Normal   Pulling                20s   kubelet            Pulling image "fedora:latest"
  Normal   Pulled                 16s   kubelet            Successfully pulled image "fedora:latest" in 3.597402945s
  Warning  Failed                 15s   kubelet            Error: container create failed: parent closed synchronisation channel

As seen pod2 gets rejected

Since the problem described in this bug report should be
resolved in a recent advisory, it has been closed with a
resolution of ERRATA.

For information on the advisory (Moderate: OpenShift Container Platform 4.10.3 security update), and where to find the updated
files, follow the link below.

If the solution does not work for you, open a new bug report.

https://access.redhat.com/errata/RHSA-2022:0056