Description of problem:

Created sriovnetworknodepolicy on cluster to enable VFs, before sriovnetworknodestates sync up status changing to succeeded, created sriovnetwork and pods which using the policy and VFs, the node status changed to "Ready,SchedulingDisabled".
Waited for several hours, the status didn't recover.

#  oc describe sriovnetworknodestates.sriovnetwork.openshift.io dell-per740-14.rhts.eng.pek2.redhat.com -n openshift-sriov-network-operator | grep Sync
  Sync Status:      InProgress


# oc get nodes
NAME                                      STATUS                     ROLES    AGE   VERSION
dell-per740-13.rhts.eng.pek2.redhat.com   Ready                      master   12d   v1.22.0-rc.0+dc932e9
dell-per740-14.rhts.eng.pek2.redhat.com   Ready,SchedulingDisabled   worker   12d   v1.22.0-rc.0+dc932e9
dell-per740-31.rhts.eng.pek2.redhat.com   Ready                      master   12d   v1.22.0-rc.0+dc932e9
dell-per740-32.rhts.eng.pek2.redhat.com   Ready                      master   12d   v1.22.0-rc.0+dc932e9
dell-per740-35.rhts.eng.pek2.redhat.com   Ready                      worker   12d   v1.22.0-rc.0+dc932e9      


Version-Release number of selected component (if applicable):

Client Version: 4.8.0-0.nightly-2021-04-23-131610
Server Version: 4.9.0-0.nightly-2021-08-19-180334
Kubernetes Version: v1.22.0-rc.0+dc932e9

How reproducible:


Steps to Reproduce:
1. create sriovnetworknodepolicy
# oc create -f mlx277netpolicy.yaml -n openshift-sriov-network-operator 
sriovnetworknodepolicy.sriovnetwork.openshift.io/mlx277netpolicy created
# cat mlx277netpolicy.yaml
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovNetworkNodePolicy
metadata:
  name: mlx277netpolicy
  namespace: openshift-sriov-network-operator
spec:
  mtu: 1800
  nicSelector:
    deviceID: '1015'
    pfNames:
      - ens2f0
    vendor: '15b3'
  nodeSelector:
    feature.node.kubernetes.io/sriov-capable: 'true'
  numVfs: 10
  resourceName: mlx277netpolicy

2.create sriovnetwork

# oc create -f mlx277netdevice.yaml -n openshift-sriov-network-operator 

# cat mlx277netdevice.yaml
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovNetwork
metadata:
  name: mlx277netdevice
  namespace: openshift-sriov-network-operator
spec:
  ipam: '{ "type": "static" }'
  capabilities: '{ "ips": true }'
  vlan: 0
  spoofChk: "on"
  trust: "off"
  resourceName: mlx277netpolicy
  networkNamespace: sriov-testing


3. create pod

# oc create -f sriov-testpod1.yaml -n sriov-testing 
pod/sriov-testpod1 created

# cat sriov-testpod1.yaml 
apiVersion: v1
kind: Pod
metadata:
  name: sriov-testpod1
  annotations:
    k8s.v1.cni.cncf.io/networks:  '[
        {
                "name": "mlx277netdevice",
                "ips": ["192.168.2.5/24", "2002::5/64"]
        }
]'
spec:
  containers:
  - name: samplecontainer
    imagePullPolicy: IfNotPresent
    image: quay.io/openshifttest/hello-sdn@sha256:d5785550cf77b7932b090fcd1a2625472912fb3189d5973f177a5a2c347a1f95

4. check pod and node status

[root@dell-per740-36 sriov]# oc describe  pods
Name:         sriov-testpod1
Namespace:    sriov-testing
Priority:     0
Node:         <none>
Labels:       <none>
Annotations:  k8s.v1.cni.cncf.io/networks: [ { "name": "mlx277netdevice", "ips": ["192.168.2.5/24", "2002::5/64"] } ]
              openshift.io/scc: anyuid
Status:       Pending
IP:           
IPs:          <none>
Containers:
  samplecontainer:
    Image:      quay.io/openshifttest/hello-sdn@sha256:d5785550cf77b7932b090fcd1a2625472912fb3189d5973f177a5a2c347a1f95
    Port:       <none>
    Host Port:  <none>
    Limits:
      openshift.io/mlx277netpolicy:  1
    Requests:
      openshift.io/mlx277netpolicy:  1
    Environment:                     <none>
    Mounts:
      /etc/podnetinfo from podnetinfo (ro)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-tr8xj (ro)
Conditions:
  Type           Status
  PodScheduled   False 
Volumes:
  kube-api-access-tr8xj:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
    ConfigMapName:           openshift-service-ca.crt
    ConfigMapOptional:       <nil>
  podnetinfo:
    Type:  DownwardAPI (a volume populated by information about the pod)
    Items:
      metadata.annotations -> annotations
QoS Class:       BestEffort
Node-Selectors:  <none>
Tolerations:     node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                 node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason            Age   From               Message
  ----     ------            ----  ----               -------
  Warning  FailedScheduling  6s    default-scheduler  0/5 nodes are available: 1 Insufficient openshift.io/mlx277netpolicy, 1 node(s) were unschedulable, 3 node(s) had taint {node-role.kubernetes.io/master: }, that the pod didn't tolerate.
[root@dell-per740-36 sriov]# oc get pods
NAME             READY   STATUS    RESTARTS   AGE
sriov-testpod1   0/1     Pending   0          10s
[root@dell-per740-36 sriov]# oc get nodes
NAME                                      STATUS                     ROLES    AGE   VERSION
dell-per740-13.rhts.eng.pek2.redhat.com   Ready                      master   12d   v1.22.0-rc.0+dc932e9
dell-per740-14.rhts.eng.pek2.redhat.com   Ready,SchedulingDisabled   worker   12d   v1.22.0-rc.0+dc932e9
dell-per740-31.rhts.eng.pek2.redhat.com   Ready                      master   12d   v1.22.0-rc.0+dc932e9
dell-per740-32.rhts.eng.pek2.redhat.com   Ready                      master   12d   v1.22.0-rc.0+dc932e9
dell-per740-35.rhts.eng.pek2.redhat.com   Ready                      worker   12d   v1.22.0-rc.0+dc932e9

5. check sriovnetworknodestates
#  oc describe sriovnetworknodestates.sriovnetwork.openshift.io dell-per740-14.rhts.eng.pek2.redhat.com -n openshift-sriov-network-operator | grep Sync
  Sync Status:      InProgress

Actual results:

node status is abnormal


Expected results:


Additional info: