Bug 1948065

Description of problem:

After installing kubernetes-nmstate-operator.v4.7.0 and creating an instance, the operator fails back to Pending status with Reason: RequirementsNotMet


Version-Release number of selected component (if applicable):

kubernetes-nmstate-operator.v4.7.0  4.7.0-202103270130.p0


How reproducible:  100%


Steps to Reproduce:
1.  Following https://docs.openshift.com/container-platform/4.7/networking/k8s_nmstate/k8s-nmstate-about-the-k8s-nmstate-operator.html install the Kubernetes NMState Operator on an OCP 4.7.1 cluster into a newly created openshift-nmstate project/namespace

The operator will report InstallSucceeded and go into Ready status


2.  Following steps 7+ in the above documentation, create an instance named "nmstate"


Actual results:

The operator/ClusterServiceVersion status changes to Failed with reason message of "requirements no longer met" and then switches to Pending with reason message "requirements not met"


Expected results:

The operator/ClusterServiceVersion reports a status of Ready


Additional info:


# ./oc describe csv -n openshift-nmstate kubernetes-nmstate-operator.v4.7.0
Name:         kubernetes-nmstate-operator.v4.7.0
Namespace:    openshift-nmstate
Labels:       olm.api.15f414d5897c1859=provided
              operators.coreos.com/kubernetes-nmstate-operator.openshift-nmstate=
Annotations:  alm-examples:
                [{
                  "apiVersion": "nmstate.io/v1beta1",
                  "kind": "NMState",
                  "metadata": {
                    "name": "nmstate"
                  },
                  "spec": {
                    "nodeSelector": {
                      "beta.kubernetes.io/arch": "amd64"
                    }
                  }
                }]
              capabilities: Basic Install
              categories: OpenShift Optional
              certified: false
              containerImage:
                registry.redhat.io/openshift4/kubernetes-nmstate-rhel8-operator@sha256:95139aed7e6d71c1931a0fc9767891d4077d7157f70a92a9175cf9fd530f0599
              createdAt: 2020-011-12 17:13:03
              description: Kubernetes NMState is a declaritive means of configuring NetworkManager.
              olm.operatorGroup: openshift-nmstate-lr87z
              olm.operatorNamespace: openshift-nmstate
              olm.skipRange: >=4.3.0-0 < 4.7.0-0
              olm.targetNamespaces: openshift-nmstate
              operatorframework.io/properties:
                {"properties":[{"type":"olm.gvk","value":{"group":"nmstate.io","kind":"NMState","version":"v1beta1"}},{"type":"olm.package","value":{"pack...
              repository: https://github.com/openshift/kubernetes-nmstate
              support: Red Hat, Inc.
API Version:  operators.coreos.com/v1alpha1
Kind:         ClusterServiceVersion
Metadata:
  Creation Timestamp:  2021-04-09T21:16:11Z
  Generation:          1
  Managed Fields:
    API Version:  operators.coreos.com/v1alpha1
[...]
    Manager:         olm
    Operation:       Update
    Time:            2021-04-09T21:16:14Z
  Resource Version:  9376174
  Self Link:         /apis/operators.coreos.com/v1alpha1/namespaces/openshift-nmstate/clusterserviceversions/kubernetes-nmstate-operator.v4.7.0
  UID:               e105b2d6-321b-4c5a-8e4a-b68a4a878fa4
Spec:
  Apiservicedefinitions:
  Customresourcedefinitions:
    Owned:
      Description:   Represents an NMState deployment.
      Display Name:  NMState
      Kind:          NMState
      Name:          nmstates.nmstate.io
      Version:       v1beta1
  Description:       A Kubernetes Operator to install Kubernetes NMState
  Display Name:      Kubernetes NMState Operator
[...]
  Install:
    Spec:
      Cluster Permissions:
        Rules:
          API Groups:
            admissionregistration.k8s.io
          Resources:
            mutatingwebhookconfigurations
          Verbs:
            *
          API Groups:
            rbac.authorization.k8s.io
          Resources:
            clusterroles
            clusterrolebindings
            rolebindings
            roles
          Verbs:
            *
          API Groups:
            nmstate.io
          Resources:
            *
          Verbs:
            *
          API Groups:
            apiextensions.k8s.io
          Resources:
            *
          Verbs:
            *
          API Groups:
            apps
          Resources:
            deployments
            daemonsets
            replicasets
            statefulsets
          Verbs:
            *
          API Groups:
            
          Resources:
            serviceaccounts
            configmaps
            namespaces
          Verbs:
            *
          API Groups:
            security.openshift.io
          Resource Names:
            privileged
          Resources:
            securitycontextconstraints
          Verbs:
            use
        Service Account Name:  nmstate-operator
      Deployments:
        Name:  nmstate-operator
        Spec:
          Replicas:  1
          Selector:
            Match Labels:
              Name:  kubernetes-nmstate-operator
          Strategy:
          Template:
            Metadata:
              Creation Timestamp:  <nil>
              Labels:
                App:   kubernetes-nmstate-operator
                Name:  kubernetes-nmstate-operator
            Spec:
              Containers:
                Args:
                  --v=production
                Command:
                  manager
                Env:
                  Name:  WATCH_NAMESPACE
                  Value From:
                    Field Ref:
                      Field Path:   metadata.annotations['olm.targetNamespaces']
                  Name:             OPERATOR_NAME
                  Value:            kubernetes-nmstate-operator
                  Name:             ENABLE_PROFILER
                  Value:            False
                  Name:             PROFILER_PORT
                  Value:            6060
                  Name:             RUN_OPERATOR
                  Name:             HANDLER_IMAGE
                  Value:            registry.redhat.io/openshift4/ose-kubernetes-nmstate-handler-rhel8@sha256:fce48697d94273ea27341b962d8e91884c144a9f6c2e6988108af62852c0fa44
                  Name:             HANDLER_IMAGE_PULL_POLICY
                  Value:            Always
                  Name:             HANDLER_NAMESPACE
                  Value:            openshift-nmstate
                Image:              registry.redhat.io/openshift4/kubernetes-nmstate-rhel8-operator@sha256:95139aed7e6d71c1931a0fc9767891d4077d7157f70a92a9175cf9fd530f0599
                Image Pull Policy:  Always
                Name:               nmstate-operator
                Resources:
              Node Selector:
                node-role.kubernetes.io/master:  
              Service Account Name:              nmstate-operator
              Tolerations:
                Effect:    NoSchedule
                Key:       node-role.kubernetes.io/master
                Operator:  Exists
      Permissions:
        Rules:
          API Groups:
            
          Resources:
            services
            endpoints
            persistentvolumeclaims
            events
            configmaps
            secrets
            pods
          Verbs:
            *
          API Groups:
            apps
          Resources:
            deployments
            daemonsets
            replicasets
            statefulsets
          Verbs:
            *
          API Groups:
            policy
          Resources:
            poddisruptionbudgets
          Verbs:
            *
        Service Account Name:  nmstate-operator
        Rules:
          API Groups:
            security.openshift.io
          Resource Names:
            privileged
          Resources:
            securitycontextconstraints
          Verbs:
            use
        Service Account Name:  nmstate-handler
    Strategy:                  deployment
  Install Modes:
    Supported:  true
    Type:       OwnNamespace
    Supported:  true
    Type:       SingleNamespace
    Supported:  false
    Type:       MultiNamespace
    Supported:  false
    Type:       AllNamespaces
  Keywords:
    nmstate
    networking
    NetworkManager
  Labels:
    Olm - Owner - Enterprise - App:  kubernetes-nmstate-operator
    Olm - Status - Descriptors:      kubernetes-nmstate-operator.v4.7.0
  Links:
    Name:  Kubernetes Nmstate Operator
    URL:   https://github.com/nmstate/kubernetes-nmstate
  Maintainers:
    Email:   support
    Name:    Red Hat
  Maturity:  beta
  Provider:
    Name:  Red Hat, Inc.
  Selector:
    Match Labels:
      Name:  kubernetes-nmstate-operator
  Version:   4.7.0-202103270130.p0
Status:
  Conditions:
    Last Transition Time:  2021-04-09T21:16:11Z
    Last Update Time:      2021-04-09T21:16:11Z
    Message:               requirements not yet checked
    Phase:                 Pending
    Reason:                RequirementsUnknown
    Last Transition Time:  2021-04-09T21:16:11Z
    Last Update Time:      2021-04-09T21:16:11Z
    Message:               one or more requirements couldn't be found
    Phase:                 Pending
    Reason:                RequirementsNotMet
    Last Transition Time:  2021-04-09T21:16:14Z
    Last Update Time:      2021-04-09T21:16:14Z
    Message:               all requirements found, attempting install
    Phase:                 InstallReady
    Reason:                AllRequirementsMet
    Last Transition Time:  2021-04-09T21:16:14Z
    Last Update Time:      2021-04-09T21:16:14Z
    Message:               waiting for install components to report healthy
    Phase:                 Installing
    Reason:                InstallSucceeded
    Last Transition Time:  2021-04-09T21:16:14Z
    Last Update Time:      2021-04-09T21:16:14Z
    Message:               installing: waiting for deployment nmstate-operator to become ready: Waiting for deployment spec update to be observed...

    Phase:                 Installing
    Reason:                InstallWaiting
    Last Transition Time:  2021-04-09T21:16:19Z
    Last Update Time:      2021-04-09T21:16:19Z
    Message:               install strategy completed with no errors
    Phase:                 Succeeded
    Reason:                InstallSucceeded
    Last Transition Time:  2021-04-09T21:18:52Z
    Last Update Time:      2021-04-09T21:18:52Z
    Message:               requirements no longer met
    Phase:                 Failed
    Reason:                RequirementsNotMet
    Last Transition Time:  2021-04-09T21:18:52Z
    Last Update Time:      2021-04-09T21:18:52Z
    Message:               requirements not met
    Phase:                 Pending
    Reason:                RequirementsNotMet
  Last Transition Time:    2021-04-09T21:18:52Z
  Last Update Time:        2021-04-09T21:18:52Z
  Message:                 one or more requirements couldn't be found
  Phase:                   Pending
  Reason:                  RequirementsNotMet
  Requirement Status:
    Group:    apiextensions.k8s.io
    Kind:     CustomResourceDefinition
    Message:  CRD is present and Established condition is true
    Name:     nmstates.nmstate.io
    Status:   Present
    Uuid:     71e30e11-d6f3-4d5f-bec1-b7ad9eca886c
    Version:  v1
    Dependents:
      Group:    rbac.authorization.k8s.io
      Kind:     PolicyRule
      Message:  namespaced rule:{"verbs":["*"],"apiGroups":[""],"resources":["services","endpoints","persistentvolumeclaims","events","configmaps","secrets","pods"]}
      Status:   Satisfied
      Version:  v1
      Group:    rbac.authorization.k8s.io
      Kind:     PolicyRule
      Message:  namespaced rule:{"verbs":["*"],"apiGroups":["apps"],"resources":["deployments","daemonsets","replicasets","statefulsets"]}
      Status:   Satisfied
      Version:  v1
      Group:    rbac.authorization.k8s.io
      Kind:     PolicyRule
      Message:  namespaced rule:{"verbs":["*"],"apiGroups":["policy"],"resources":["poddisruptionbudgets"]}
      Status:   Satisfied
      Version:  v1
      Group:    rbac.authorization.k8s.io
      Kind:     PolicyRule
      Message:  cluster rule:{"verbs":["*"],"apiGroups":["admissionregistration.k8s.io"],"resources":["mutatingwebhookconfigurations"]}
      Status:   Satisfied
      Version:  v1
      Group:    rbac.authorization.k8s.io
      Kind:     PolicyRule
      Message:  cluster rule:{"verbs":["*"],"apiGroups":["rbac.authorization.k8s.io"],"resources":["clusterroles","clusterrolebindings","rolebindings","roles"]}
      Status:   Satisfied
      Version:  v1
      Group:    rbac.authorization.k8s.io
      Kind:     PolicyRule
      Message:  cluster rule:{"verbs":["*"],"apiGroups":["nmstate.io"],"resources":["*"]}
      Status:   Satisfied
      Version:  v1
      Group:    rbac.authorization.k8s.io
      Kind:     PolicyRule
      Message:  cluster rule:{"verbs":["*"],"apiGroups":["apiextensions.k8s.io"],"resources":["*"]}
      Status:   Satisfied
      Version:  v1
      Group:    rbac.authorization.k8s.io
      Kind:     PolicyRule
      Message:  cluster rule:{"verbs":["*"],"apiGroups":["apps"],"resources":["deployments","daemonsets","replicasets","statefulsets"]}
      Status:   Satisfied
      Version:  v1
      Group:    rbac.authorization.k8s.io
      Kind:     PolicyRule
      Message:  cluster rule:{"verbs":["*"],"apiGroups":[""],"resources":["serviceaccounts","configmaps","namespaces"]}
      Status:   Satisfied
      Version:  v1
      Group:    rbac.authorization.k8s.io
      Kind:     PolicyRule
      Message:  cluster rule:{"verbs":["use"],"apiGroups":["security.openshift.io"],"resources":["securitycontextconstraints"],"resourceNames":["privileged"]}
      Status:   Satisfied
      Version:  v1
    Group:      
    Kind:       ServiceAccount
    Message:    
    Name:       nmstate-operator
    Status:     Present
    Version:    v1
    Group:      
    Kind:       ServiceAccount
    Message:    Service account is not owned by this ClusterServiceVersion
    Name:       nmstate-handler
    Status:     PresentNotSatisfied
    Version:    v1
Events:
  Type     Reason               Age                From                        Message
  ----     ------               ----               ----                        -------
  Normal   RequirementsUnknown  52m                operator-lifecycle-manager  requirements not yet checked
  Normal   AllRequirementsMet   52m                operator-lifecycle-manager  all requirements found, attempting install
  Normal   InstallSucceeded     52m (x2 over 52m)  operator-lifecycle-manager  waiting for install components to report healthy
  Normal   InstallWaiting       52m                operator-lifecycle-manager  installing: waiting for deployment nmstate-operator to become ready: Waiting for deployment spec update to be observed...
  Normal   InstallWaiting       52m (x2 over 52m)  operator-lifecycle-manager  installing: waiting for deployment nmstate-operator to become ready: Waiting for rollout to finish: 0 of 1 updated replicas are available...
  Normal   InstallSucceeded     52m (x2 over 52m)  operator-lifecycle-manager  install strategy completed with no errors
  Normal   RequirementsNotMet   50m (x2 over 52m)  operator-lifecycle-manager  one or more requirements couldn't be found
  Warning  RequirementsNotMet   50m                operator-lifecycle-manager  requirements no longer met
  Normal   RequirementsNotMet   50m                operator-lifecycle-manager  requirements not met





Best I can tell the issue is related to "Service account is not owned by this ClusterServiceVersion"


# ./oc get -n openshift-nmstate sa nmstate-handler -oyaml
apiVersion: v1
imagePullSecrets:
- name: nmstate-handler-dockercfg-7x7ns
kind: ServiceAccount
metadata:
  creationTimestamp: "2021-04-09T21:16:13Z"
  labels:
    nmstate.io: ""
    operators.coreos.com/kubernetes-nmstate-operator.openshift-nmstate: ""
  managedFields:
  - apiVersion: v1
    fieldsType: FieldsV1
    fieldsV1:
      f:metadata:
        f:ownerReferences: {}
    manager: catalog
    operation: Update
    time: "2021-04-09T21:16:13Z"
  - apiVersion: v1
    fieldsType: FieldsV1
    fieldsV1:
      f:secrets:
        .: {}
        k:{"name":"nmstate-handler-token-v5ttw"}:
          .: {}
          f:name: {}
    manager: kube-controller-manager
    operation: Update
    time: "2021-04-09T21:16:13Z"
  - apiVersion: v1
    fieldsType: FieldsV1
    fieldsV1:
      f:metadata:
        f:labels:
          .: {}
          f:operators.coreos.com/kubernetes-nmstate-operator.openshift-nmstate: {}
    manager: olm
    operation: Update
    time: "2021-04-09T21:16:21Z"
  - apiVersion: v1
    fieldsType: FieldsV1
    fieldsV1:
      f:imagePullSecrets: {}
      f:secrets:
        k:{"name":"nmstate-handler-dockercfg-7x7ns"}:
          .: {}
          f:name: {}
    manager: openshift-controller-manager
    operation: Update
    time: "2021-04-09T21:16:21Z"
  - apiVersion: v1
    fieldsType: FieldsV1
    fieldsV1:
      f:metadata:
        f:labels:
          f:nmstate.io: {}
        f:ownerReferences:
          k:{"uid":"3b8e5b77-58e5-4549-8bdd-378c0d007510"}:
            .: {}
            f:apiVersion: {}
            f:blockOwnerDeletion: {}
            f:controller: {}
            f:kind: {}
            f:name: {}
            f:uid: {}
    manager: manager
    operation: Update
    time: "2021-04-09T21:18:05Z"
  name: nmstate-handler
  namespace: openshift-nmstate
  ownerReferences:
  - apiVersion: nmstate.io/v1beta1
    blockOwnerDeletion: true
    controller: true
    kind: NMState
    name: nmstate
    uid: 3b8e5b77-58e5-4549-8bdd-378c0d007510
  resourceVersion: "9375770"
  selfLink: /api/v1/namespaces/openshift-nmstate/serviceaccounts/nmstate-handler
  uid: 65d1d703-997d-4d9c-9aef-366f71f447f4
secrets:
- name: nmstate-handler-token-v5ttw
- name: nmstate-handler-dockercfg-7x7ns