Description of the problem: Upgrade to 2.5 hang due to Irreconcilable errors of grc-sub and search-prod-sub in MCH

Release version:

Operator snapshot version:
- ACM 2.5.0-DOWNSTREAM-2022-03-03-03-15-27 (Final S3)
- MCE 2.0.0-DOWNANDBACK-2022-03-07-12-37-51

OCP version: 4.8.31

Browser Info:

Steps to reproduce:
1. Install ACM 2.4.2
2. Upgrade to ACM 2.5

Actual results:
MCH is in 'Updating' state

Expected results:

Additional info:

oc get mch
NAME              STATUS     AGE
multiclusterhub   Updating   126m


oc get mch multiclusterhub -oyaml
apiVersion: operator.open-cluster-management.io/v1
kind: MultiClusterHub
metadata:
  annotations:
    installer.open-cluster-management.io/mce-subscription-spec: '{“channel": "stable-2.0","installPlanApproval":
      "Automatic","name": "multicluster-engine","source": "brew-mce-iib","sourceNamespace":
      "openshift-marketplace","startingCSV": "multicluster-engine.v2.0.0"}'
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"operator.open-cluster-management.io/v1","kind":"MultiClusterHub","metadata":{"annotations":{"mch-imageRepository":"brew.registry.redhat.io/rh-osbs/rhacm2"},"name":"multiclusterhub","namespace":"ocm"},"spec":{"customCAConfigmap":"custom-ca","enableClusterBackup":true,"imagePullSecret":"multiclusterhub-operator-pull-secret"}}
    mch-imageRepository: brew.registry.redhat.io/rh-osbs/rhacm2
  creationTimestamp: "2022-03-09T17:24:06Z"
  finalizers:
  - finalizer.operator.open-cluster-management.io
  generation: 3
  name: multiclusterhub
  namespace: ocm
  resourceVersion: "22120299"
  uid: 25e1ab41-8169-422a-b86b-fe2ad36e3413
spec:
  availabilityConfig: High
  customCAConfigmap: custom-ca
  enableClusterBackup: true
  imagePullSecret: multiclusterhub-operator-pull-secret
  ingress:
    sslCiphers:
    - ECDHE-ECDSA-AES256-GCM-SHA384
    - ECDHE-RSA-AES256-GCM-SHA384
    - ECDHE-ECDSA-AES128-GCM-SHA256
    - ECDHE-RSA-AES128-GCM-SHA256
  separateCertificateManagement: false
status:
  components:
    application-chart-sub:
      lastTransitionTime: "2022-03-09T17:24:57Z"
      reason: UpgradeSuccessful
      status: "True"
      type: Deployed
    cluster-backup-chart-sub:
      lastTransitionTime: "2022-03-09T17:25:04Z"
      reason: UpgradeSuccessful
      status: "True"
      type: Deployed
    cluster-lifecycle-sub:
      lastTransitionTime: "2022-03-09T17:25:05Z"
      reason: UpgradeSuccessful
      status: "True"
      type: Deployed
    console-chart-sub:
      lastTransitionTime: "2022-03-09T17:24:59Z"
      reason: UpgradeSuccessful
      status: "True"
      type: Deployed
    grc-sub:
      lastTransitionTime: "2022-03-09T19:30:38Z"
      message: 'failed to get candidate release: rendered manifests contain a resource
        that already exists. Unable to continue with update: ClusterManagementAddOn
        "cert-policy-controller" in namespace "" exists and cannot be imported into
        the current release: invalid ownership metadata; label validation error: missing
        key "app.kubernetes.io/managed-by": must be set to "Helm"; annotation validation
        error: missing key "meta.helm.sh/release-name": must be set to "grc-6caf0";
        annotation validation error: missing key "meta.helm.sh/release-namespace":
        must be set to "ocm"'
      reason: ReconcileError
      status: "True"
      type: Irreconcilable
    local-cluster:
      lastTransitionTime: "2022-03-09T18:57:52Z"
      message: No conditions available
      reason: No conditions available
      status: Unknown
      type: Unknown
    management-ingress-sub:
      lastTransitionTime: "2022-03-09T17:25:00Z"
      reason: UpgradeSuccessful
      status: "True"
      type: Deployed
    multicluster-engine:
      lastTransitionTime: "2022-03-09T19:31:37Z"
      reason: ComponentsAvailable
      status: "True"
      type: Available
    multicluster-engine-csv:
      lastTransitionTime: "2022-03-09T19:31:37Z"
      message: install strategy completed with no errors
      reason: InstallSucceeded
      status: "True"
    multicluster-engine-sub:
      lastTransitionTime: "2022-03-09T19:31:37Z"
      message: 'installPlanApproval: Automatic. installPlan: multicluster-engine/install-m96zv'
      reason: AtLatestKnown
      status: "True"
    multiclusterhub-repo:
      lastTransitionTime: "2022-03-09T18:47:27Z"
      reason: MinimumReplicasAvailable
      status: "True"
      type: Available
    policyreport-sub:
      lastTransitionTime: "2022-03-09T17:24:59Z"
      reason: UpgradeSuccessful
      status: "True"
      type: Deployed
    search-prod-sub:
      lastTransitionTime: "2022-03-09T19:30:38Z"
      message: 'rendered manifests contain a resource that already exists. Unable
        to continue with update: ClusterManagementAddOn "search-collector" in namespace
        "" exists and cannot be imported into the current release: invalid ownership
        metadata; label validation error: missing key "app.kubernetes.io/managed-by":
        must be set to "Helm"; annotation validation error: missing key "meta.helm.sh/release-name":
        must be set to "search-prod-970f9"; annotation validation error: missing key
        "meta.helm.sh/release-namespace": must be set to "ocm"'
      reason: UpgradeError
      status: "True"
      type: ReleaseFailed
  conditions:
  - lastTransitionTime: "2022-03-09T18:58:56Z"
    lastUpdateTime: "2022-03-09T18:58:56Z"
    message: Not all hub components ready.
    reason: ComponentsUnavailable
    status: "False"
    type: Complete
  - lastTransitionTime: "2022-03-09T17:24:07Z"
    lastUpdateTime: "2022-03-09T18:59:59Z"
    message: Created new resource
    reason: NewResourceCreated
    status: "True"
    type: Progressing
  currentVersion: 2.4.2
  desiredVersion: 2.5.0
  phase: Updating


oc get managedclusteraddon -n local-cluster
NAME                          AVAILABLE   DEGRADED   PROGRESSING
application-manager           True                   False
cert-policy-controller        False                  False
config-policy-controller      False
governance-policy-framework   Unknown
iam-policy-controller         False                  False
work-manager                  True                   False