Bug 1974757

this bug doesn't block OCP4.8.0 -release -  the fix will be delivered the the ACM channel on the AI-operator add verified then

FailedQA

Version:
quay.io/acm-d/acm-custom-registry:2.3.0-DOWNSTREAM-2021-06-28-15-34-59
clusterversion: 4.8.0-rc.1

oc get proxy cluster -o yaml
apiVersion: config.openshift.io/v1
kind: Proxy
metadata:
  creationTimestamp: "2021-06-29T18:41:45Z"
  generation: 1
  name: cluster
  resourceVersion: "697"
  uid: 1a68c850-c30a-4e8e-a01b-5aa30937ab42
spec:
  httpProxy: http://[2000::beef]:3128
  httpsProxy: http://[2000::beef]:3128
  noProxy: 2000::/64,.vlan614.rdu2.scalelab.redhat.com,fd01::/48,fd02::/112
  trustedCA:
    name: ""
status:
  httpProxy: http://[2000::beef]:3128
  httpsProxy: http://[2000::beef]:3128
  noProxy: .cluster.local,.svc,.vlan614.rdu2.scalelab.redhat.com,127.0.0.1,2000::/64,api-int.vlan614.rdu2.scalelab.redhat.com,fd01::/48,fd02::/112,localhost




oc get agentclusterinstall sno-1-1 -o json|jq ".status.conditions[0]"
{
  "lastProbeTime": "2021-06-30T04:06:17Z",
  "lastTransitionTime": "2021-06-30T04:06:17Z",
  "message": "The Spec could not be synced due to backend error: command oc adm release info -o template --template '{{.metadata.version}}' --insecure=false quay.io/openshift-release-dev/ocp-release:4.8.0-rc.1-x86_64 exited with non-zero exit code 1: \nerror: unable to read image quay.io/openshift-release-dev/ocp-release:4.8.0-rc.1-x86_64: Get \"https://quay.io/v2/\": dial tcp 50.16.140.223:443: connect: network is unreachable\n",
  "reason": "BackendError",
  "status": "False",
  "type": "SpecSynced"
}
[ro

@sasha could you please paste the content of assisted config map as well? In particular env variables.

oc get cm -n rhacm assisted-service -o yaml
apiVersion: v1
data:
  AGENT_DOCKER_IMAGE: registry.redhat.io/rhacm2/assisted-installer-agent-rhel8@sha256:3ca16d5713faa15b0172dc4f7a5175c7f25732f08fdee969641c17593f1078dd
  AUTH_TYPE: local
  BASE_DNS_DOMAINS: ""
  CHECK_CLUSTER_VERSION: "True"
  CONTROLLER_IMAGE: registry.redhat.io/rhacm2/assisted-installer-reporter-rhel8@sha256:bc38ca8bf6de46f35e0024a2e4f13562f861b7ba17165ecc7a8fe31b5a49b9f2
  CREATE_S3_BUCKET: "False"
  DEPLOY_TARGET: k8s
  ENABLE_KUBE_API: "True"
  ENABLE_SINGLE_NODE_DNSMASQ: "True"
  HTTPS_CERT_FILE: /etc/assisted-tls-config/tls.crt
  HTTPS_KEY_FILE: /etc/assisted-tls-config/tls.key
  HW_VALIDATOR_REQUIREMENTS: '[{"version":"default","master":{"cpu_cores":4,"ram_mib":16384,"disk_size_gb":120,"installation_disk_speed_threshold_ms":10},"worker":{"cpu_cores":2,"ram_mib":8192,"disk_size_gb":120,"installation_disk_speed_threshold_ms":10}}]'
  INSTALL_INVOKER: assisted-installer-operator
  INSTALL_RH_CA: "false"
  INSTALLER_IMAGE: registry.redhat.io/rhacm2/assisted-installer-rhel8@sha256:9e05d86c0afee726b0c1e98f68bcbf923f4934325887dff32d0b003640a34817
  IPV6_SUPPORT: "True"
  ISO_CACHE_DIR: /data/cache
  ISO_IMAGE_TYPE: minimal-iso
  ISO_WORKSPACE_BASE_DIR: /data
  JWKS_URL: https://api.openshift.com/.well-known/jwks.json
  LOG_FORMAT: text
  LOG_LEVEL: info
  NAMESPACE: rhacm
  OPENSHIFT_VERSIONS: '{"4.8":{"display_name":"4.8","release_image":null,"release_version":null,"rhcos_image":"http://e24-h01-000-r640.rdu2.scalelab.redhat.com/ocp_images/rhcos-live.x86_64.iso","rhcos_rootfs":"http://e24-h01-000-r640.rdu2.scalelab.redhat.com/ocp_images/rhcos-live-rootfs.x86_64.img","rhcos_version":"48.84.202106231817-0","support_level":null}}'
  PUBLIC_CONTAINER_REGISTRIES: quay.io,registry.svc.ci.openshift.org
  REGISTRY_CREDS: ""
  S3_USE_SSL: "false"
  SELF_VERSION: registry.redhat.io/rhacm2/agent-service-rhel8@sha256:92340474ac0cbfdf5cf0ff7d280016b8022f55899555f89165c2c48a668b1672
  SERVE_HTTPS: "True"
  SERVICE_BASE_URL: https://assisted-service-rhacm.apps.vlan614.rdu2.scalelab.redhat.com
  SERVICE_CA_CERT_PATH: /etc/assisted-ingress-cert/ca-bundle.crt
  SKIP_CERT_VERIFICATION: "False"
  STORAGE: filesystem
  WITH_AMS_SUBSCRIPTIONS: "False"
kind: ConfigMap
metadata:
  creationTimestamp: "2021-06-29T20:11:47Z"
  name: assisted-service
  namespace: rhacm
  ownerReferences:
  - apiVersion: agent-install.openshift.io/v1beta1
    blockOwnerDeletion: true
    controller: true
    kind: AgentServiceConfig
    name: agent
    uid: 81c3fc51-67de-4efe-b4f4-09ffcc321c5d
  resourceVersion: "134121"
  uid: 64ba0d7b-2b7a-4bb5-aa73-4728fdd7e8b0

Verified:

Version:
quay.io/acm-d/acm-custom-registry:2.3.0-DOWNSTREAM-2021-07-09-09-53-06
4.8.0-rc.3




oc get network cluster -o yaml
apiVersion: config.openshift.io/v1
kind: Network
metadata:
  creationTimestamp: "2021-07-08T19:47:39Z"
  generation: 2
  name: cluster
  resourceVersion: "4048"
  uid: 4fc61495-051c-4418-b5b2-9eac62b0b096
spec:
  clusterNetwork:
  - cidr: fd01::/48
    hostPrefix: 64
  externalIP:
    policy: {}
  networkType: OVNKubernetes
  serviceNetwork:
  - fd02::/112
status:
  clusterNetwork:
  - cidr: fd01::/48
    hostPrefix: 64
  clusterNetworkMTU: 1400
  networkType: OVNKubernetes
  serviceNetwork:
  - fd02::/112


oc get proxy cluster -o yaml
apiVersion: config.openshift.io/v1
kind: Proxy
metadata:
  creationTimestamp: "2021-07-08T19:47:39Z"
  generation: 1
  name: cluster
  resourceVersion: "683"
  uid: 7eb039e9-6289-4c35-8bb9-055e309e6467
spec:
  httpProxy: http://[1000::beef]:3128
  httpsProxy: http://[1000::beef]:3128
  noProxy: 1000::/64,.vlan614.rdu2.scalelab.redhat.com,fd01::/48,fd02::/112
  trustedCA:
    name: ""
status:
  httpProxy: http://[1000::beef]:3128
  httpsProxy: http://[1000::beef]:3128
  noProxy: .cluster.local,.svc,.vlan614.rdu2.scalelab.redhat.com,1000::/64,127.0.0.1,api-int.vlan614.rdu2.scalelab.redhat.com,fd01::/48,fd02::/112,localhost



Created the following objects:
---
apiVersion: hive.openshift.io/v1
kind: ClusterImageSet
metadata:
  name: "4.8"
  namespace: "assisted-installer"
spec:
  releaseImage: quay.io/openshift-release-dev/ocp-release:4.8.0-rc.3-x86_64

---
---
apiVersion: v1
kind: Namespace
metadata:
  name: sno-1-1
  labels:
    name: sno-1-1
---
---
apiVersion: v1
kind: Secret
type: kubernetes.io/dockerconfigjson
metadata:
  name: pull-secret
  namespace: sno-1-1
stringData:
  .dockerconfigjson: 'SECRET'
---
apiVersion: extensions.hive.openshift.io/v1beta1
kind: AgentClusterInstall
metadata:
  name: sno-1-1
  namespace: sno-1-1
spec:
  clusterDeploymentRef:
    name: sno-1-1
  imageSetRef:
    name: "4.8"
  networking:
    clusterNetwork:
    - cidr: fd01::/48
      hostPrefix: 64
    machineNetwork:
    - cidr: 1000::/64
    serviceNetwork:
    - fd02::/112
  provisionRequirements:
    controlPlaneAgents: 1
  sshPublicKey: '<KEY>'
---
apiVersion: hive.openshift.io/v1
kind: ClusterDeployment
metadata:
  name: sno-1-1
  namespace: sno-1-1
spec:
  baseDomain: vlan614.rdu2.scalelab.redhat.com
  clusterName: sno-1-1
  clusterInstallRef:
    group: extensions.hive.openshift.io
    kind: AgentClusterInstall
    name: sno-1-1
    version: v1beta1
  platform:
    agentBareMetal:
      agentSelector:
        matchLabels:
          clustername: sno-1-1
  pullSecretRef:
    name: pull-secret
---
apiVersion: agent.open-cluster-management.io/v1
kind: KlusterletAddonConfig
metadata:
  name: sno-1-1
  namespace: sno-1-1
spec:
  clusterName: sno-1-1
  clusterNamespace: sno-1-1
  clusterLabels:
    cloud: auto-detect
    vendor: auto-detect
  applicationManager:
    enabled: true
  certPolicyController:
    enabled: true
  iamPolicyController:
    enabled: true
  policyController:
    enabled: true
  searchCollector:
    enabled: false
---
apiVersion: cluster.open-cluster-management.io/v1
kind: ManagedCluster
metadata:
  name: sno-1-1
  namespace: sno-1-1
spec:
  hubAcceptsClient: true
---
apiVersion: agent-install.openshift.io/v1beta1
kind: NMStateConfig
metadata:
  name: sno-1-1-static-nmstateconfig
  namespace: sno-1-1
  labels:
    nmstate_config_cluster_name: sno-1-1-static
spec:
  config:
    interfaces:
      - name: eth0
        type: ethernet
        state: up
        ipv6:
          enabled: true
          address:
          - ip: 1000::1:1
            prefix-length: 64
          dhcp: false
        ipv4:
          enabled: false
    dns-resolver:
        config:
          server:
          - 1000::beef
    routes:
        config:
          - destination: ::/0
            next-hop-address: fe80::c8d3:7ba5:1115:a79c
            next-hop-interface: eth0
  interfaces:
    - name: "eth0"
      macAddress: "52:54:00:b1:01:01"
---
apiVersion: agent-install.openshift.io/v1beta1
kind: InfraEnv
metadata:
  name: sno-1-1
  namespace: sno-1-1
spec:
  clusterRef:
    name: sno-1-1
    namespace: sno-1-1
  agentLabelSelector:
    matchLabels:
      clustername: sno-1-1
  pullSecretRef:
    name: pull-secret
  sshAuthorizedKey: '<KEY>'
  nmStateConfigLabelSelector:
    matchLabels:
      nmstate_config_cluster_name: sno-1-1-static

---
apiVersion: v1
data:
  password: <pass>
  username: <user>
kind: Secret
metadata:
  name: sno-1-1-bmc-secret
  namespace: sno-1-1
type: Opaque

---
apiVersion: metal3.io/v1alpha1
kind: BareMetalHost
metadata:
  name: sno-1-1
  namespace: sno-1-1
  annotations:
    inspect.metal3.io: disabled
    bmac.agent-install.openshift.io/hostname: sno-1-1.vlan614.rdu2.scalelab.redhat.com
  labels:
    infraenvs.agent-install.openshift.io: "sno-1-1"
spec:
  bmc:
    address: "redfish-virtualmedia+http://[1000::ac01]:9000/redfish/v1/Systems/1012033b-cb11-4b17-920b-0d8ef71dc601"
    credentialsName: "sno-1-1-bmc-secret"
    disableCertificateVerification: true
  bootMACAddress: 52:54:00:b1:01:01
  automatedCleaningMode: disabled
  online: true






oc get agentclusterinstalls.extensions.hive.openshift.io -n sno-1-1 sno-1-1  -o json|jq ".status.conditions[0]"
{
  "lastProbeTime": "2021-07-09T20:35:28Z",
  "lastTransitionTime": "2021-07-09T20:35:28Z",
  "message": "SyncOK",
  "reason": "SyncOK",
  "status": "True",
  "type": "SpecSynced"
}

Since the problem described in this bug report should be
resolved in a recent advisory, it has been closed with a
resolution of ERRATA.

For information on the advisory (Moderate: OpenShift Container Platform 4.8.2 bug fix and security update), and where to find the updated
files, follow the link below.

If the solution does not work for you, open a new bug report.

https://access.redhat.com/errata/RHSA-2021:2438