Bug 1746155 - Image registry operator upgrade failure contributing to severe CI upgrade status
Summary: Image registry operator upgrade failure contributing to severe CI upgrade status
Keywords:
Status: CLOSED WORKSFORME
Alias: None
Product: OpenShift Container Platform
Classification: Red Hat
Component: Image Registry
Version: 4.2.0
Hardware: Unspecified
OS: Unspecified
unspecified
high
Target Milestone: ---
: 4.2.0
Assignee: Adam Kaplan
QA Contact: Wenjing Zheng
URL:
Whiteboard: buildcop
: 1746524 (view as bug list)
Depends On: 1748436
Blocks:
TreeView+ depends on / blocked
 
Reported: 2019-08-27 19:02 UTC by Robb Hamilton
Modified: 2019-09-05 13:00 UTC (History)
5 users (show)

Fixed In Version:
Doc Type: If docs needed, set a value
Doc Text:
Clone Of:
Environment:
Last Closed: 2019-09-05 13:00:59 UTC
Target Upstream Version:
Embargoed:


Attachments (Terms of Use)


Links
System ID Private Priority Status Summary Last Updated
Github openshift cluster-image-registry-operator pull 376 0 None closed Bug 1746155: Revert mount of trusted CA bundle 2020-08-31 06:39:23 UTC

Internal Links: 1748436

Comment 3 Adam Kaplan 2019-08-28 19:38:00 UTC
*** Bug 1746524 has been marked as a duplicate of this bug. ***

Comment 4 Adam Kaplan 2019-08-29 13:11:42 UTC
Moving to MODIFIED - https://github.com/openshift/cluster-image-registry-operator/pull/373 is no longer relevant to this bug fix.

Comment 6 Daneyon Hansen 2019-08-29 15:46:16 UTC
From my understanding https://github.com/openshift/cluster-image-registry-operator/pull/379 fixes this bug.

Comment 9 Daneyon Hansen 2019-09-03 22:37:41 UTC
I can not reproduce this bug. Details:

$ ./bin/openshift-install version
./bin/openshift-install unreleased-master-1680-g8c9abe40f7616303c03cafdc9ad612cd8fa7bd6b-dirty
built from commit 8c9abe40f7616303c03cafdc9ad612cd8fa7bd6b
release image registry.svc.ci.openshift.org/origin/release:4.2

$ oc get clusterversion
NAME      VERSION                         AVAILABLE   PROGRESSING   SINCE   STATUS
version   4.2.0-0.okd-2019-09-03-192548   True        False         98m     Cluster version is 4.2.0-0.okd-2019-09-03-192548

$ oc describe co image-registry
Name:         image-registry
Namespace:    
Labels:       <none>
Annotations:  <none>
API Version:  config.openshift.io/v1
Kind:         ClusterOperator
Metadata:
  Creation Timestamp:  2019-09-03T20:39:17Z
  Generation:          1
  Resource Version:    13495
  Self Link:           /apis/config.openshift.io/v1/clusteroperators/image-registry
  UID:                 e623a48b-ce8a-11e9-8818-022205e59796
Spec:
Status:
  Conditions:
    Last Transition Time:  2019-09-03T20:43:21Z
    Message:               The registry is ready
    Reason:                Ready
    Status:                True
    Type:                  Available
    Last Transition Time:  2019-09-03T20:43:21Z
    Message:               The registry is ready
    Reason:                Ready
    Status:                False
    Type:                  Progressing
    Last Transition Time:  2019-09-03T20:39:20Z
    Status:                False
    Type:                  Degraded
  Extension:               <nil>
  Related Objects:
    Group:      rbac.authorization.k8s.io
    Name:       system:registry
    Resource:   clusterroles
    Group:      rbac.authorization.k8s.io
    Name:       registry-registry-role
    Resource:   clusterrolebindings
    Group:      
    Name:       registry
    Namespace:  openshift-image-registry
    Resource:   serviceaccounts
    Group:      
    Name:       serviceca
    Namespace:  openshift-image-registry
    Resource:   configmaps
    Group:      
    Name:       image-registry-certificates
    Namespace:  openshift-image-registry
    Resource:   configmaps
    Group:      
    Name:       image-registry-private-configuration
    Namespace:  openshift-image-registry
    Resource:   secrets
    Group:      config.openshift.io
    Name:       cluster
    Resource:   images
    Group:      apps
    Name:       node-ca
    Namespace:  openshift-image-registry
    Resource:   daemonsets
    Group:      
    Name:       image-registry
    Namespace:  openshift-image-registry
    Resource:   services
    Group:      apps
    Name:       image-registry
    Namespace:  openshift-image-registry
    Resource:   deployments
  Versions:
    Name:     operator
    Version:  4.2.0-0.okd-2019-09-03-192548
Events:       <none>

$ oc get cm/cluster-config-v1 -n kube-system -o yaml
apiVersion: v1
data:
  install-config: |
    additionalTrustBundle: |
      -----BEGIN CERTIFICATE-----
      <MY_PROXY_CERTS>
      -----END CERTIFICATE-----
    apiVersion: v1
    baseDomain: devcluster.openshift.com
    compute:
    - hyperthreading: Enabled
      name: worker
      platform: {}
      replicas: 3
    controlPlane:
      hyperthreading: Enabled
      name: master
      platform:
        aws:
          rootVolume:
            iops: 0
            size: 120
            type: gp2
          type: m5.xlarge
          zones:
          - us-west-2a
          - us-west-2b
          - us-west-2c
          - us-west-2d
      replicas: 3
    metadata:
      creationTimestamp: null
      name: latest-proxy
    networking:
      clusterNetwork:
      - cidr: 10.128.0.0/14
        hostPrefix: 23
      machineCIDR: 10.0.0.0/16
      networkType: OpenShiftSDN
      serviceNetwork:
      - 172.30.0.0/16
    platform:
      aws:
        region: us-west-2
    proxy:
      httpProxy: http://jcallen:6cpbEH6uCepwEhNr2iB05ixP@52.73.102.120:3129
      httpsProxy: http://jcallen:6cpbEH6uCepwEhNr2iB05ixP@52.73.102.120:3129
    pullSecret: ""
    sshKey: <MY_SSH_KEY>
kind: ConfigMap
metadata:
  creationTimestamp: "2019-09-03T20:32:20Z"
  name: cluster-config-v1
  namespace: kube-system
  resourceVersion: "51"
  selfLink: /api/v1/namespaces/kube-system/configmaps/cluster-config-v1
  uid: edf829aa-ce89-11e9-b324-024e6a40328c

$ oc get proxy/cluster -o yaml
apiVersion: config.openshift.io/v1
kind: Proxy
metadata:
  creationTimestamp: "2019-09-03T20:32:34Z"
  generation: 1
  name: cluster
  resourceVersion: "432"
  selfLink: /apis/config.openshift.io/v1/proxies/cluster
  uid: f6149e65-ce89-11e9-b324-024e6a40328c
spec:
  httpProxy: http://jcallen:6cpbEH6uCepwEhNr2iB05ixP@52.73.102.120:3129
  httpsProxy: http://jcallen:6cpbEH6uCepwEhNr2iB05ixP@52.73.102.120:3129
  trustedCA:
    name: user-ca-bundle
status:
  httpProxy: http://jcallen:6cpbEH6uCepwEhNr2iB05ixP@52.73.102.120:3129
  httpsProxy: http://jcallen:6cpbEH6uCepwEhNr2iB05ixP@52.73.102.120:3129
  noProxy: .cluster.local,.svc,.us-west-2.compute.internal,10.0.0.0/16,10.128.0.0/14,127.0.0.1,169.254.169.254,172.30.0.0/16,api-int.latest-proxy.devcluster.openshift.com,api.latest-proxy.devcluster.openshift.com,etcd-0.latest-proxy.devcluster.openshift.com,etcd-1.latest-proxy.devcluster.openshift.com,etcd-2.latest-proxy.devcluster.openshift.com,localhost

Comment 10 XiuJuan Wang 2019-09-05 08:54:24 UTC
Can't reproduece this issue when upgrade from 4.1.14 to 4.2.0-0.nightly-2019-09-04-142146.

$ oc get co  image-registry   -o json  | jq '.status'
{
  "conditions": [
    {
      "lastTransitionTime": "2019-09-05T08:10:15Z",
      "message": "The registry is ready",
      "reason": "Ready",
      "status": "True",
      "type": "Available"
    },
    {
      "lastTransitionTime": "2019-09-05T08:10:15Z",
      "message": "The registry is ready",
      "reason": "Ready",
      "status": "False",
      "type": "Progressing"
    },
    {
      "lastTransitionTime": "2019-09-05T00:55:16Z",
      "status": "False",
      "type": "Degraded"
    }
  ],
  "extension": null,
  "relatedObjects": [
    {
      "group": "rbac.authorization.k8s.io",
      "name": "system:registry",
      "resource": "clusterroles"
    },
    {
      "group": "rbac.authorization.k8s.io",
      "name": "registry-registry-role",
      "resource": "clusterrolebindings"
    },
    {
      "group": "",
      "name": "registry",
      "namespace": "openshift-image-registry",
      "resource": "serviceaccounts"
    },
    {
      "group": "",
      "name": "serviceca",
      "namespace": "openshift-image-registry",
      "resource": "configmaps"
    },
    {
      "group": "",
      "name": "image-registry-certificates",
      "namespace": "openshift-image-registry",
      "resource": "configmaps"
    },
    {
      "group": "",
      "name": "image-registry-private-configuration",
      "namespace": "openshift-image-registry",
      "resource": "secrets"
    },
    {
      "group": "config.openshift.io",
      "name": "cluster",
      "resource": "images"
    },
    {
      "group": "apps",
      "name": "node-ca",
      "namespace": "openshift-image-registry",
      "resource": "daemonsets"
    },
    {
      "group": "",
      "name": "image-registry",
      "namespace": "openshift-image-registry",
      "resource": "services"
    },
    {
      "group": "apps",
      "name": "image-registry",
      "namespace": "openshift-image-registry",
      "resource": "deployments"
    }
  ],
  "versions": [
    {
      "name": "operator",
      "version": "4.2.0-0.nightly-2019-09-04-142146"
    }
  ]
}


Note You need to log in before you can comment on or make changes to this bug.