Bug 1619116

Description of problem:
Upgrade failed at [openshift_web_console : Verify that the console is running]
fatal: [x]: FAILED! => {"attempts": 60, "changed": false, "results": {"cmd": "/usr/local/bin/oc get deployment webconsole -o json -n openshift-web-console", "results": [{"apiVersion": "extensions/v1beta1", "kind": "Deployment", "metadata": {"annotations": {"deployment.kubernetes.io/revision": "1", "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"apps/v1beta1\",\"kind\":\"Deployment\",\"metadata\":{\"annotations\":{},\"labels\":{\"app\":\"openshift-web-console\",\"webconsole\":\"true\"},\"name\":\"webconsole\",\"namespace\":\"openshift-web-console\"},\"spec\":{\"replicas\":1,\"strategy\":{\"rollingUpdate\":{\"maxUnavailable\":\"100%\"},\"type\":\"RollingUpdate\"},\"template\":{\"metadata\":{\"labels\":{\"app\":\"openshift-web-console\",\"webconsole\":\"true\"},\"name\":\"webconsole\"},\"spec\":{\"containers\":[{\"command\":[\"/usr/bin/origin-web-console\",\"--audit-log-path=-\",\"-v=0\",\"--config=/var/webconsole-config/webconsole-config.yaml\"],\"image\":\"registry.reg-aws.openshift.com:443/openshift3/ose-web-console:v3.11\",\"imagePullPolicy\":\"IfNotPresent\",\"livenessProbe\":{\"exec\":{\"command\":[\"/bin/sh\",\"-c\",\"if [[ ! -f /tmp/webconsole-config.hash ]]; then \\\\\\n  md5sum /var/webconsole-config/webconsole-config.yaml \\u003e /tmp/webconsole-config.hash; \\\\\\nelif [[ $(md5sum /var/webconsole-config/webconsole-config.yaml) != $(cat /tmp/webconsole-config.hash) ]]; then \\\\\\n  echo 'webconsole-config.yaml has changed.'; \\\\\\n  exit 1; \\\\\\nfi \\u0026\\u0026 curl -k -f https://0.0.0.0:8443/console/\"]}},\"name\":\"webconsole\",\"ports\":[{\"containerPort\":8443}],\"readinessProbe\":{\"httpGet\":{\"path\":\"/healthz\",\"port\":8443,\"scheme\":\"HTTPS\"}},\"resources\":{\"requests\":{\"cpu\":\"100m\",\"memory\":\"100Mi\"}},\"volumeMounts\":[{\"mountPath\":\"/var/serving-cert\",\"name\":\"serving-cert\"},{\"mountPath\":\"/var/webconsole-config\",\"name\":\"webconsole-config\"}]}],\"nodeSelector\":{\"node-role.kubernetes.io/master\":\"true\"},\"serviceAccountName\":\"webconsole\",\"volumes\":[{\"name\":\"serving-cert\",\"secret\":{\"defaultMode\":288,\"secretName\":\"webconsole-serving-cert\"}},{\"configMap\":{\"defaultMode\":288,\"name\":\"webconsole-config\"},\"name\":\"webconsole-config\"}]}}}}\n"}, "creationTimestamp": "2018-08-20T03:20:33Z", "generation": 2, "labels": {"app": "openshift-web-console", "webconsole": "true"}, "name": "webconsole", "namespace": "openshift-web-console", "resourceVersion": "17898", "selfLink": "/apis/extensions/v1beta1/namespaces/openshift-web-console/deployments/webconsole", "uid": "ffcf4891-a427-11e8-8a66-42010af0003d"}, "spec": {"progressDeadlineSeconds": 600, "replicas": 1, "revisionHistoryLimit": 2, "selector": {"matchLabels": {"app": "openshift-web-console", "webconsole": "true"}}, "strategy": {"rollingUpdate": {"maxSurge": "25%", "maxUnavailable": "100%"}, "type": "RollingUpdate"}, "template": {"metadata": {"creationTimestamp": null, "labels": {"app": "openshift-web-console", "webconsole": "true"}, "name": "webconsole"}, "spec": {"containers": [{"command": ["/usr/bin/origin-web-console", "--audit-log-path=-", "-v=0", "--config=/var/webconsole-config/webconsole-config.yaml"], "image": "registry.reg-aws.openshift.com:443/openshift3/ose-web-console:v3.11", "imagePullPolicy": "IfNotPresent", "livenessProbe": {"exec": {"command": ["/bin/sh", "-c", "if [[ ! -f /tmp/webconsole-config.hash ]]; then \\\n  md5sum /var/webconsole-config/webconsole-config.yaml > /tmp/webconsole-config.hash; \\\nelif [[ $(md5sum /var/webconsole-config/webconsole-config.yaml) != $(cat /tmp/webconsole-config.hash) ]]; then \\\n  echo 'webconsole-config.yaml has changed.'; \\\n  exit 1; \\\nfi && curl -k -f https://0.0.0.0:8443/console/"]}, "failureThreshold": 3, "periodSeconds": 10, "successThreshold": 1, "timeoutSeconds": 1}, "name": "webconsole", "ports": [{"containerPort": 8443, "protocol": "TCP"}], "readinessProbe": {"failureThreshold": 3, "httpGet": {"path": "/healthz", "port": 8443, "scheme": "HTTPS"}, "periodSeconds": 10, "successThreshold": 1, "timeoutSeconds": 1}, "resources": {"requests": {"cpu": "100m", "memory": "100Mi"}}, "terminationMessagePath": "/dev/termination-log", "terminationMessagePolicy": "File", "volumeMounts": [{"mountPath": "/var/serving-cert", "name": "serving-cert"}, {"mountPath": "/var/webconsole-config", "name": "webconsole-config"}]}], "dnsPolicy": "ClusterFirst", "nodeSelector": {"node-role.kubernetes.io/master": "true"}, "restartPolicy": "Always", "schedulerName": "default-scheduler", "securityContext": {}, "serviceAccount": "webconsole", "serviceAccountName": "webconsole", "terminationGracePeriodSeconds": 30, "volumes": [{"name": "serving-cert", "secret": {"defaultMode": 288, "secretName": "webconsole-serving-cert"}}, {"configMap": {"defaultMode": 288, "name": "webconsole-config"}, "name": "webconsole-config"}]}}}, "status": {"conditions": [{"lastTransitionTime": "2018-08-20T03:20:33Z", "lastUpdateTime": "2018-08-20T03:20:33Z", "message": "Deployment has minimum availability.", "reason": "MinimumReplicasAvailable", "status": "True", "type": "Available"}, {"lastTransitionTime": "2018-08-20T03:20:33Z", "lastUpdateTime": "2018-08-20T03:20:47Z", "message": "ReplicaSet \"webconsole-785689b664\" has successfully progressed.", "reason": "NewReplicaSetAvailable", "status": "True", "type": "Progressing"}], "observedGeneration": 1, "replicas": 1, "unavailableReplicas": 1, "updatedReplicas": 1}}], "returncode": 0}, "state": "list"}

======================================================
Webconsole and master controller keep restart.
[root@qe-jliu-auto-dlvc-master-etcd-1 master]# oc get po
NAME                          READY     STATUS             RESTARTS   AGE
webconsole-785689b664-rkcb6   0/1       CrashLoopBackOff   21         2h

Events:
  Type     Reason     Age                From                                      Message
  ----     ------     ----               ----                                      
  Warning  Unhealthy  49m                kubelet, qe-jliu-auto-dlvc-master-etcd-1  Readiness probe failed: Get https://10.128.0.4:8443/healthz: net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)
  Normal   Pulled     49m (x4 over 50m)  kubelet, qe-jliu-auto-dlvc-master-etcd-1  Container image "registry.reg-aws.openshift.com:443/openshift3/ose-web-console:v3.10" already present on machine
  Normal   Created    49m (x4 over 50m)  kubelet, qe-jliu-auto-dlvc-master-etcd-1  Created container
  Normal   Started    49m (x4 over 50m)  kubelet, qe-jliu-auto-dlvc-master-etcd-1  Started container
  Warning  Unhealthy  49m                kubelet, qe-jliu-auto-dlvc-master-etcd-1  Liveness probe failed:   % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
  0     0    0     0    0     0      0      0 --:--:--  0:00:03 --:--:--     0
  Warning  BackOff  47s (x228 over 50m)  kubelet, qe-jliu-auto-dlvc-master-etcd-1  Back-off restarting failed container


[root@qe-jliu-auto-dlvc-master-etcd-1 master]# oc get pod -n kube-system
NAME                                                 READY     STATUS             RESTARTS   AGE
master-api-qe-jliu-auto-dlvc-master-etcd-1           1/1       Running            1          46m
master-controllers-qe-jliu-auto-dlvc-master-etcd-1   0/1       CrashLoopBackOff   14         46m
master-etcd-qe-jliu-auto-dlvc-master-etcd-1          1/1       Running            0          51m


[root@qe-jliu-auto-dlvc-master-etcd-1 master]# /usr/local/bin/master-logs controllers controllers
I0820 06:05:06.993234       1 client_builder.go:233] Verified credential for cluster-quota-reconciliation-controller/openshift-infra
I0820 06:05:07.010891       1 request.go:1099] body was not decodable (unable to check for Status): couldn't get version/kind; json parse error: json: cannot unmarshal string into Go value of type struct { APIVersion string "json:\"apiVersion,omitempty\""; Kind string "json:\"kind,omitempty\"" }
F0820 06:05:07.017402       1 controller_manager.go:127] Error starting "openshift.io/cluster-quota-reconciliation" (unable to retrieve the complete list of server APIs: servicecatalog.k8s.io/v1beta1: the server is currently unable to handle the request)


Version-Release number of the following components:
ansible-2.6.2-1.el7ae.noarch
openshift-ansible-3.11.0-0.17.0.git.0.d3888d7None.noarch

How reproducible:
always

Steps to Reproduce:
1. Run upgrade ocp  from v3.10 to v3.11
2.
3.

Actual results:
Upgrade failed.

Expected results:
Upgrade succeed.

Additional info:
Please attach logs from ansible-playbook with the -vvv flag