Description of problem: https://openshift-gce-devel.appspot.com/build/origin-ci-test/pr-logs/pull/openshift_installer/1653/pull-ci-openshift-installer-master-e2e-aws/5495 failed quite a few tests on trying to fetch logs from pods. curl -s https://storage.googleapis.com/origin-ci-test/pr-logs/pull/openshift_installer/1653/pull-ci-openshift-installer-master-e2e-aws/5495/artifacts/e2e-aws/csr.json | jq '.items[] | select (.status == {})' { "apiVersion": "certificates.k8s.io/v1beta1", "kind": "CertificateSigningRequest", "metadata": { "creationTimestamp": "2019-04-22T20:13:11Z", "generateName": "csr-", "name": "csr-2wdn9", "resourceVersion": "16933", "selfLink": "/apis/certificates.k8s.io/v1beta1/certificatesigningrequests/csr-2wdn9", "uid": "0d639f16-653b-11e9-8279-0a78a3c1e790" }, "spec": { "groups": [ "system:nodes", "system:authenticated" ], "request": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURSBSRVFVRVNULS0tLS0KTUlJQlJqQ0I3QUlCQURCS01SVXdFd1lEVlFRS0V3eHplWE4wWlcwNmJtOWtaWE14TVRBdkJnTlZCQU1US0hONQpjM1JsYlRwdWIyUmxPbWx3TFRFd0xUQXRNVFF4TFRJMU1DNWxZekl1YVc1MFpYSnVZV3d3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFRSWlqNUJ5NVZ2TGhYcklsa05paTZJTmU1RXovOHF1djJpV3NTa2lHU2sKNnUzMGJHTUNHMnI3alB0SVRTZVhoZXA4SCtobHhpQ0F2UFY4c0EzSXp0YkpvRUF3UGdZSktvWklodmNOQVFrTwpNVEV3THpBdEJnTlZIUkVFSmpBa2doeHBjQzB4TUMwd0xURTBNUzB5TlRBdVpXTXlMbWx1ZEdWeWJtRnNod1FLCkFJMzZNQW9HQ0NxR1NNNDlCQU1DQTBrQU1FWUNJUURJMHZycjRyUWNFZW9oYndzRnRpbmZFYTBEZC8rcmxrQWUKNnhxUFhHZUg4UUloQUk4TWprUGFScGVLcjgxWnVpMmxpR3ZqNklDdTJUaGZ5amw5K01iU3ZrUkMKLS0tLS1FTkQgQ0VSVElGSUNBVEUgUkVRVUVTVC0tLS0tCg==", "usages": [ "digital signature", "key encipherment", "server auth" ], "username": "system:node:ip-10-0-141-250.ec2.internal" }, "status": {} } { "apiVersion": "certificates.k8s.io/v1beta1", "kind": "CertificateSigningRequest", "metadata": { "creationTimestamp": "2019-04-22T20:03:11Z", "generateName": "csr-", "name": "csr-44v97", "resourceVersion": "6891", "selfLink": "/apis/certificates.k8s.io/v1beta1/certificatesigningrequests/csr-44v97", "uid": "a7a6c9e4-6539-11e9-b953-12ba86d8a5e6" }, "spec": { "groups": [ "system:nodes", "system:authenticated" ], "request": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURSBSRVFVRVNULS0tLS0KTUlJQlJqQ0I3QUlCQURCS01SVXdFd1lEVlFRS0V3eHplWE4wWlcwNmJtOWtaWE14TVRBdkJnTlZCQU1US0hONQpjM1JsYlRwdWIyUmxPbWx3TFRFd0xUQXRNVFV6TFRFM055NWxZekl1YVc1MFpYSnVZV3d3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFTSFIyUzBjMDZqeXFiazZOTGxDVlVTS282N1I5ditENVFrUkVLaHpqdXYKQlZ4enFtVFZYUEJkSHRKR01tVHk3Q2xmUjRleTVnYmVxcytCZjg4M2YyOHBvRUF3UGdZSktvWklodmNOQVFrTwpNVEV3THpBdEJnTlZIUkVFSmpBa2doeHBjQzB4TUMwd0xURTFNeTB4TnpjdVpXTXlMbWx1ZEdWeWJtRnNod1FLCkFKbXhNQW9HQ0NxR1NNNDlCQU1DQTBrQU1FWUNJUURtM2ZBT2MxUWlVWVZwOGFleXcxMGpTbjdDOG1INU9QWnQKYVd5VjR2aU41Z0loQUwyektMRXlDeXBOdC9yMXdLV080ZWh2SW83bEdlbVdjUXBaaE9JZ1hVeGcKLS0tLS1FTkQgQ0VSVElGSUNBVEUgUkVRVUVTVC0tLS0tCg==", "usages": [ "digital signature", "key encipherment", "server auth" ], "username": "system:node:ip-10-0-153-177.ec2.internal" }, "status": {} } { "apiVersion": "certificates.k8s.io/v1beta1", "kind": "CertificateSigningRequest", "metadata": { "creationTimestamp": "2019-04-22T20:03:00Z", "generateName": "csr-", "name": "csr-kgn9k", "resourceVersion": "6747", "selfLink": "/apis/certificates.k8s.io/v1beta1/certificatesigningrequests/csr-kgn9k", "uid": "a11c8c62-6539-11e9-b953-12ba86d8a5e6" }, "spec": { "groups": [ "system:nodes", "system:authenticated" ], "request": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURSBSRVFVRVNULS0tLS0KTUlJQlJUQ0I3QUlCQURCS01SVXdFd1lEVlFRS0V3eHplWE4wWlcwNmJtOWtaWE14TVRBdkJnTlZCQU1US0hONQpjM1JsYlRwdWIyUmxPbWx3TFRFd0xUQXRNVFF4TFRJMU1DNWxZekl1YVc1MFpYSnVZV3d3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFSMzRGemtoT3hQblcxODZFOGRqNHJnN0RkSm1tT1VPSWljd3pUN0JMSUsKdEtCVWRtWDhWKzVnNzZVeXFPNHhOT0pDVGpLSjF5NXZSR0hueENIaVg5QW9vRUF3UGdZSktvWklodmNOQVFrTwpNVEV3THpBdEJnTlZIUkVFSmpBa2doeHBjQzB4TUMwd0xURTBNUzB5TlRBdVpXTXlMbWx1ZEdWeWJtRnNod1FLCkFJMzZNQW9HQ0NxR1NNNDlCQU1DQTBnQU1FVUNJREVLSzA3L2VIOUZJK2hPWnZsRXhrTWVxNkFrSml4M1NIVVYKU1RDVm5LVkVBaUVBc1Uzek1EQmFvVVFubmx6VTZzNUo5ZmJjMnlSa3QycCtKb2ZMVGxCbmFWOD0KLS0tLS1FTkQgQ0VSVElGSUNBVEUgUkVRVUVTVC0tLS0tCg==", "usages": [ "digital signature", "key encipherment", "server auth" ], "username": "system:node:ip-10-0-141-250.ec2.internal" }, "status": {} } { "apiVersion": "certificates.k8s.io/v1beta1", "kind": "CertificateSigningRequest", "metadata": { "creationTimestamp": "2019-04-22T20:03:12Z", "generateName": "csr-", "name": "csr-t9xmn", "resourceVersion": "6902", "selfLink": "/apis/certificates.k8s.io/v1beta1/certificatesigningrequests/csr-t9xmn", "uid": "a830af51-6539-11e9-b953-12ba86d8a5e6" }, "spec": { "groups": [ "system:nodes", "system:authenticated" ], "request": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURSBSRVFVRVNULS0tLS0KTUlJQlJUQ0I3QUlCQURCS01SVXdFd1lEVlFRS0V3eHplWE4wWlcwNmJtOWtaWE14TVRBdkJnTlZCQU1US0hONQpjM1JsYlRwdWIyUmxPbWx3TFRFd0xUQXRNVFk1TFRJd01TNWxZekl1YVc1MFpYSnVZV3d3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFUeCtEcXdTenJRcFdEZXlDMDFQdFU5UCtTcXpXdFk2K3Y2a1gwL1dXNzQKUTJGcmhsZisrMkovUnV3MDVkS1VKQ1h3MzdkS3Q3Qlc2LzNkSDI4NktIU0JvRUF3UGdZSktvWklodmNOQVFrTwpNVEV3THpBdEJnTlZIUkVFSmpBa2doeHBjQzB4TUMwd0xURTJPUzB5TURFdVpXTXlMbWx1ZEdWeWJtRnNod1FLCkFLbkpNQW9HQ0NxR1NNNDlCQU1DQTBnQU1FVUNJUUNjUTJqbkp1NVQzaW1CeGlMUDFlUEVySklJaFdyY1Z4SncKZW5BN01Rd2k0Z0lnWHpEOU40ckdrK0hyMGJXSlN5UDN2TmJWNVI0NTBtTzFINmdkLzhGdW4vQT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUgUkVRVUVTVC0tLS0tCg==", "usages": [ "digital signature", "key encipherment", "server auth" ], "username": "system:node:ip-10-0-169-201.ec2.internal" }, "status": {} } [2:05:33] ➜ machine-config-operator git:(plumb_cloud_config) curl -s https://storage.googleapis.com/origin-ci-test/pr-logs/pull/openshift_installer/1653/pull-ci-openshift-installer-master-e2e-aws/5495/artifacts/e2e-aws/pods/openshift-cluster-machine-approver_machine-approver-5bb6cfc4c6-5n2x6_machine-approver-controller.log.gz | gunzip | rg 'csr-t9xmn' I0422 20:03:12.085702 1 main.go:97] CSR csr-t9xmn added I0422 20:03:12.086388 1 main.go:166] Error syncing csr csr-t9xmn: Put https://127.0.0.1:6443/apis/certificates.k8s.io/v1beta1/certificatesigningrequests/csr-t9xmn/approval: dial tcp 127.0.0.1:6443: connect: connection refused I0422 20:03:12.091578 1 main.go:97] CSR csr-t9xmn added I0422 20:03:12.091595 1 main.go:107] CSR csr-t9xmn is already approved [2:08:25] ➜ machine-config-operator git:(plumb_cloud_config) curl -s https://storage.googleapis.com/origin-ci-test/pr-logs/pull/openshift_installer/1653/pull-ci-openshift-installer-master-e2e-aws/5495/artifacts/e2e-aws/pods/openshift-cluster-machine-approver_machine-approver-5bb6cfc4c6-5n2x6_machine-approver-controller.log.gz | gunzip | rg 'csr-kgn9k' I0422 20:03:00.209750 1 main.go:97] CSR csr-kgn9k added I0422 20:03:00.210473 1 main.go:166] Error syncing csr csr-kgn9k: Put https://127.0.0.1:6443/apis/certificates.k8s.io/v1beta1/certificatesigningrequests/csr-kgn9k/approval: dial tcp 127.0.0.1:6443: connect: connection refused I0422 20:03:00.215668 1 main.go:97] CSR csr-kgn9k added I0422 20:03:00.215695 1 main.go:107] CSR csr-kgn9k is already approved It seems like the CSRs are not approved but the cluster-machine-approver is still marking them as already approved?
This is most likely due to doing status updates on the informer cache instead of a copy. I've opened https://github.com/openshift/cluster-machine-approver/pull/19 for this.
Verified. 4.1.0-0.nightly-2019-04-23-223857 $ curl -s https://storage.googleapis.com/origin-ci-test/logs/release-openshift-ocp-installer-e2e-aws-4.1/58/artifacts/e2e-aws/pods/openshift-cluster-machine-approver_machine-approver-86c845b897-gjm72_machine-approver-controller.log.gz | gunzip | rg 'csr-' I0423 23:03:03.565380 1 main.go:97] CSR csr-crm9v added I0423 23:03:03.617276 1 main.go:149] CSR csr-crm9v approved I0423 23:03:03.625515 1 main.go:97] CSR csr-jp497 added I0423 23:03:03.625587 1 main.go:107] CSR csr-jp497 is already approved I0423 23:03:03.625704 1 main.go:97] CSR csr-mnmd9 added I0423 23:03:03.625756 1 main.go:107] CSR csr-mnmd9 is already approved I0423 23:03:03.625859 1 main.go:97] CSR csr-nhbh7 added I0423 23:03:03.648124 1 main.go:149] CSR csr-nhbh7 approved I0423 23:03:03.648277 1 main.go:97] CSR csr-qgm7v added I0423 23:03:03.648328 1 main.go:107] CSR csr-qgm7v is already approved I0423 23:03:03.648373 1 main.go:97] CSR csr-bhjn8 added I0423 23:03:03.675538 1 main.go:149] CSR csr-bhjn8 approved I0423 23:04:58.590703 1 main.go:97] CSR csr-zv5dw added I0423 23:04:58.614571 1 main.go:123] CSR csr-zv5dw not authorized: Doesn't match expected prefix I0423 23:05:03.836035 1 main.go:97] CSR csr-lg4nl added I0423 23:05:03.864861 1 main.go:123] CSR csr-lg4nl not authorized: Doesn't match expected prefix I0423 23:05:10.835145 1 main.go:97] CSR csr-nnk5m added I0423 23:05:10.899159 1 main.go:149] CSR csr-nnk5m approved I0423 23:05:15.973982 1 main.go:97] CSR csr-wfcm2 added I0423 23:05:15.994951 1 main.go:149] CSR csr-wfcm2 approved I0423 23:05:20.986958 1 main.go:97] CSR csr-xnpbl added I0423 23:05:20.999129 1 main.go:123] CSR csr-xnpbl not authorized: Doesn't match expected prefix I0423 23:05:33.763003 1 main.go:97] CSR csr-gqv2q added I0423 23:05:33.791896 1 main.go:149] CSR csr-gqv2q approved I0423 23:18:07.661499 1 main.go:97] CSR tester-csr-5ht97 added I0423 23:18:07.673536 1 main.go:123] CSR tester-csr-5ht97 not authorized: Doesn't match expected prefix
Since the problem described in this bug report should be resolved in a recent advisory, it has been closed with a resolution of ERRATA. For information on the advisory, and where to find the updated files, follow the link below. If the solution does not work for you, open a new bug report. https://access.redhat.com/errata/RHBA-2019:0758