Description of problem: Upgrade from 4.1.21 to 4.2.2 on channel fast-4.2 failing. clusteroperator/machine-config is degraded with error "Unable to apply 4.2.2: timed out waiting for the condition during waitForControllerConfigToBeCompleted: controllerconfig is not completed: status for ControllerConfig machine-config-controller is being reported for 14, expecting it for 15" I1031 16:18:10.014435 1 start.go:50] Version: v4.2.1-201910230818-dirty (d73d5c6c95499f2820853c799366a157b0c0bd09) E1031 16:20:05.732688 1 event.go:247] Could not construct reference to: '&v1.ConfigMap{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"machine-config-controller", GenerateName:"", Namespace:"openshift-machine-config-operator", SelfLink:"/api/v1/namespaces/openshift-machine-config-operator/configmaps/machine-config-controller", UID:"cb9e6e13-ad92-11e9-b23e-0a35c159340a", ResourceVersion:"85972205", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:63699514960, loc:(*time.Location)(0x28f62e0)}}, DeletionTimestamp:(*v1.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string{"control-plane.alpha.kubernetes.io/leader":"{\"holderIdentity\":\"machine-config-controller-f6bc569fd-thfqh_07a4f5a8-fbfa-11e9-80a0-0a580a8200c9\",\"leaseDurationSeconds\":90,\"acquireTime\":\"2019-10-31T16:20:05Z\",\"renewTime\":\"2019-10-31T16:20:05Z\",\"leaderTransitions\":58}"}, OwnerReferences:[]v1.OwnerReference(nil), Initializers:(*v1.Initializers)(nil), Finalizers:[]string(nil), ClusterName:"", ManagedFields:[]v1.ManagedFieldsEntry(nil)}, Data:map[string]string(nil), BinaryData:map[string][]uint8(nil)}' due to: 'no kind is registered for the type v1.ConfigMap in scheme "github.com/openshift/machine-config-operator/cmd/common/helpers.go:30"'. Will not report event: 'Normal' 'LeaderElection' 'machine-config-controller-f6bc569fd-thfqh_07a4f5a8-fbfa-11e9-80a0-0a580a8200c9 became leader' E1031 16:20:05.867664 1 runtime.go:69] Observed a panic: "invalid memory address or nil pointer dereference" (runtime error: invalid memory address or nil pointer dereference) /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:76 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:65 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:51 /opt/rh/go-toolset-1.11/root/usr/lib/go-toolset-1.11-golang/src/runtime/asm_amd64.s:522 /opt/rh/go-toolset-1.11/root/usr/lib/go-toolset-1.11-golang/src/runtime/panic.go:513 /opt/rh/go-toolset-1.11/root/usr/lib/go-toolset-1.11-golang/src/runtime/panic.go:82 /opt/rh/go-toolset-1.11/root/usr/lib/go-toolset-1.11-golang/src/runtime/signal_unix.go:390 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache/store.go:84 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache/controller.go:261 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache/controller.go:261 /go/src/github.com/openshift/machine-config-operator/pkg/controller/node/node_controller.go:604 /go/src/github.com/openshift/machine-config-operator/pkg/controller/node/node_controller.go:615 /go/src/github.com/openshift/machine-config-operator/pkg/controller/node/node_controller.go:125 /go/src/github.com/openshift/machine-config-operator/pkg/controller/node/node_controller.go:397 /go/src/github.com/openshift/machine-config-operator/pkg/controller/node/node_controller.go:115 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache/controller.go:195 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache/shared_informer.go:554 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:265 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:265 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache/shared_informer.go:548 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache/shared_informer.go:546 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache/shared_informer.go:390 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 /opt/rh/go-toolset-1.11/root/usr/lib/go-toolset-1.11-golang/src/runtime/asm_amd64.s:1333 panic: runtime error: invalid memory address or nil pointer dereference [recovered] panic: runtime error: invalid memory address or nil pointer dereference [signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0xe60bf5] goroutine 184 [running]: github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/runtime.HandleCrash(0x0, 0x0, 0x0) /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:58 +0x108 panic(0x15aec80, 0x28dbcd0) /opt/rh/go-toolset-1.11/root/usr/lib/go-toolset-1.11-golang/src/runtime/panic.go:513 +0x1b9 github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1.(*MachineConfigPool).GetNamespace(0x0, 0x0, 0x19e98a0) <autogenerated>:1 +0x5 github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache.MetaNamespaceKeyFunc(0x1778ea0, 0x0, 0xc0007825a0, 0x1528080, 0xc000446320, 0x12a05f200) /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache/store.go:84 +0x114 github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache.DeletionHandlingMetaNamespaceKeyFunc(0x1778ea0, 0x0, 0xc000446320, 0x12a05f200, 0x0, 0x0) /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache/controller.go:261 +0x6a github.com/openshift/machine-config-operator/pkg/controller/node.(*Controller).enqueueAfter(0xc0004ec000, 0x0, 0x12a05f200) /go/src/github.com/openshift/machine-config-operator/pkg/controller/node/node_controller.go:604 +0x41 github.com/openshift/machine-config-operator/pkg/controller/node.(*Controller).enqueueDefault(0xc0004ec000, 0x0) /go/src/github.com/openshift/machine-config-operator/pkg/controller/node/node_controller.go:615 +0x44 github.com/openshift/machine-config-operator/pkg/controller/node.(*Controller).enqueueDefault-fm(0x0) /go/src/github.com/openshift/machine-config-operator/pkg/controller/node/node_controller.go:125 +0x34 github.com/openshift/machine-config-operator/pkg/controller/node.(*Controller).addNode(0xc0004ec000, 0x1794e00, 0xc0003312c0) /go/src/github.com/openshift/machine-config-operator/pkg/controller/node/node_controller.go:397 +0x1a1 github.com/openshift/machine-config-operator/pkg/controller/node.(*Controller).addNode-fm(0x1794e00, 0xc0003312c0) /go/src/github.com/openshift/machine-config-operator/pkg/controller/node/node_controller.go:115 +0x3e github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache.ResourceEventHandlerFuncs.OnAdd(0xc0002fae00, 0xc0002fae10, 0xc0002fae20, 0x1794e00, 0xc0003312c0) /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache/controller.go:195 +0x49 github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x0, 0xc000848d80, 0xc0008b9270) /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache/shared_informer.go:554 +0x21d github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc000093e18, 0x429692, 0xc0008b92a0) /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:265 +0x51 github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache.(*processorListener).run.func1() /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache/shared_informer.go:548 +0x79 github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00081f768) /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x54 github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000093f68, 0xdf8475800, 0x0, 0x1582101, 0xc0004becc0) /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xbe github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait.Until(0xc00081f768, 0xdf8475800, 0xc0004becc0) /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88 +0x4d github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache.(*processorListener).run(0xc0004fa980) /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache/shared_informer.go:546 +0x8d github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache.(*processorListener).run-fm() /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache/shared_informer.go:390 +0x2a github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc000589e20, 0xc000196230) /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x4f created by github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait.(*Group).Start /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62 Version-Release number of selected component (if applicable): How reproducible: Steps to Reproduce: 1. 2. 3. Actual results: Expected results: Additional info: Link to must-gather file: https://access.redhat.com/hydra/rest/cases/02507901/attachments/94a00cc8-a53b-46b7-a07c-924ad9c15e5f?usePresignedUrl=true
@Jerry: if you check the must gather and see the pivot error in the MCD logs, you can mark as a dupe for the above ^^
Looks very similar, trying to go 4.1.23 -> 4.2.4: I1115 05:55:28.977780 1 start.go:50] Version: v4.2.4-201911050122-dirty (55bb5fc17da0c3d76e4ee6a55732f0cba93e8520) E1115 05:57:24.633975 1 event.go:247] Could not construct reference to: '&v1.ConfigMap{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"machine-config-controller", GenerateName:"", Namespace:"openshift-machine-config-operator", SelfLink:"/api/v1/namespaces/openshift-machine-config-operator/configmaps/machine-config-controller", UID:"940c8210-bf6a-11e9-bd90-005056ba4735", ResourceVersion:"55357601", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:63701476808, loc:(*time.Location)(0x28f62e0)}}, DeletionTimestamp:(*v1.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string{"control-plane.alpha.kubernetes.io/leader":"{\"holderIdentity\":\"machine-config-controller-8455fcc6dc-vjf8t_86ed5c56-076c-11ea-984c-0a580a8101b6\",\"leaseDurationSeconds\":90,\"acquireTime\":\"2019-11-15T05:57:24Z\",\"renewTime\":\"2019-11-15T05:57:24Z\",\"leaderTransitions\":36}"}, OwnerReferences:[]v1.OwnerReference(nil), Initializers:(*v1.Initializers)(nil), Finalizers:[]string(nil), ClusterName:"", ManagedFields:[]v1.ManagedFieldsEntry(nil)}, Data:map[string]string(nil), BinaryData:map[string][]uint8(nil)}' due to: 'no kind is registered for the type v1.ConfigMap in scheme "github.com/openshift/machine-config-operator/cmd/common/helpers.go:30"'. Will not report event: 'Normal' 'LeaderElection' 'machine-config-controller-8455fcc6dc-vjf8t_86ed5c56-076c-11ea-984c-0a580a8101b6 became leader' E1115 05:57:24.680814 1 runtime.go:69] Observed a panic: "invalid memory address or nil pointer dereference" (runtime error: invalid memory address or nil pointer dereference) /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:76 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:65 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:51 /opt/rh/go-toolset-1.11/root/usr/lib/go-toolset-1.11-golang/src/runtime/asm_amd64.s:522 /opt/rh/go-toolset-1.11/root/usr/lib/go-toolset-1.11-golang/src/runtime/panic.go:513 /opt/rh/go-toolset-1.11/root/usr/lib/go-toolset-1.11-golang/src/runtime/panic.go:82 /opt/rh/go-toolset-1.11/root/usr/lib/go-toolset-1.11-golang/src/runtime/signal_unix.go:390 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache/store.go:84 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache/controller.go:261 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache/controller.go:261 /go/src/github.com/openshift/machine-config-operator/pkg/controller/node/node_controller.go:604 /go/src/github.com/openshift/machine-config-operator/pkg/controller/node/node_controller.go:615 /go/src/github.com/openshift/machine-config-operator/pkg/controller/node/node_controller.go:125 /go/src/github.com/openshift/machine-config-operator/pkg/controller/node/node_controller.go:397 /go/src/github.com/openshift/machine-config-operator/pkg/controller/node/node_controller.go:115 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache/controller.go:195 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache/shared_informer.go:554 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:265 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:265 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache/shared_informer.go:548 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache/shared_informer.go:546 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache/shared_informer.go:390 /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 /opt/rh/go-toolset-1.11/root/usr/lib/go-toolset-1.11-golang/src/runtime/asm_amd64.s:1333 panic: runtime error: invalid memory address or nil pointer dereference [recovered] panic: runtime error: invalid memory address or nil pointer dereference [signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0xe60bf5] goroutine 212 [running]: github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/runtime.HandleCrash(0x0, 0x0, 0x0) /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:58 +0x108 panic(0x15aec80, 0x28dbcd0) /opt/rh/go-toolset-1.11/root/usr/lib/go-toolset-1.11-golang/src/runtime/panic.go:513 +0x1b9 github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1.(*MachineConfigPool).GetNamespace(0x0, 0x0, 0x19e98a0) <autogenerated>:1 +0x5 github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache.MetaNamespaceKeyFunc(0x1778ea0, 0x0, 0xc000140a20, 0x1528080, 0xc000954540, 0x12a05f200) /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache/store.go:84 +0x114 github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache.DeletionHandlingMetaNamespaceKeyFunc(0x1778ea0, 0x0, 0xc000954540, 0x12a05f200, 0x0, 0x0) /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache/controller.go:261 +0x6a github.com/openshift/machine-config-operator/pkg/controller/node.(*Controller).enqueueAfter(0xc00013c640, 0x0, 0x12a05f200) /go/src/github.com/openshift/machine-config-operator/pkg/controller/node/node_controller.go:604 +0x41 github.com/openshift/machine-config-operator/pkg/controller/node.(*Controller).enqueueDefault(0xc00013c640, 0x0) /go/src/github.com/openshift/machine-config-operator/pkg/controller/node/node_controller.go:615 +0x44 github.com/openshift/machine-config-operator/pkg/controller/node.(*Controller).enqueueDefault-fm(0x0) /go/src/github.com/openshift/machine-config-operator/pkg/controller/node/node_controller.go:125 +0x34 github.com/openshift/machine-config-operator/pkg/controller/node.(*Controller).addNode(0xc00013c640, 0x1794e00, 0xc00034d580) /go/src/github.com/openshift/machine-config-operator/pkg/controller/node/node_controller.go:397 +0x1a1 github.com/openshift/machine-config-operator/pkg/controller/node.(*Controller).addNode-fm(0x1794e00, 0xc00034d580) /go/src/github.com/openshift/machine-config-operator/pkg/controller/node/node_controller.go:115 +0x3e github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache.ResourceEventHandlerFuncs.OnAdd(0xc0003ca9c0, 0xc0003ca9d0, 0xc0003ca9e0, 0x1794e00, 0xc00034d580) /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache/controller.go:195 +0x49 github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x0, 0xc0003f0000, 0xc0000a7720) /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache/shared_informer.go:554 +0x21d github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc00098fe18, 0x429692, 0xc0000a7750) /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:265 +0x51 github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache.(*processorListener).run.func1() /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache/shared_informer.go:548 +0x79 github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0006abf68) /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x54 github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00098ff68, 0xdf8475800, 0x0, 0x1582101, 0xc000565140) /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xbe github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait.Until(0xc0006abf68, 0xdf8475800, 0xc000565140) /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88 +0x4d github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache.(*processorListener).run(0xc000330300) /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache/shared_informer.go:546 +0x8d github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache.(*processorListener).run-fm() /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/client-go/tools/cache/shared_informer.go:390 +0x2a github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00040ced0, 0xc0009542b0) /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x4f created by github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait.(*Group).Start /go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62
@Chet The crash issue is being tracked here: https://github.com/openshift/machine-config-operator/issues/1270 Looks like a nil is being pushed into a queue.
*** This bug has been marked as a duplicate of bug 1775009 ***