Bug 1957502

Summary:	Infrequent panic in kube-apiserver in aws-serial job
Product:	OpenShift Container Platform	Reporter:	Clayton Coleman <ccoleman>
Component:	kube-apiserver	Assignee:	David Eads <deads>
Status:	CLOSED ERRATA	QA Contact:	Ke Wang <kewang>
Severity:	high	Docs Contact:
Priority:	unspecified
Version:	4.8	CC:	aos-bugs, kewang, mfojtik, sttts, xxia
Target Milestone:	---
Target Release:	4.8.0
Hardware:	Unspecified
OS:	Unspecified
Whiteboard:
Fixed In Version:		Doc Type:	If docs needed, set a value
Doc Text:		Story Points:	---
Clone Of:		Environment:
Last Closed:	2021-07-27 23:06:39 UTC	Type:	Bug
Regression:	---	Mount Type:	---
Documentation:	---	CRM:
Verified Versions:		Category:	---
oVirt Team:	---	RHEL 7.3 requirements from Atomic Host:
Cloudforms Team:	---	Target Upstream Version:
Embargoed:

Description Clayton Coleman 2021-05-05 21:37:21 UTC

pods/openshift-kube-apiserver_kube-apiserver-ip-10-0-132-168.us-west-1.compute.internal_kube-apiserver.log.gz:E0505 14:38:27.582470      19 runtime.go:76] Observed a panic: runtime error: invalid memory address or nil pointer dereference

https://prow.ci.openshift.org/view/gs/origin-ci-test/logs/periodic-ci-openshift-release-master-ci-4.8-e2e-aws-serial/1389928085503086592

E0505 14:38:27.582470      19 runtime.go:76] Observed a panic: runtime error: invalid memory address or nil pointer dereference
goroutine 8505117 [running]:
k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/filters.(*timeoutHandler).ServeHTTP.func1.1(0xc02e9f9260)
	/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/filters/timeout.go:102 +0x125
panic(0x49d9640, 0x83684f0)
	/usr/lib/golang/src/runtime/panic.go:965 +0x1b9
k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/filters.WithAudit.func1.1(0xc0290eac80, 0x5cc4378, 0xc031866d50, 0x7fb4276122c0, 0xc000eae900, 0xc000eae8a0, 0x1, 0x1, 0x0, 0x0)
	/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/filters/audit.go:89 +0x216
panic(0x49d9640, 0x83684f0)
	/usr/lib/golang/src/runtime/panic.go:965 +0x1b9
type..eq.k8s.io/kubernetes/openshift-kube-apiserver/filters/deprecatedapirequest.userKey(0xc01a0b20d0, 0xc00f8d6db8, 0xd673691f539063a9)
	<autogenerated>:1 +0x99
k8s.io/kubernetes/openshift-kube-apiserver/filters/deprecatedapirequest.(*hourlyRequestCounts).User(0xc0451a19e0, 0xc0294d6100, 0x3b, 0xc046ef9f40, 0x1c, 0x0)
	/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/openshift-kube-apiserver/filters/deprecatedapirequest/request_counts.go:265 +0x197
k8s.io/kubernetes/openshift-kube-apiserver/filters/deprecatedapirequest.(*hourlyRequestCounts).IncrementRequestCount(0xc0451a19e0, 0xc0294d6100, 0x3b, 0xc046ef9f40, 0x1c, 0x5350ade, 0x3, 0x1)
	/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/openshift-kube-apiserver/filters/deprecatedapirequest/request_counts.go:276 +0x53
k8s.io/kubernetes/openshift-kube-apiserver/filters/deprecatedapirequest.(*resourceRequestCounts).IncrementRequestCount(0xc05fd0d860, 0xe, 0xc0294d6100, 0x3b, 0xc046ef9f40, 0x1c, 0x5350ade, 0x3, 0x1)
	/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/openshift-kube-apiserver/filters/deprecatedapirequest/request_counts.go:192 +0x96
k8s.io/kubernetes/openshift-kube-apiserver/filters/deprecatedapirequest.(*apiRequestCounts).IncrementRequestCount(0xc03f10b470, 0x0, 0x0, 0xc006eb9115, 0x2, 0xc006eb913b, 0xa, 0xe, 0xc0294d6100, 0x3b, ...)
	/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/openshift-kube-apiserver/filters/deprecatedapirequest/request_counts.go:92 +0xc6
k8s.io/kubernetes/openshift-kube-apiserver/filters/deprecatedapirequest.(*controller).LogRequest(0xc0007a1b30, 0x0, 0x0, 0xc006eb9115, 0x2, 0xc006eb913b, 0xa, 0xc01cc958e11902d2, 0x3dbeef6a1a3, 0x83aadc0, ...)
	/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/openshift-kube-apiserver/filters/deprecatedapirequest/apiaccess_count_controller.go:71 +0x1d5
k8s.io/kubernetes/openshift-kube-apiserver/filters.WithDeprecatedApiRequestLogging.func1(0x5cbd8a0, 0xc04bf2d030, 0xc04f504e00)
	/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/openshift-kube-apiserver/filters/deprecatedapirequest_filter.go:28 +0x390
net/http.HandlerFunc.ServeHTTP(0xc00c406870, 0x5cbd8a0, 0xc04bf2d030, 0xc04f504e00)
	/usr/lib/golang/src/net/http/server.go:2069 +0x44
k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/filterlatency.trackCompleted.func1(0x5cbd8a0, 0xc04bf2d030, 0xc04f504e00)
	/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/filterlatency/filterlatency.go:95 +0x193
net/http.HandlerFunc.ServeHTTP(0xc00c4068a0, 0x5cbd8a0, 0xc04bf2d030, 0xc04f504e00)
	/usr/lib/golang/src/net/http/server.go:2069 +0x44
k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/filters.WithAuthorization.func1(0x5cbd8a0, 0xc04bf2d030, 0xc04f504e00)
	/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/filters/authorization.go:64 +0x603
net/http.HandlerFunc.ServeHTTP(0xc0099f5d40, 0x5cbd8a0, 0xc04bf2d030, 0xc04f504e00)
	/usr/lib/golang/src/net/http/server.go:2069 +0x44
k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/filterlatency.trackStarted.func1(0x5cbd8a0, 0xc04bf2d030, 0xc04f504e00)
	/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/filterlatency/filterlatency.go:71 +0x186
net/http.HandlerFunc.ServeHTTP(0xc0099f5d80, 0x5cbd8a0, 0xc04bf2d030, 0xc04f504e00)
	/usr/lib/golang/src/net/http/server.go:2069 +0x44
k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/filterlatency.trackCompleted.func1(0x5cbd8a0, 0xc04bf2d030, 0xc04f504e00)
	/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/filterlatency/filterlatency.go:95 +0x193
net/http.HandlerFunc.ServeHTTP(0xc00c4068d0, 0x5cbd8a0, 0xc04bf2d030, 0xc04f504e00)
	/usr/lib/golang/src/net/http/server.go:2069 +0x44
k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/filters.WithPriorityAndFairness.func1.4()
	/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/filters/priority-and-fairness.go:127 +0x1ba
k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/util/flowcontrol.(*configController).Handle.func2()
	/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/util/flowcontrol/apf_filter.go:176 +0x222
k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/queueset.(*request).Finish.func1(0xc067667ad0, 0xc01a0b29ef, 0xc043506cb0)
	/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/queueset/queueset.go:329 +0x62
k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/queueset.(*request).Finish(0xc067667ad0, 0xc043506cb0, 0xc0444b3060)
	/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/queueset/queueset.go:330 +0x5d
k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/util/flowcontrol.(*configController).Handle(0xc000c5e600, 0x5cc4378, 0xc031866d50, 0xc053868e70, 0x5cc52c8, 0xc045f55140, 0xc0208eea90, 0xc0208eeaa0, 0xc0578a36e0)
	/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/util/flowcontrol/apf_filter.go:166 +0x907
k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/filters.WithPriorityAndFairness.func1(0x5cbd8a0, 0xc04bf2d030, 0xc0362a1700)
	/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/filters/priority-and-fairness.go:130 +0x606
net/http.HandlerFunc.ServeHTTP(0xc00c406900, 0x5cbd8a0, 0xc04bf2d030, 0xc0362a1700)
	/usr/lib/golang/src/net/http/server.go:2069 +0x44
k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/filterlatency.trackStarted.func1(0x5cbd8a0, 0xc04bf2d030, 0xc0362a1700)
	/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/filterlatency/filterlatency.go:71 +0x186
net/http.HandlerFunc.ServeHTTP(0xc0099f5dc0, 0x5cbd8a0, 0xc04bf2d030, 0xc0362a1700)
	/usr/lib/golang/src/net/http/server.go:2069 +0x44
k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/filterlatency.trackCompleted.func1(0x5cbd8a0, 0xc04bf2d030, 0xc0362a1700)
	/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/filterlatency/filterlatency.go:95 +0x193
net/http.HandlerFunc.ServeHTTP(0xc00c406930, 0x5cbd8a0, 0xc04bf2d030, 0xc0362a1700)
	/usr/lib/golang/src/net/http/server.go:2069 +0x44
k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/filters.WithImpersonation.func1(0x5cbd8a0, 0xc04bf2d030, 0xc0362a1700)
	/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/filters/impersonation.go:50 +0x240d
net/http.HandlerFunc.ServeHTTP(0xc0099f5e00, 0x5cbd8a0, 0xc04bf2d030, 0xc0362a1700)
	/usr/lib/golang/src/net/http/server.go:2069 +0x44
k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/filterlatency.trackStarted.func1(0x5cbd8a0, 0xc04bf2d030, 0xc0362a1700)
	/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/filterlatency/filterlatency.go:71 +0x186
net/http.HandlerFunc.ServeHTTP(0xc0099f5e40, 0x5cbd8a0, 0xc04bf2d030, 0xc0362a1700)
	/usr/lib/golang/src/net/http/server.go:2069 +0x44
k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/filterlatency.trackCompleted.func1(0x5cbd8a0, 0xc04bf2d030, 0xc0362a1700)
	/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/filterlatency/filterlatency.go:95 +0x193
net/http.HandlerFunc.ServeHTTP(0xc00c406960, 0x5cbd8a0, 0xc04bf2d030, 0xc0362a1700)
	/usr/lib/golang/src/net/http/server.go:2069 +0x44
k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/filters.WithAudit.func1(0x7fb426c8f228, 0xc04bf2d008, 0xc0362a1600)
	/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/filters/audit.go:111 +0x50d
net/http.HandlerFunc.ServeHTTP(0xc0099f5e80, 0x7fb426c8f228, 0xc04bf2d008, 0xc0362a1600)
	/usr/lib/golang/src/net/http/server.go:2069 +0x44
k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/filterlatency.trackStarted.func1(0x7fb426c8f228, 0xc04bf2d008, 0xc0362a1600)
	/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/filterlatency/filterlatency.go:71 +0x186
net/http.HandlerFunc.ServeHTTP(0xc0099f5ec0, 0x7fb426c8f228, 0xc04bf2d008, 0xc0362a1600)
	/usr/lib/golang/src/net/http/server.go:2069 +0x44
k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/filterlatency.trackCompleted.func1(0x7fb426c8f228, 0xc04bf2d008, 0xc0362a1600)
	/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/filterlatency/filterlatency.go:95 +0x193
net/http.HandlerFunc.ServeHTTP(0xc00c4069c0, 0x7fb426c8f228, 0xc04bf2d008, 0xc0362a1600)
	/usr/lib/golang/src/net/http/server.go:2069 +0x44
k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/filters.withAuthentication.func1(0x7fb426c8f228, 0xc04bf2d008, 0xc0362a1600)
	/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/filters/authentication.go:80 +0x75c
net/http.HandlerFunc.ServeHTTP(0xc0013f6fc0, 0x7fb426c8f228, 0xc04bf2d008, 0xc0362a0e00)
	/usr/lib/golang/src/net/http/server.go:2069 +0x44
k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/filterlatency.trackStarted.func1(0x7fb426c8f228, 0xc04bf2d008, 0xc0362a0d00)
	/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/filterlatency/filterlatency.go:80 +0x38c
net/http.HandlerFunc.ServeHTTP(0xc0099f5f40, 0x7fb426c8f228, 0xc04bf2d008, 0xc0362a0d00)
	/usr/lib/golang/src/net/http/server.go:2069 +0x44
k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/filters.WithCORS.func1(0x7fb426c8f228, 0xc04bf2d008, 0xc0362a0d00)
	/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/filters/cors.go:75 +0x1d9
net/http.HandlerFunc.ServeHTTP(0xc0013f70e0, 0x7fb426c8f228, 0xc04bf2d008, 0xc0362a0d00)
	/usr/lib/golang/src/net/http/server.go:2069 +0x44
k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/filters.(*timeoutHandler).ServeHTTP.func1(0xc02e9f9260, 0xc00eb0cea0, 0x5cc53a8, 0xc04bf2d008, 0xc0362a0d00)
	/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/filters/timeout.go:107 +0xb8
created by k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/filters.(*timeoutHandler).ServeHTTP
	/go/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/filters/timeout.go:93 +0x1f4

Comment 1 Clayton Coleman 2021-05-05 21:38:41 UTC

Also https://prow.ci.openshift.org/view/gs/origin-ci-test/pr-logs/pull/openshift_installer/4876/pull-ci-openshift-installer-master-e2e-aws/1389774810229248000 and https://prow.ci.openshift.org/view/gs/origin-ci-test/pr-logs/pull/openshift_builder/244/pull-ci-openshift-builder-master-e2e-aws-builds/1389872188479246336

Comment 2 Stefan Schimanski 2021-05-06 09:48:31 UTC

Looks like a bug in request count code.

Comment 7 errata-xmlrpc 2021-07-27 23:06:39 UTC

Since the problem described in this bug report should be
resolved in a recent advisory, it has been closed with a
resolution of ERRATA.

For information on the advisory (Moderate: OpenShift Container Platform 4.8.2 bug fix and security update), and where to find the updated
files, follow the link below.

If the solution does not work for you, open a new bug report.

https://access.redhat.com/errata/RHSA-2021:2438