Bug 1900013

Summary: Revert KUBELET_LOG_LEVEL back to level 3 (
Product: OpenShift Container Platform Reporter: Ryan Phillips <rphillips>
Component: Machine Config OperatorAssignee: Antonio Murdaca <amurdaca>
Machine Config Operator sub component: Machine Config Operator QA Contact: Michael Nguyen <mnguyen>
Status: CLOSED ERRATA Docs Contact:
Severity: urgent    
Priority: urgent CC: amurdaca, erich, john.johansson, kgarriso, mkrejci, mnguyen, mrobson, nnosenzo, rheinzma, rioliu, rphillips, scott.worthington, sreber, ssadhale, steven.barre
Version: 4.6   
Target Milestone: ---   
Target Release: 4.5.z   
Hardware: All   
OS: Linux   
Whiteboard:
Fixed In Version: Doc Type: If docs needed, set a value
Doc Text:
Story Points: ---
Clone Of: 1896329 Environment:
Last Closed: 2020-12-15 20:28:46 UTC Type: ---
Regression: --- Mount Type: ---
Documentation: --- CRM:
Verified Versions: Category: ---
oVirt Team: --- RHEL 7.3 requirements from Atomic Host:
Cloudforms Team: --- Target Upstream Version:
Embargoed:
Bug Depends On: 1896329    
Bug Blocks:    

Comment 3 Michael Nguyen 2020-12-07 15:47:31 UTC
$ export KUBECONFIG=/home/mnguyen/kubeconfig
$ oc get nodes
NAME                                       STATUS   ROLES    AGE   VERSION
ci-ln-b2r0562-f76d1-b97km-master-0         Ready    master   55m   v1.18.3+fa69cae
ci-ln-b2r0562-f76d1-b97km-master-1         Ready    master   56m   v1.18.3+fa69cae
ci-ln-b2r0562-f76d1-b97km-master-2         Ready    master   55m   v1.18.3+fa69cae
ci-ln-b2r0562-f76d1-b97km-worker-b-v4477   Ready    worker   40m   v1.18.3+fa69cae
ci-ln-b2r0562-f76d1-b97km-worker-c-p2pdc   Ready    worker   41m   v1.18.3+fa69cae
ci-ln-b2r0562-f76d1-b97km-worker-d-nmr4z   Ready    worker   41m   v1.18.3+fa69cae
$ oc get clusterversion
NAME      VERSION                             AVAILABLE   PROGRESSING   SINCE   STATUS
version   4.5.0-0.nightly-2020-12-05-205859   True        False         22m     Cluster version is 4.5.0-0.nightly-2020-12-05-205859
$ oc debug node/ci-ln-b2r0562-f76d1-b97km-master-0 
Starting pod/ci-ln-b2r0562-f76d1-b97km-master-0-debug ...
To use host binaries, run `chroot /host`
If you don't see a command prompt, try pressing enter.
sh-4.2# chroot /host
sh-4.4# systemctl cat kubelet.service
# /etc/systemd/system/kubelet.service
[Unit]
Description=Kubernetes Kubelet
Wants=rpc-statd.service network-online.target crio.service
After=network-online.target crio.service

[Service]
Type=notify
ExecStartPre=/bin/mkdir --parents /etc/kubernetes/manifests
ExecStartPre=/bin/rm -f /var/lib/kubelet/cpu_manager_state
Environment="KUBELET_LOG_LEVEL=3"
EnvironmentFile=/etc/os-release
EnvironmentFile=-/etc/kubernetes/kubelet-workaround
EnvironmentFile=-/etc/kubernetes/kubelet-env

ExecStart=/usr/bin/hyperkube \
    kubelet \
      --config=/etc/kubernetes/kubelet.conf \
      --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
      --kubeconfig=/var/lib/kubelet/kubeconfig \
      --container-runtime=remote \
      --container-runtime-endpoint=/var/run/crio/crio.sock \
      --runtime-cgroups=/system.slice/crio.service \
      --node-labels=node-role.kubernetes.io/master,node.openshift.io/os_id=${ID} \
      --minimum-container-ttl-duration=6m0s \
      --cloud-provider=gce \
      --volume-plugin-dir=/etc/kubernetes/kubelet-plugins/volume/exec \
      --cloud-config=/etc/kubernetes/cloud.conf \
      --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \
      --pod-infra-container-image=quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ca85a2e>
      --v=${KUBELET_LOG_LEVEL}

Restart=always
RestartSec=10

[Install]
WantedBy=multi-user.target
sh-4.4# exit
exit
sh-4.2# exit
exit

Removing debug pod ...
$ oc debug node/ci-ln-b2r0562-f76d1-b97km-worker-b-v4477 -- chroot /host systemctl cat kubelet.service
Starting pod/ci-ln-b2r0562-f76d1-b97km-worker-b-v4477-debug ...
To use host binaries, run `chroot /host`
# /etc/systemd/system/kubelet.service
[Unit]
Description=Kubernetes Kubelet
Wants=rpc-statd.service network-online.target crio.service
After=network-online.target crio.service

[Service]
Type=notify
ExecStartPre=/bin/mkdir --parents /etc/kubernetes/manifests
ExecStartPre=/bin/rm -f /var/lib/kubelet/cpu_manager_state
Environment="KUBELET_LOG_LEVEL=3"
EnvironmentFile=/etc/os-release
EnvironmentFile=-/etc/kubernetes/kubelet-workaround
EnvironmentFile=-/etc/kubernetes/kubelet-env

ExecStart=/usr/bin/hyperkube \
    kubelet \
      --config=/etc/kubernetes/kubelet.conf \
      --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
      --kubeconfig=/var/lib/kubelet/kubeconfig \
      --container-runtime=remote \
      --container-runtime-endpoint=/var/run/crio/crio.sock \
      --runtime-cgroups=/system.slice/crio.service \
      --node-labels=node-role.kubernetes.io/worker,node.openshift.io/os_id=${ID} \
      --minimum-container-ttl-duration=6m0s \
      --volume-plugin-dir=/etc/kubernetes/kubelet-plugins/volume/exec \
      --cloud-provider=gce \
      --cloud-config=/etc/kubernetes/cloud.conf \
      --pod-infra-container-image=quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ca85a2e41d6887dc0c608a98be1b08bb0d99a0a04cd0873e8f0a2804bfee6219 \
      --v=${KUBELET_LOG_LEVEL}

Restart=always
RestartSec=10

[Install]
WantedBy=multi-user.target

# /etc/systemd/system/kubelet.service.d/10-mco-default-env.conf
[Unit]
Description=MCO environment configuration

Removing debug pod ...
$

Comment 5 errata-xmlrpc 2020-12-15 20:28:46 UTC
Since the problem described in this bug report should be
resolved in a recent advisory, it has been closed with a
resolution of ERRATA.

For information on the advisory (Moderate: OpenShift Container Platform 4.5.23 security and bug fix update), and where to find the updated
files, follow the link below.

If the solution does not work for you, open a new bug report.

https://access.redhat.com/errata/RHSA-2020:5359