From 69a67556c77d9ef2fd9110a8c8e60b834c05a3c7 Mon Sep 17 00:00:00 2001 From: Paco Xu Date: Thu, 8 Aug 2024 21:02:13 +0800 Subject: [PATCH 1/2] kubelet: add warning log and events for cgroup v2 running on kernel < 5.8 --- pkg/kubelet/cm/container_manager.go | 3 +++ pkg/kubelet/events/event.go | 1 + pkg/kubelet/kubelet.go | 16 ++++++++++++++-- pkg/util/kernel/constants.go | 5 +++++ 4 files changed, 23 insertions(+), 2 deletions(-) diff --git a/pkg/kubelet/cm/container_manager.go b/pkg/kubelet/cm/container_manager.go index 99fb69c8b98..15215cc86c8 100644 --- a/pkg/kubelet/cm/container_manager.go +++ b/pkg/kubelet/cm/container_manager.go @@ -46,6 +46,9 @@ import ( const ( // Warning message for the users still using cgroup v1 CgroupV1MaintenanceModeWarning = "Cgroup v1 support is in maintenance mode, please migrate to Cgroup v2." + + // Warning message for the users using cgroup v2 on kernel older than 5.8 + CgroupV2KernelWarning = "Cgroup v2 is being used on a kernel older than 5.8, which may have limited functionality." ) type ActivePodsFunc func() []*v1.Pod diff --git a/pkg/kubelet/events/event.go b/pkg/kubelet/events/event.go index a9948ba6a54..f23c68c1726 100644 --- a/pkg/kubelet/events/event.go +++ b/pkg/kubelet/events/event.go @@ -77,6 +77,7 @@ const ( FailedPrepareDynamicResources = "FailedPrepareDynamicResources" PossibleMemoryBackedVolumesOnDisk = "PossibleMemoryBackedVolumesOnDisk" CgroupV1 = "CgroupV1" + CgroupV2 = "CgroupV2" ) // Image manager event reason list diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 68d790242db..a39ee656b86 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -52,6 +52,7 @@ import ( "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/version" "k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" clientset "k8s.io/client-go/kubernetes" @@ -120,6 +121,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/volumemanager" httpprobe "k8s.io/kubernetes/pkg/probe/http" "k8s.io/kubernetes/pkg/security/apparmor" + utilkernel "k8s.io/kubernetes/pkg/util/kernel" "k8s.io/kubernetes/pkg/util/oom" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/csi" @@ -1645,7 +1647,7 @@ func (kl *Kubelet) Run(updates <-chan kubetypes.PodUpdate) { os.Exit(1) } - kl.warnCgroupV1Usage() + kl.cgroupVersionCheck() // Start volume manager go kl.volumeManager.Run(kl.sourcesReady, wait.NeverStop) @@ -3067,11 +3069,21 @@ func (kl *Kubelet) UnprepareDynamicResources(pod *v1.Pod) error { return kl.containerManager.UnprepareDynamicResources(pod) } -func (kl *Kubelet) warnCgroupV1Usage() { +func (kl *Kubelet) cgroupVersionCheck() { cgroupVersion := kl.containerManager.GetNodeConfig().CgroupVersion if cgroupVersion == 1 { kl.recorder.Eventf(kl.nodeRef, v1.EventTypeWarning, events.CgroupV1, cm.CgroupV1MaintenanceModeWarning) klog.V(2).InfoS("Warning: cgroup v1", "message", cm.CgroupV1MaintenanceModeWarning) + } else { + kernelVersion, err := utilkernel.GetVersion() + if err != nil { + klog.V(2).ErrorS(err, "Warning: cannot determine kernel version, unable to determine if cgroup v2 is supported", "message", cm.CgroupV1MaintenanceModeWarning) + } + + if kernelVersion.LessThan(version.MustParseGeneric(utilkernel.CgroupV2KernelVersion)) { + kl.recorder.Eventf(kl.nodeRef, v1.EventTypeWarning, events.CgroupV2, cm.CgroupV2KernelWarning) + klog.V(2).InfoS("Warning: cgroup v2", "message", cm.CgroupV2KernelWarning) + } } metrics.CgroupVersion.Set(float64(cgroupVersion)) } diff --git a/pkg/util/kernel/constants.go b/pkg/util/kernel/constants.go index 6775027e7a0..2bf27ac0750 100644 --- a/pkg/util/kernel/constants.go +++ b/pkg/util/kernel/constants.go @@ -54,3 +54,8 @@ const TmpfsNoswapSupportKernelVersion = "6.4" // nftables mode with by default. This is not directly related to any specific kernel // commit; see https://issues.k8s.io/122743#issuecomment-1893922424 const NFTablesKubeProxyKernelVersion = "5.13" + +// CgroupV2KernelVersion is the minimal kernel version required for cgroup v2 support. +// `cpu.stat` file was added to root cgroup. +// (ref: https://github.com/torvalds/linux/commit/936f2a70f2077f64fab1dcb3eca71879e82ecd3f) +const CgroupV2KernelVersion = "5.8" From 259671bd43559697482df606769b53db38432e02 Mon Sep 17 00:00:00 2001 From: Paco Xu Date: Mon, 26 Aug 2024 15:12:52 +0800 Subject: [PATCH 2/2] check root cpu.stat instead of kernel version for cgroup v2 --- pkg/kubelet/cm/container_manager.go | 9 +++-- pkg/kubelet/events/event.go | 1 - pkg/kubelet/kubelet.go | 25 ++------------ pkg/kubelet/kubelet_linux.go | 51 +++++++++++++++++++++++++++++ pkg/kubelet/kubelet_others.go | 25 ++++++++++++++ pkg/util/kernel/constants.go | 5 --- 6 files changed, 85 insertions(+), 31 deletions(-) create mode 100644 pkg/kubelet/kubelet_linux.go create mode 100644 pkg/kubelet/kubelet_others.go diff --git a/pkg/kubelet/cm/container_manager.go b/pkg/kubelet/cm/container_manager.go index 15215cc86c8..203a9cb3675 100644 --- a/pkg/kubelet/cm/container_manager.go +++ b/pkg/kubelet/cm/container_manager.go @@ -45,10 +45,13 @@ import ( const ( // Warning message for the users still using cgroup v1 - CgroupV1MaintenanceModeWarning = "Cgroup v1 support is in maintenance mode, please migrate to Cgroup v2." + CgroupV1MaintenanceModeWarning = "cgroup v1 support is in maintenance mode, please migrate to cgroup v2" - // Warning message for the users using cgroup v2 on kernel older than 5.8 - CgroupV2KernelWarning = "Cgroup v2 is being used on a kernel older than 5.8, which may have limited functionality." + // Warning message for the users using cgroup v2 on kernel doesn't support root `cpu.stat`. + // `cpu.stat` was added to root cgroup in kernel 5.8. + // (ref: https://github.com/torvalds/linux/commit/936f2a70f2077f64fab1dcb3eca71879e82ecd3f) + CgroupV2KernelWarning = "cgroup v2 is being used on a kernel, which doesn't support root `cpu.stat`." + + "Kubelet will continue, but may experience instability or wrong behavior" ) type ActivePodsFunc func() []*v1.Pod diff --git a/pkg/kubelet/events/event.go b/pkg/kubelet/events/event.go index f23c68c1726..a9948ba6a54 100644 --- a/pkg/kubelet/events/event.go +++ b/pkg/kubelet/events/event.go @@ -77,7 +77,6 @@ const ( FailedPrepareDynamicResources = "FailedPrepareDynamicResources" PossibleMemoryBackedVolumesOnDisk = "PossibleMemoryBackedVolumesOnDisk" CgroupV1 = "CgroupV1" - CgroupV2 = "CgroupV2" ) // Image manager event reason list diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index a39ee656b86..78f804933b3 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -52,7 +52,6 @@ import ( "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/version" "k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" clientset "k8s.io/client-go/kubernetes" @@ -121,7 +120,6 @@ import ( "k8s.io/kubernetes/pkg/kubelet/volumemanager" httpprobe "k8s.io/kubernetes/pkg/probe/http" "k8s.io/kubernetes/pkg/security/apparmor" - utilkernel "k8s.io/kubernetes/pkg/util/kernel" "k8s.io/kubernetes/pkg/util/oom" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/csi" @@ -1647,7 +1645,9 @@ func (kl *Kubelet) Run(updates <-chan kubetypes.PodUpdate) { os.Exit(1) } - kl.cgroupVersionCheck() + if err := kl.cgroupVersionCheck(); err != nil { + klog.V(2).InfoS("Warning: cgroup check", "error", err) + } // Start volume manager go kl.volumeManager.Run(kl.sourcesReady, wait.NeverStop) @@ -3068,22 +3068,3 @@ func (kl *Kubelet) PrepareDynamicResources(pod *v1.Pod) error { func (kl *Kubelet) UnprepareDynamicResources(pod *v1.Pod) error { return kl.containerManager.UnprepareDynamicResources(pod) } - -func (kl *Kubelet) cgroupVersionCheck() { - cgroupVersion := kl.containerManager.GetNodeConfig().CgroupVersion - if cgroupVersion == 1 { - kl.recorder.Eventf(kl.nodeRef, v1.EventTypeWarning, events.CgroupV1, cm.CgroupV1MaintenanceModeWarning) - klog.V(2).InfoS("Warning: cgroup v1", "message", cm.CgroupV1MaintenanceModeWarning) - } else { - kernelVersion, err := utilkernel.GetVersion() - if err != nil { - klog.V(2).ErrorS(err, "Warning: cannot determine kernel version, unable to determine if cgroup v2 is supported", "message", cm.CgroupV1MaintenanceModeWarning) - } - - if kernelVersion.LessThan(version.MustParseGeneric(utilkernel.CgroupV2KernelVersion)) { - kl.recorder.Eventf(kl.nodeRef, v1.EventTypeWarning, events.CgroupV2, cm.CgroupV2KernelWarning) - klog.V(2).InfoS("Warning: cgroup v2", "message", cm.CgroupV2KernelWarning) - } - } - metrics.CgroupVersion.Set(float64(cgroupVersion)) -} diff --git a/pkg/kubelet/kubelet_linux.go b/pkg/kubelet/kubelet_linux.go new file mode 100644 index 00000000000..c1f82b1af69 --- /dev/null +++ b/pkg/kubelet/kubelet_linux.go @@ -0,0 +1,51 @@ +//go:build linux + +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubelet + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + v1 "k8s.io/api/core/v1" + "k8s.io/kubernetes/pkg/kubelet/cm" + "k8s.io/kubernetes/pkg/kubelet/cm/util" + "k8s.io/kubernetes/pkg/kubelet/events" + "k8s.io/kubernetes/pkg/kubelet/metrics" +) + +func (kl *Kubelet) cgroupVersionCheck() error { + cgroupVersion := kl.containerManager.GetNodeConfig().CgroupVersion + metrics.CgroupVersion.Set(float64(cgroupVersion)) + switch cgroupVersion { + case 1: + kl.recorder.Eventf(kl.nodeRef, v1.EventTypeWarning, events.CgroupV1, cm.CgroupV1MaintenanceModeWarning) + return errors.New(cm.CgroupV1MaintenanceModeWarning) + case 2: + cpustat := filepath.Join(util.CgroupRoot, "cpu.stat") + if _, err := os.Stat(cpustat); os.IsNotExist(err) { + // if `/sys/fs/cgroup/cpu.stat` does not exist, log a warning + return errors.New(cm.CgroupV2KernelWarning) + } + default: + return fmt.Errorf("unsupported cgroup version: %d", cgroupVersion) + } + return nil +} diff --git a/pkg/kubelet/kubelet_others.go b/pkg/kubelet/kubelet_others.go new file mode 100644 index 00000000000..0bd5a586981 --- /dev/null +++ b/pkg/kubelet/kubelet_others.go @@ -0,0 +1,25 @@ +//go:build !linux + +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubelet + +// cgroupVersionCheck performs a version check for the cgroup. +// This method is not applicable for non-Linux operating systems. +func (kl *Kubelet) cgroupVersionCheck() error { + return nil +} diff --git a/pkg/util/kernel/constants.go b/pkg/util/kernel/constants.go index 2bf27ac0750..6775027e7a0 100644 --- a/pkg/util/kernel/constants.go +++ b/pkg/util/kernel/constants.go @@ -54,8 +54,3 @@ const TmpfsNoswapSupportKernelVersion = "6.4" // nftables mode with by default. This is not directly related to any specific kernel // commit; see https://issues.k8s.io/122743#issuecomment-1893922424 const NFTablesKubeProxyKernelVersion = "5.13" - -// CgroupV2KernelVersion is the minimal kernel version required for cgroup v2 support. -// `cpu.stat` file was added to root cgroup. -// (ref: https://github.com/torvalds/linux/commit/936f2a70f2077f64fab1dcb3eca71879e82ecd3f) -const CgroupV2KernelVersion = "5.8"