diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index 32a0b5b8b5e..7102e6bade8 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -347,6 +347,7 @@ func AddKubeletConfigFlags(mainfs *pflag.FlagSet, c *kubeletconfig.KubeletConfig "vmodule": true, "log-flush-frequency": true, "provider-id": true, + "fail-cgroupv1": true, } fs.VisitAll(func(f *pflag.Flag) { if notDeprecated[f.Name] { @@ -369,6 +370,7 @@ func AddKubeletConfigFlags(mainfs *pflag.FlagSet, c *kubeletconfig.KubeletConfig fs.Var(&utilflag.IPVar{Val: &c.Address}, "address", "The IP address for the Kubelet to serve on (set to '0.0.0.0' or '::' for listening on all interfaces and IP address families)") fs.Int32Var(&c.Port, "port", c.Port, "The port for the Kubelet to serve on.") fs.Int32Var(&c.ReadOnlyPort, "read-only-port", c.ReadOnlyPort, "The read-only port for the Kubelet to serve on with no authentication/authorization (set to 0 to disable)") + fs.BoolVar(&c.FailCgroupV1, "fail-cgroupv1", c.FailCgroupV1, "Prevent the kubelet from starting on the host using cgroup v1.") // runtime flags fs.StringVar(&c.ContainerRuntimeEndpoint, "container-runtime-endpoint", c.ContainerRuntimeEndpoint, "The endpoint of container runtime service. Unix Domain Sockets are supported on Linux, while npipe and tcp endpoints are supported on Windows. Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime'") diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index 5333e2c4057..9526e2c28dc 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -62402,6 +62402,13 @@ func schema_k8sio_kubelet_config_v1beta1_KubeletConfiguration(ref common.Referen Format: "", }, }, + "failCgroupV1": { + SchemaProps: spec.SchemaProps{ + Description: "FailCgroupV1 prevents the kubelet from starting on hosts that use cgroup v1. By default, this is set to 'false', meaning the kubelet is allowed to start on cgroup v1 hosts unless this option is explicitly enabled. Default: false", + Type: []string{"boolean"}, + Format: "", + }, + }, }, Required: []string{"containerRuntimeEndpoint"}, }, diff --git a/pkg/kubelet/apis/config/helpers_test.go b/pkg/kubelet/apis/config/helpers_test.go index a58852a40a3..7e7ca3fdd21 100644 --- a/pkg/kubelet/apis/config/helpers_test.go +++ b/pkg/kubelet/apis/config/helpers_test.go @@ -300,5 +300,6 @@ var ( "Tracing.Endpoint", "Tracing.SamplingRatePerMillion", "LocalStorageCapacityIsolation", + "FailCgroupV1", ) ) diff --git a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml index 15c2061b2ca..bd3637e9091 100644 --- a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml +++ b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml @@ -37,6 +37,7 @@ enforceNodeAllocatable: eventBurst: 100 eventRecordQPS: 50 evictionPressureTransitionPeriod: 5m0s +failCgroupV1: false failSwapOn: true fileCheckFrequency: 20s hairpinMode: promiscuous-bridge diff --git a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml index e62977d438c..f733b6ef09b 100644 --- a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml +++ b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml @@ -37,6 +37,7 @@ enforceNodeAllocatable: eventBurst: 10 eventRecordQPS: 5 evictionPressureTransitionPeriod: 5m0s +failCgroupV1: false failSwapOn: true fileCheckFrequency: 20s hairpinMode: promiscuous-bridge diff --git a/pkg/kubelet/apis/config/types.go b/pkg/kubelet/apis/config/types.go index dd61ac23fd0..60d502b55cc 100644 --- a/pkg/kubelet/apis/config/types.go +++ b/pkg/kubelet/apis/config/types.go @@ -494,6 +494,13 @@ type KubeletConfiguration struct { // If not specified the default value is ContainerRuntimeEndpoint // +optional ImageServiceEndpoint string + + // FailCgroupV1 prevents the kubelet from starting on hosts + // that use cgroup v1. By default, this is set to 'false', meaning + // the kubelet is allowed to start on cgroup v1 hosts unless this + // option is explicitly enabled. + // +optional + FailCgroupV1 bool } // KubeletAuthorizationMode denotes the authorization mode for the kubelet diff --git a/pkg/kubelet/apis/config/v1beta1/defaults.go b/pkg/kubelet/apis/config/v1beta1/defaults.go index 1fcf1c9a340..a4f25677260 100644 --- a/pkg/kubelet/apis/config/v1beta1/defaults.go +++ b/pkg/kubelet/apis/config/v1beta1/defaults.go @@ -268,6 +268,9 @@ func SetDefaults_KubeletConfiguration(obj *kubeletconfigv1beta1.KubeletConfigura if obj.SeccompDefault == nil { obj.SeccompDefault = utilpointer.Bool(false) } + if obj.FailCgroupV1 == nil { + obj.FailCgroupV1 = utilpointer.Bool(false) + } if obj.MemoryThrottlingFactor == nil { obj.MemoryThrottlingFactor = utilpointer.Float64(DefaultMemoryThrottlingFactor) } diff --git a/pkg/kubelet/apis/config/v1beta1/defaults_test.go b/pkg/kubelet/apis/config/v1beta1/defaults_test.go index b24d91c79e3..4cdd71da287 100644 --- a/pkg/kubelet/apis/config/v1beta1/defaults_test.go +++ b/pkg/kubelet/apis/config/v1beta1/defaults_test.go @@ -125,6 +125,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { EnableProfilingHandler: utilpointer.Bool(true), EnableDebugFlagsHandler: utilpointer.Bool(true), SeccompDefault: utilpointer.Bool(false), + FailCgroupV1: utilpointer.Bool(false), MemoryThrottlingFactor: utilpointer.Float64(DefaultMemoryThrottlingFactor), RegisterNode: utilpointer.Bool(true), LocalStorageCapacityIsolation: utilpointer.Bool(true), @@ -255,6 +256,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { EnableProfilingHandler: utilpointer.Bool(false), EnableDebugFlagsHandler: utilpointer.Bool(false), SeccompDefault: utilpointer.Bool(false), + FailCgroupV1: utilpointer.Bool(false), MemoryThrottlingFactor: utilpointer.Float64(0), RegisterNode: utilpointer.Bool(false), LocalStorageCapacityIsolation: utilpointer.Bool(false), @@ -356,6 +358,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { EnableProfilingHandler: utilpointer.Bool(false), EnableDebugFlagsHandler: utilpointer.Bool(false), SeccompDefault: utilpointer.Bool(false), + FailCgroupV1: utilpointer.Bool(false), MemoryThrottlingFactor: utilpointer.Float64(0), RegisterNode: utilpointer.Bool(false), LocalStorageCapacityIsolation: utilpointer.Bool(false), @@ -508,6 +511,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { EnableProfilingHandler: utilpointer.Bool(true), EnableDebugFlagsHandler: utilpointer.Bool(true), SeccompDefault: utilpointer.Bool(true), + FailCgroupV1: utilpointer.Bool(true), MemoryThrottlingFactor: utilpointer.Float64(1), RegisterNode: utilpointer.Bool(true), LocalStorageCapacityIsolation: utilpointer.Bool(true), @@ -657,6 +661,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { EnableProfilingHandler: utilpointer.Bool(true), EnableDebugFlagsHandler: utilpointer.Bool(true), SeccompDefault: utilpointer.Bool(true), + FailCgroupV1: utilpointer.Bool(true), MemoryThrottlingFactor: utilpointer.Float64(1), RegisterNode: utilpointer.Bool(true), LocalStorageCapacityIsolation: utilpointer.Bool(true), @@ -749,6 +754,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { EnableProfilingHandler: utilpointer.Bool(true), EnableDebugFlagsHandler: utilpointer.Bool(true), SeccompDefault: utilpointer.Bool(false), + FailCgroupV1: utilpointer.Bool(false), MemoryThrottlingFactor: utilpointer.Float64Ptr(DefaultMemoryThrottlingFactor), RegisterNode: utilpointer.Bool(true), LocalStorageCapacityIsolation: utilpointer.Bool(true), @@ -841,6 +847,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { EnableProfilingHandler: utilpointer.Bool(true), EnableDebugFlagsHandler: utilpointer.Bool(true), SeccompDefault: utilpointer.Bool(false), + FailCgroupV1: utilpointer.Bool(false), MemoryThrottlingFactor: utilpointer.Float64Ptr(DefaultMemoryThrottlingFactor), RegisterNode: utilpointer.Bool(true), LocalStorageCapacityIsolation: utilpointer.Bool(true), @@ -933,6 +940,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { EnableProfilingHandler: utilpointer.Bool(true), EnableDebugFlagsHandler: utilpointer.Bool(true), SeccompDefault: utilpointer.Bool(false), + FailCgroupV1: utilpointer.Bool(false), MemoryThrottlingFactor: utilpointer.Float64(DefaultMemoryThrottlingFactor), RegisterNode: utilpointer.Bool(true), LocalStorageCapacityIsolation: utilpointer.Bool(true), diff --git a/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go b/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go index c0ae585e778..64500bb7040 100644 --- a/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go +++ b/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go @@ -526,6 +526,9 @@ func autoConvert_v1beta1_KubeletConfiguration_To_config_KubeletConfiguration(in } out.ContainerRuntimeEndpoint = in.ContainerRuntimeEndpoint out.ImageServiceEndpoint = in.ImageServiceEndpoint + if err := v1.Convert_Pointer_bool_To_bool(&in.FailCgroupV1, &out.FailCgroupV1, s); err != nil { + return err + } return nil } @@ -719,6 +722,9 @@ func autoConvert_config_KubeletConfiguration_To_v1beta1_KubeletConfiguration(in } out.ContainerRuntimeEndpoint = in.ContainerRuntimeEndpoint out.ImageServiceEndpoint = in.ImageServiceEndpoint + if err := v1.Convert_bool_To_Pointer_bool(&in.FailCgroupV1, &out.FailCgroupV1, s); err != nil { + return err + } return nil } diff --git a/pkg/kubelet/apis/config/validation/validation_linux.go b/pkg/kubelet/apis/config/validation/validation_linux.go new file mode 100644 index 00000000000..97407b7fbe6 --- /dev/null +++ b/pkg/kubelet/apis/config/validation/validation_linux.go @@ -0,0 +1,36 @@ +//go:build linux +// +build linux + +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validation + +import ( + "fmt" + + libcontainercgroups "github.com/opencontainers/runc/libcontainer/cgroups" + kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" +) + +// validateKubeletOSConfiguration validates os specific kubelet configuration and returns an error if it is invalid. +func validateKubeletOSConfiguration(kc *kubeletconfig.KubeletConfiguration) error { + if kc.FailCgroupV1 && !libcontainercgroups.IsCgroup2UnifiedMode() { + return fmt.Errorf("kubelet is configured to not run on a host using cgroup v1. cgroup v1 support is in maintenance mode") + } + + return nil +} diff --git a/pkg/kubelet/apis/config/validation/validation_others.go b/pkg/kubelet/apis/config/validation/validation_others.go index 29c708b4610..c50143116c2 100644 --- a/pkg/kubelet/apis/config/validation/validation_others.go +++ b/pkg/kubelet/apis/config/validation/validation_others.go @@ -1,5 +1,5 @@ -//go:build !windows -// +build !windows +//go:build !windows && !linux +// +build !windows,!linux /* Copyright 2018 The Kubernetes Authors. diff --git a/staging/src/k8s.io/kubelet/config/v1beta1/types.go b/staging/src/k8s.io/kubelet/config/v1beta1/types.go index b17d0a7a157..1f7bbc93931 100644 --- a/staging/src/k8s.io/kubelet/config/v1beta1/types.go +++ b/staging/src/k8s.io/kubelet/config/v1beta1/types.go @@ -851,6 +851,14 @@ type KubeletConfiguration struct { // If not specified, the value in containerRuntimeEndpoint is used. // +optional ImageServiceEndpoint string `json:"imageServiceEndpoint,omitempty"` + + // FailCgroupV1 prevents the kubelet from starting on hosts + // that use cgroup v1. By default, this is set to 'false', meaning + // the kubelet is allowed to start on cgroup v1 hosts unless this + // option is explicitly enabled. + // Default: false + // +optional + FailCgroupV1 *bool `json:"failCgroupV1,omitempty"` } type KubeletAuthorizationMode string diff --git a/staging/src/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go b/staging/src/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go index 0e20d63ed4d..613a039a755 100644 --- a/staging/src/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go @@ -485,6 +485,11 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) { *out = new(bool) **out = **in } + if in.FailCgroupV1 != nil { + in, out := &in.FailCgroupV1, &out.FailCgroupV1 + *out = new(bool) + **out = **in + } return }