From 3c7ed00e2248da6a7825be93638d0ef28019ab0d Mon Sep 17 00:00:00 2001 From: Francesco Romani Date: Mon, 3 Mar 2025 11:00:30 +0100 Subject: [PATCH] node: kep-2625: cpu manager policy options GA Move the support for CPUManager Policy Options to GA Signed-off-by: Francesco Romani --- cmd/kubelet/app/server.go | 10 +------ hack/local-up-cluster.sh | 3 +-- pkg/features/kube_features.go | 3 ++- pkg/generated/openapi/zz_generated.openapi.go | 2 +- pkg/kubelet/apis/config/types.go | 1 - pkg/kubelet/cm/cpumanager/policy_options.go | 6 +++-- .../cm/cpumanager/policy_options_test.go | 27 ++++++++++--------- .../k8s.io/kubelet/config/v1beta1/types.go | 1 - .../reference/versioned_feature_list.yaml | 4 +++ test/e2e_node/cpu_manager_test.go | 1 - 10 files changed, 28 insertions(+), 30 deletions(-) diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 8121eca8374..a7ed8e0014a 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -826,14 +826,6 @@ func run(ctx context.Context, s *options.KubeletServer, kubeDeps *kubelet.Depend return fmt.Errorf("--qos-reserved value failed to parse: %w", err) } - var cpuManagerPolicyOptions map[string]string - if utilfeature.DefaultFeatureGate.Enabled(features.CPUManagerPolicyOptions) { - cpuManagerPolicyOptions = s.CPUManagerPolicyOptions - } else if s.CPUManagerPolicyOptions != nil { - return fmt.Errorf("CPU Manager policy options %v require feature gates %q, %q enabled", - s.CPUManagerPolicyOptions, features.CPUManager, features.CPUManagerPolicyOptions) - } - var topologyManagerPolicyOptions map[string]string if utilfeature.DefaultFeatureGate.Enabled(features.TopologyManagerPolicyOptions) { topologyManagerPolicyOptions = s.TopologyManagerPolicyOptions @@ -877,7 +869,7 @@ func run(ctx context.Context, s *options.KubeletServer, kubeDeps *kubelet.Depend }, QOSReserved: *experimentalQOSReserved, CPUManagerPolicy: s.CPUManagerPolicy, - CPUManagerPolicyOptions: cpuManagerPolicyOptions, + CPUManagerPolicyOptions: s.CPUManagerPolicyOptions, CPUManagerReconcilePeriod: s.CPUManagerReconcilePeriod.Duration, MemoryManagerPolicy: s.MemoryManagerPolicy, MemoryManagerReservedMemory: s.ReservedMemory, diff --git a/hack/local-up-cluster.sh b/hack/local-up-cluster.sh index 0bae9849ba7..05659602e7f 100755 --- a/hack/local-up-cluster.sh +++ b/hack/local-up-cluster.sh @@ -167,8 +167,7 @@ function usage { echo "Example 1: hack/local-up-cluster.sh -o _output/dockerized/bin/linux/amd64/ (run from docker output)" echo "Example 2: hack/local-up-cluster.sh -O (auto-guess the bin path for your platform)" echo "Example 3: hack/local-up-cluster.sh (build a local copy of the source)" - echo "Example 4: FEATURE_GATES=CPUManagerPolicyOptions=true \\" - echo " TOPOLOGY_MANAGER_POLICY=\"single-numa-node\" \\" + echo "Example 4: TOPOLOGY_MANAGER_POLICY=\"single-numa-node\" \\" echo " CPUMANAGER_POLICY=\"static\" \\" echo " CPUMANAGER_POLICY_OPTIONS=full-pcpus-only=\"true\" \\" echo " CPUMANAGER_RECONCILE_PERIOD=\"5s\" \\" diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 3c6e9614d98..7464f10f1aa 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -114,7 +114,7 @@ const ( // for details about the removal of this feature gate. CPUManagerPolicyBetaOptions featuregate.Feature = "CPUManagerPolicyBetaOptions" - // owner: @fromanirh + // owner: @ffromani // // Allow the usage of options to fine-tune the cpumanager policies. CPUManagerPolicyOptions featuregate.Feature = "CPUManagerPolicyOptions" @@ -1049,6 +1049,7 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate CPUManagerPolicyOptions: { {Version: version.MustParse("1.22"), Default: false, PreRelease: featuregate.Alpha}, {Version: version.MustParse("1.23"), Default: true, PreRelease: featuregate.Beta}, + {Version: version.MustParse("1.33"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.36 }, CronJobsScheduledAnnotation: { diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index e10bee85623..4bb4f665fcd 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -66530,7 +66530,7 @@ func schema_k8sio_kubelet_config_v1beta1_KubeletConfiguration(ref common.Referen }, "cpuManagerPolicyOptions": { SchemaProps: spec.SchemaProps{ - Description: "cpuManagerPolicyOptions is a set of key=value which \tallows to set extra options to fine tune the behaviour of the cpu manager policies. Requires both the \"CPUManager\" and \"CPUManagerPolicyOptions\" feature gates to be enabled. Default: nil", + Description: "cpuManagerPolicyOptions is a set of key=value which \tallows to set extra options to fine tune the behaviour of the cpu manager policies. Default: nil", Type: []string{"object"}, AdditionalProperties: &spec.SchemaOrBool{ Allows: true, diff --git a/pkg/kubelet/apis/config/types.go b/pkg/kubelet/apis/config/types.go index 1cc258880a1..4a04856ce16 100644 --- a/pkg/kubelet/apis/config/types.go +++ b/pkg/kubelet/apis/config/types.go @@ -257,7 +257,6 @@ type KubeletConfiguration struct { CPUManagerPolicy string // CPUManagerPolicyOptions is a set of key=value which allows to set extra options // to fine tune the behaviour of the cpu manager policies. - // Requires both the "CPUManager" and "CPUManagerPolicyOptions" feature gates to be enabled. CPUManagerPolicyOptions map[string]string // CPU Manager reconciliation period. // Requires the CPUManager feature gate to be enabled. diff --git a/pkg/kubelet/cm/cpumanager/policy_options.go b/pkg/kubelet/cm/cpumanager/policy_options.go index fcf95d23eec..65b415bdad6 100644 --- a/pkg/kubelet/cm/cpumanager/policy_options.go +++ b/pkg/kubelet/cm/cpumanager/policy_options.go @@ -44,11 +44,12 @@ var ( PreferAlignByUnCoreCacheOption, ) betaOptions = sets.New[string]( - FullPCPUsOnlyOption, StrictCPUReservationOption, DistributeCPUsAcrossNUMAOption, ) - stableOptions = sets.New[string]() + stableOptions = sets.New[string]( + FullPCPUsOnlyOption, + ) ) // CheckPolicyOptionAvailable verifies if the given option can be used depending on the Feature Gate Settings. @@ -66,6 +67,7 @@ func CheckPolicyOptionAvailable(option string) error { return fmt.Errorf("CPU Manager Policy Beta-level Options not enabled, but option %q provided", option) } + // if the option is stable, we need no CPUManagerPolicy*Options feature gate check return nil } diff --git a/pkg/kubelet/cm/cpumanager/policy_options_test.go b/pkg/kubelet/cm/cpumanager/policy_options_test.go index e23c2453415..ef14618f4c3 100644 --- a/pkg/kubelet/cm/cpumanager/policy_options_test.go +++ b/pkg/kubelet/cm/cpumanager/policy_options_test.go @@ -70,18 +70,6 @@ func TestPolicyOptionsAvailable(t *testing.T) { featureGateEnable: true, expectedAvailable: false, }, - { - option: FullPCPUsOnlyOption, - featureGate: pkgfeatures.CPUManagerPolicyBetaOptions, - featureGateEnable: true, - expectedAvailable: true, - }, - { - option: FullPCPUsOnlyOption, - featureGate: pkgfeatures.CPUManagerPolicyBetaOptions, - featureGateEnable: false, - expectedAvailable: false, - }, { option: AlignBySocketOption, featureGate: pkgfeatures.CPUManagerPolicyAlphaOptions, @@ -143,6 +131,21 @@ func TestPolicyOptionsAvailable(t *testing.T) { } } +func TestPolicyOptionsAlwaysAvailableOnceGA(t *testing.T) { + options := []string{ + FullPCPUsOnlyOption, + } + for _, option := range options { + t.Run(option, func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.CPUManagerPolicyAlphaOptions, false) + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.CPUManagerPolicyBetaOptions, false) + if err := CheckPolicyOptionAvailable(option); err != nil { + t.Errorf("option %q should be available even with all featuregate disabled", option) + } + }) + } +} + func TestValidateStaticPolicyOptions(t *testing.T) { testCases := []struct { description string diff --git a/staging/src/k8s.io/kubelet/config/v1beta1/types.go b/staging/src/k8s.io/kubelet/config/v1beta1/types.go index 163640b811a..710dd50e770 100644 --- a/staging/src/k8s.io/kubelet/config/v1beta1/types.go +++ b/staging/src/k8s.io/kubelet/config/v1beta1/types.go @@ -406,7 +406,6 @@ type KubeletConfiguration struct { SingleProcessOOMKill *bool `json:"singleProcessOOMKill,omitempty"` // cpuManagerPolicyOptions is a set of key=value which allows to set extra options // to fine tune the behaviour of the cpu manager policies. - // Requires both the "CPUManager" and "CPUManagerPolicyOptions" feature gates to be enabled. // Default: nil // +optional CPUManagerPolicyOptions map[string]string `json:"cpuManagerPolicyOptions,omitempty"` diff --git a/test/compatibility_lifecycle/reference/versioned_feature_list.yaml b/test/compatibility_lifecycle/reference/versioned_feature_list.yaml index e6bf4c342fa..a7c9ce7e9e8 100644 --- a/test/compatibility_lifecycle/reference/versioned_feature_list.yaml +++ b/test/compatibility_lifecycle/reference/versioned_feature_list.yaml @@ -303,6 +303,10 @@ lockToDefault: false preRelease: Beta version: "1.23" + - default: true + lockToDefault: true + preRelease: GA + version: "1.33" - name: CRDValidationRatcheting versionedSpecs: - default: false diff --git a/test/e2e_node/cpu_manager_test.go b/test/e2e_node/cpu_manager_test.go index 13bb31dae58..50f29c36915 100644 --- a/test/e2e_node/cpu_manager_test.go +++ b/test/e2e_node/cpu_manager_test.go @@ -303,7 +303,6 @@ func configureCPUManagerInKubelet(oldCfg *kubeletconfig.KubeletConfiguration, ku newCfg.FeatureGates = make(map[string]bool) } - newCfg.FeatureGates["CPUManagerPolicyOptions"] = kubeletArguments.enableCPUManagerOptions newCfg.FeatureGates["CPUManagerPolicyBetaOptions"] = kubeletArguments.enableCPUManagerOptions newCfg.FeatureGates["CPUManagerPolicyAlphaOptions"] = kubeletArguments.enableCPUManagerOptions newCfg.FeatureGates["DisableCPUQuotaWithExclusiveCPUs"] = kubeletArguments.disableCPUQuotaWithExclusiveCPUs