Merge pull request #127670 from Jefftree/port-kcm-feature

Port all remaining features to versioned
2025-07-30 15:05:27 +00:00 · 2024-09-30 15:54:13 +01:00 · 2024-09-30 15:54:13 +01:00 · 2fb37e2489
commit 2fb37e2489
parent 8bc073a5fe d918804a5a
7 changed files with 147 additions and 120 deletions
--- a/hack/verify-test-featuregates.sh
+++ b/hack/verify-test-featuregates.sh
@ -43,7 +43,7 @@ fi


 # ensure all generic features are added in alphabetic order
-lines=$(git grep 'genericfeatures[.].*:' -- pkg/features/kube_features.go)
+lines=$(git grep 'genericfeatures[.].*:' -- pkg/features/versioned_kube_features.go)
 sorted_lines=$(echo "$lines" | sort -f)
 if [[ "$lines" != "$sorted_lines" ]]; then
  echo "Generic features in pkg/features/kube_features.go not sorted" >&2
--- a/pkg/features/kube_features.go
+++ b/pkg/features/kube_features.go
@ -18,7 +18,6 @@ package features

 import (
 	"k8s.io/apimachinery/pkg/util/runtime"
-	genericfeatures "k8s.io/apiserver/pkg/features"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	clientfeatures "k8s.io/client-go/features"
 	"k8s.io/component-base/featuregate"
@ -157,12 +156,6 @@ const (
 	// Enables kubelet to detect CSI volume condition and send the event of the abnormal volume to the corresponding pod that is using it.
 	CSIVolumeHealth featuregate.Feature = "CSIVolumeHealth"

-	// owner: @nckturner
-	// kep:  http://kep.k8s.io/2699
-	// alpha: v1.27
-	// Enable webhooks in cloud controller manager
-	CloudControllerManagerWebhook featuregate.Feature = "CloudControllerManagerWebhook"
-
 	// owner: @adrianreber
 	// kep: https://kep.k8s.io/2008
 	// alpha: v1.25
@ -233,7 +226,9 @@ const (

 	// owner: @HirazawaUi
 	// kep: http://kep.k8s.io/4004
-	// Deprecated: v1.29 (default off)
+	// alpha: v1.29
+	// Deprecated: v1.31 (default off)
+	//
 	// DisableNodeKubeProxyVersion disable the status.nodeInfo.kubeProxyVersion field of v1.Node
 	DisableNodeKubeProxyVersion featuregate.Feature = "DisableNodeKubeProxyVersion"

@ -615,6 +610,14 @@ const (
 	// Allow users to recover from volume expansion failure
 	RecoverVolumeExpansionFailure featuregate.Feature = "RecoverVolumeExpansionFailure"

+	// owner: @AkihiroSuda
+	// kep: https://kep.k8s.io/3857
+	// alpha: v1.30
+	// beta: v1.31
+	//
+	// Allows recursive read-only mounts.
+	RecursiveReadOnlyMounts featuregate.Feature = "RecursiveReadOnlyMounts"
+
 	// owner: @adrianmoisey
 	// kep: https://kep.k8s.io/4427
 	// alpha: v1.32
@ -759,6 +762,12 @@ const (
 	// Enables a StatefulSet to start from an arbitrary non zero ordinal
 	StatefulSetStartOrdinal featuregate.Feature = "StatefulSetStartOrdinal"

+	// owner: @ahutsunshine
+	// beta: v1.30
+	//
+	// Allows namespace indexer for namespace scope resources in apiserver cache to accelerate list operations.
+	StorageNamespaceIndex featuregate.Feature = "StorageNamespaceIndex"
+
 	// owner: @nilekhc
 	// kep: https://kep.k8s.io/4192
 	// alpha: v1.30
@ -909,12 +918,6 @@ const (
 	// releases.
 	UserNamespacesPodSecurityStandards featuregate.Feature = "UserNamespacesPodSecurityStandards"

-	// owner: @ahutsunshine
-	// beta: v1.30
-	//
-	// Allows namespace indexer for namespace scope resources in apiserver cache to accelerate list operations.
-	StorageNamespaceIndex featuregate.Feature = "StorageNamespaceIndex"
-
 	// owner: @jsafrane
 	// kep: https://kep.k8s.io/1710
 	// alpha: v1.30
@ -922,14 +925,6 @@ const (
 	// instead of changing each file on the volumes recursively.
 	SELinuxMount featuregate.Feature = "SELinuxMount"

-	// owner: @AkihiroSuda
-	// kep: https://kep.k8s.io/3857
-	// alpha: v1.30
-	// beta: v1.31
-	//
-	// Allows recursive read-only mounts.
-	RecursiveReadOnlyMounts featuregate.Feature = "RecursiveReadOnlyMounts"
-
 	// owner: @everpeace
 	// kep: https://kep.k8s.io/3619
 	// alpha: v1.31
@ -964,29 +959,4 @@ func init() {
 //
 // Entries are separated from each other with blank lines to avoid sweeping gofmt changes
 // when adding or removing one entry.
-var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{
-	// inherited features from generic apiserver, relisted here to get a conflict if it is changed
-	// unintentionally on either side:
-
-	genericfeatures.KMSv1: {Default: false, PreRelease: featuregate.Deprecated},
-
-	// features with duplicate definition in apiserver/controller-manager
-
-	CloudControllerManagerWebhook: {Default: false, PreRelease: featuregate.Alpha},
-
-	InPlacePodVerticalScaling: {Default: false, PreRelease: featuregate.Alpha},
-
-	// features that enable backwards compatibility but are scheduled to be removed
-	// ...
-	HPAScaleToZero: {Default: false, PreRelease: featuregate.Alpha},
-
-	AllowDNSOnlyNodeCSR: {Default: false, PreRelease: featuregate.Deprecated}, // remove after 1.33
-
-	AllowInsecureKubeletCertificateSigningRequests: {Default: false, PreRelease: featuregate.Deprecated}, // remove in 1.33
-
-	DisableNodeKubeProxyVersion: {Default: false, PreRelease: featuregate.Deprecated}, // default on in 1.33
-
-	StorageNamespaceIndex: {Default: true, PreRelease: featuregate.Beta},
-
-	RecursiveReadOnlyMounts: {Default: true, PreRelease: featuregate.Beta},
-}
+var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{}
--- a/pkg/features/versioned_kube_features.go
+++ b/pkg/features/versioned_kube_features.go
@ -21,6 +21,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/version"
 	genericfeatures "k8s.io/apiserver/pkg/features"
 	"k8s.io/component-base/featuregate"
+	kcmfeatures "k8s.io/controller-manager/pkg/features"
 )

 // defaultVersionedKubernetesFeatureGates consists of all known Kubernetes-specific feature keys with VersionedSpecs.
@ -34,6 +35,14 @@ import (
 //
 // Entries are alphabetized.
 var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate.VersionedSpecs{
+	AllowDNSOnlyNodeCSR: {
+		{Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Deprecated},
+	},
+
+	AllowInsecureKubeletCertificateSigningRequests: {
+		{Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Deprecated},
+	},
+
 	AllowOverwriteTerminationGracePeriodSeconds: {
 		{Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Deprecated},
 	},
@ -56,6 +65,10 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate
 		{Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Alpha},
 	},

+	kcmfeatures.CloudControllerManagerWebhook: {
+		{Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha},
+	},
+
 	ClusterTrustBundle: {
 		{Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha},
 	},
@ -146,6 +159,11 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate
 		{Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.GA, LockToDefault: true},
 	},

+	DisableNodeKubeProxyVersion: {
+		{Version: version.MustParse("1.29"), Default: false, PreRelease: featuregate.Alpha},
+		{Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Deprecated},
+	},
+
 	DRAControlPlaneController: {
 		{Version: version.MustParse("1.26"), Default: false, PreRelease: featuregate.Alpha},
 	},
@ -232,6 +250,11 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate
 		{Version: version.MustParse("1.24"), Default: true, PreRelease: featuregate.GA, LockToDefault: true},
 	},

+	genericfeatures.KMSv1: {
+		{Version: version.MustParse("1.28"), Default: true, PreRelease: featuregate.Deprecated},
+		{Version: version.MustParse("1.29"), Default: false, PreRelease: featuregate.Deprecated},
+	},
+
 	genericfeatures.MutatingAdmissionPolicy: {
 		{Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha},
 	},
@ -251,6 +274,12 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate
 		{Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.Beta},
 	},

+	genericfeatures.RetryGenerateName: {
+		{Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha},
+		{Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.Beta},
+		{Version: version.MustParse("1.32"), Default: true, LockToDefault: true, PreRelease: featuregate.GA},
+	},
+
 	genericfeatures.SeparateCacheWatchRPC: {
 		{Version: version.MustParse("1.28"), Default: true, PreRelease: featuregate.Beta},
 	},
@ -331,6 +360,10 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate
 		{Version: version.MustParse("1.30"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.32
 	},

+	HPAScaleToZero: {
+		{Version: version.MustParse("1.16"), Default: false, PreRelease: featuregate.Alpha},
+	},
+
 	ImageMaximumGCAge: {
 		{Version: version.MustParse("1.29"), Default: false, PreRelease: featuregate.Alpha},
 		{Version: version.MustParse("1.30"), Default: true, PreRelease: featuregate.Beta},
@ -340,6 +373,10 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate
 		{Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Alpha},
 	},

+	InPlacePodVerticalScaling: {
+		{Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha},
+	},
+
 	InTreePluginPortworxUnregister: {
 		{Version: version.MustParse("1.23"), Default: false, PreRelease: featuregate.Alpha},
 	},
@ -541,6 +578,10 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate
 	RecoverVolumeExpansionFailure: {
 		{Version: version.MustParse("1.23"), Default: false, PreRelease: featuregate.Alpha},
 	},
+	RecursiveReadOnlyMounts: {
+		{Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha},
+		{Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.Beta},
+	},

 	RelaxedDNSSearchValidation: {
 		{Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Alpha},
@ -585,6 +626,10 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate
 		{Version: version.MustParse("1.29"), Default: true, PreRelease: featuregate.Beta},
 	},

+	StorageNamespaceIndex: {
+		{Version: version.MustParse("1.30"), Default: true, PreRelease: featuregate.Beta},
+	},
+
 	ServiceAccountTokenJTI: {
 		{Version: version.MustParse("1.29"), Default: false, PreRelease: featuregate.Alpha},
 		{Version: version.MustParse("1.30"), Default: true, PreRelease: featuregate.Beta},
@ -698,9 +743,4 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate
 	WindowsHostNetwork: {
 		{Version: version.MustParse("1.26"), Default: true, PreRelease: featuregate.Alpha},
 	},
-	genericfeatures.RetryGenerateName: {
-		{Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha},
-		{Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.Beta},
-		{Version: version.MustParse("1.32"), Default: true, LockToDefault: true, PreRelease: featuregate.GA},
-	},
 }
--- a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go
+++ b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go
@ -363,6 +363,11 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate
 		{Version: version.MustParse("1.24"), Default: true, PreRelease: featuregate.GA, LockToDefault: true},
 	},

+	KMSv1: {
+		{Version: version.MustParse("1.28"), Default: true, PreRelease: featuregate.Deprecated},
+		{Version: version.MustParse("1.29"), Default: false, PreRelease: featuregate.Deprecated},
+	},
+
 	MutatingAdmissionPolicy: {
 		{Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha},
 	},
@ -448,9 +453,6 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate
 	},
 }

-// defaultKubernetesFeatureGates consists of all known Kubernetes-specific feature keys.
-// To add a new feature, define a key for it above and add it here. The features will be
-// available throughout Kubernetes binaries.
-var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{
-	KMSv1: {Default: false, PreRelease: featuregate.Deprecated},
-}
+// defaultKubernetesFeatureGates consists of legacy unversioned Kubernetes-specific feature keys.
+// Please do not add to this struct and use defaultVersionedKubernetesFeatureGates instead.
+var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{}
--- a/staging/src/k8s.io/controller-manager/pkg/features/kube_features.go
+++ b/staging/src/k8s.io/controller-manager/pkg/features/kube_features.go
@ -17,6 +17,7 @@ limitations under the License.
 package features

 import (
+	"k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/component-base/featuregate"
 )

@ -39,12 +40,14 @@ const (
 	CloudControllerManagerWebhook featuregate.Feature = "CloudControllerManagerWebhook"
 )

-func SetupCurrentKubernetesSpecificFeatureGates(featuregates featuregate.MutableFeatureGate) error {
-	return featuregates.Add(cloudPublicFeatureGates)
+func SetupCurrentKubernetesSpecificFeatureGates(featuregates featuregate.MutableVersionedFeatureGate) error {
+	return featuregates.AddVersioned(versionedCloudPublicFeatureGates)
 }

-// cloudPublicFeatureGates consists of cloud-specific feature keys.
-// To add a new feature, define a key for it at k8s.io/api/pkg/features and add it here.
-var cloudPublicFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{
-	CloudControllerManagerWebhook: {Default: false, PreRelease: featuregate.Alpha},
+// versionedCloudPublicFeatureGates consists of versioned cloud-specific feature keys.
+// To add a new feature, define a key for it above and add it here.
+var versionedCloudPublicFeatureGates = map[featuregate.Feature]featuregate.VersionedSpecs{
+	CloudControllerManagerWebhook: {
+		{Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha},
+	},
 }
--- a/test/featuregates_linter/test_data/unversioned_feature_list.yaml
+++ b/test/featuregates_linter/test_data/unversioned_feature_list.yaml
@ -1,21 +1,3 @@
- name: AllowDNSOnlyNodeCSR
-  versionedSpecs:
-  - default: false
-    lockToDefault: false
-    preRelease: Deprecated
-    version: ""
- name: AllowInsecureKubeletCertificateSigningRequests
-  versionedSpecs:
-  - default: false
-    lockToDefault: false
-    preRelease: Deprecated
-    version: ""
- name: CloudControllerManagerWebhook
-  versionedSpecs:
-  - default: false
-    lockToDefault: false
-    preRelease: Alpha
-    version: ""
 - name: ComponentSLIs
  versionedSpecs:
  - default: true
@ -28,30 +10,6 @@
    lockToDefault: false
    preRelease: Beta
    version: ""
- name: DisableNodeKubeProxyVersion
-  versionedSpecs:
-  - default: false
-    lockToDefault: false
-    preRelease: Deprecated
-    version: ""
- name: HPAScaleToZero
-  versionedSpecs:
-  - default: false
-    lockToDefault: false
-    preRelease: Alpha
-    version: ""
- name: InPlacePodVerticalScaling
-  versionedSpecs:
-  - default: false
-    lockToDefault: false
-    preRelease: Alpha
-    version: ""
- name: KMSv1
-  versionedSpecs:
-  - default: false
-    lockToDefault: false
-    preRelease: Deprecated
-    version: ""
 - name: LoggingAlphaOptions
  versionedSpecs:
  - default: false
@ -64,15 +22,3 @@
    lockToDefault: false
    preRelease: Beta
    version: ""
- name: RecursiveReadOnlyMounts
-  versionedSpecs:
-  - default: true
-    lockToDefault: false
-    preRelease: Beta
-    version: ""
- name: StorageNamespaceIndex
-  versionedSpecs:
-  - default: true
-    lockToDefault: false
-    preRelease: Beta
-    version: ""
--- a/test/featuregates_linter/test_data/versioned_feature_list.yaml
+++ b/test/featuregates_linter/test_data/versioned_feature_list.yaml
@ -26,6 +26,18 @@
    lockToDefault: true
    preRelease: GA
    version: "1.30"
+- name: AllowDNSOnlyNodeCSR
+  versionedSpecs:
+  - default: false
+    lockToDefault: false
+    preRelease: Deprecated
+    version: "1.31"
+- name: AllowInsecureKubeletCertificateSigningRequests
+  versionedSpecs:
+  - default: false
+    lockToDefault: false
+    preRelease: Deprecated
+    version: "1.31"
 - name: AllowOverwriteTerminationGracePeriodSeconds
  versionedSpecs:
  - default: false
@ -134,6 +146,12 @@
    lockToDefault: false
    preRelease: Alpha
    version: "1.31"
+- name: CloudControllerManagerWebhook
+  versionedSpecs:
+  - default: false
+    lockToDefault: false
+    preRelease: Alpha
+    version: "1.27"
 - name: ClusterTrustBundle
  versionedSpecs:
  - default: false
@ -320,6 +338,16 @@
    lockToDefault: true
    preRelease: GA
    version: "1.31"
+- name: DisableNodeKubeProxyVersion
+  versionedSpecs:
+  - default: false
+    lockToDefault: false
+    preRelease: Alpha
+    version: "1.29"
+  - default: false
+    lockToDefault: false
+    preRelease: Deprecated
+    version: "1.31"
 - name: DRAControlPlaneController
  versionedSpecs:
  - default: false
@ -412,6 +440,12 @@
    lockToDefault: true
    preRelease: GA
    version: "1.30"
+- name: HPAScaleToZero
+  versionedSpecs:
+  - default: false
+    lockToDefault: false
+    preRelease: Alpha
+    version: "1.16"
 - name: ImageMaximumGCAge
  versionedSpecs:
  - default: false
@ -428,6 +462,12 @@
    lockToDefault: false
    preRelease: Alpha
    version: "1.31"
+- name: InPlacePodVerticalScaling
+  versionedSpecs:
+  - default: false
+    lockToDefault: false
+    preRelease: Alpha
+    version: "1.27"
 - name: InTreePluginPortworxUnregister
  versionedSpecs:
  - default: false
@ -484,6 +524,16 @@
    lockToDefault: false
    preRelease: Beta
    version: "1.31"
+- name: KMSv1
+  versionedSpecs:
+  - default: true
+    lockToDefault: false
+    preRelease: Deprecated
+    version: "1.28"
+  - default: false
+    lockToDefault: false
+    preRelease: Deprecated
+    version: "1.29"
 - name: KubeletCgroupDriverFromCRI
  versionedSpecs:
  - default: false
@ -842,6 +892,16 @@
    lockToDefault: false
    preRelease: Alpha
    version: "1.23"
+- name: RecursiveReadOnlyMounts
+  versionedSpecs:
+  - default: false
+    lockToDefault: false
+    preRelease: Alpha
+    version: "1.30"
+  - default: true
+    lockToDefault: false
+    preRelease: Beta
+    version: "1.31"
 - name: RelaxedDNSSearchValidation
  versionedSpecs:
  - default: false
@ -1048,6 +1108,12 @@
    lockToDefault: true
    preRelease: GA
    version: "1.31"
+- name: StorageNamespaceIndex
+  versionedSpecs:
+  - default: true
+    lockToDefault: false
+    preRelease: Beta
+    version: "1.30"
 - name: StorageVersionAPI
  versionedSpecs:
  - default: false