diff --git a/hack/verify-test-featuregates.sh b/hack/verify-test-featuregates.sh index bd1f1b12375..b0f76c0250b 100755 --- a/hack/verify-test-featuregates.sh +++ b/hack/verify-test-featuregates.sh @@ -43,7 +43,7 @@ fi # ensure all generic features are added in alphabetic order -lines=$(git grep 'genericfeatures[.].*:' -- pkg/features/kube_features.go) +lines=$(git grep 'genericfeatures[.].*:' -- pkg/features/versioned_kube_features.go) sorted_lines=$(echo "$lines" | sort -f) if [[ "$lines" != "$sorted_lines" ]]; then echo "Generic features in pkg/features/kube_features.go not sorted" >&2 diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 743d2b69c2c..a024ec5c835 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -18,7 +18,6 @@ package features import ( "k8s.io/apimachinery/pkg/util/runtime" - genericfeatures "k8s.io/apiserver/pkg/features" utilfeature "k8s.io/apiserver/pkg/util/feature" clientfeatures "k8s.io/client-go/features" "k8s.io/component-base/featuregate" @@ -157,12 +156,6 @@ const ( // Enables kubelet to detect CSI volume condition and send the event of the abnormal volume to the corresponding pod that is using it. CSIVolumeHealth featuregate.Feature = "CSIVolumeHealth" - // owner: @nckturner - // kep: http://kep.k8s.io/2699 - // alpha: v1.27 - // Enable webhooks in cloud controller manager - CloudControllerManagerWebhook featuregate.Feature = "CloudControllerManagerWebhook" - // owner: @adrianreber // kep: https://kep.k8s.io/2008 // alpha: v1.25 @@ -233,7 +226,9 @@ const ( // owner: @HirazawaUi // kep: http://kep.k8s.io/4004 - // Deprecated: v1.29 (default off) + // alpha: v1.29 + // Deprecated: v1.31 (default off) + // // DisableNodeKubeProxyVersion disable the status.nodeInfo.kubeProxyVersion field of v1.Node DisableNodeKubeProxyVersion featuregate.Feature = "DisableNodeKubeProxyVersion" @@ -615,6 +610,14 @@ const ( // Allow users to recover from volume expansion failure RecoverVolumeExpansionFailure featuregate.Feature = "RecoverVolumeExpansionFailure" + // owner: @AkihiroSuda + // kep: https://kep.k8s.io/3857 + // alpha: v1.30 + // beta: v1.31 + // + // Allows recursive read-only mounts. + RecursiveReadOnlyMounts featuregate.Feature = "RecursiveReadOnlyMounts" + // owner: @adrianmoisey // kep: https://kep.k8s.io/4427 // alpha: v1.32 @@ -759,6 +762,12 @@ const ( // Enables a StatefulSet to start from an arbitrary non zero ordinal StatefulSetStartOrdinal featuregate.Feature = "StatefulSetStartOrdinal" + // owner: @ahutsunshine + // beta: v1.30 + // + // Allows namespace indexer for namespace scope resources in apiserver cache to accelerate list operations. + StorageNamespaceIndex featuregate.Feature = "StorageNamespaceIndex" + // owner: @nilekhc // kep: https://kep.k8s.io/4192 // alpha: v1.30 @@ -909,12 +918,6 @@ const ( // releases. UserNamespacesPodSecurityStandards featuregate.Feature = "UserNamespacesPodSecurityStandards" - // owner: @ahutsunshine - // beta: v1.30 - // - // Allows namespace indexer for namespace scope resources in apiserver cache to accelerate list operations. - StorageNamespaceIndex featuregate.Feature = "StorageNamespaceIndex" - // owner: @jsafrane // kep: https://kep.k8s.io/1710 // alpha: v1.30 @@ -922,14 +925,6 @@ const ( // instead of changing each file on the volumes recursively. SELinuxMount featuregate.Feature = "SELinuxMount" - // owner: @AkihiroSuda - // kep: https://kep.k8s.io/3857 - // alpha: v1.30 - // beta: v1.31 - // - // Allows recursive read-only mounts. - RecursiveReadOnlyMounts featuregate.Feature = "RecursiveReadOnlyMounts" - // owner: @everpeace // kep: https://kep.k8s.io/3619 // alpha: v1.31 @@ -964,29 +959,4 @@ func init() { // // Entries are separated from each other with blank lines to avoid sweeping gofmt changes // when adding or removing one entry. -var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ - // inherited features from generic apiserver, relisted here to get a conflict if it is changed - // unintentionally on either side: - - genericfeatures.KMSv1: {Default: false, PreRelease: featuregate.Deprecated}, - - // features with duplicate definition in apiserver/controller-manager - - CloudControllerManagerWebhook: {Default: false, PreRelease: featuregate.Alpha}, - - InPlacePodVerticalScaling: {Default: false, PreRelease: featuregate.Alpha}, - - // features that enable backwards compatibility but are scheduled to be removed - // ... - HPAScaleToZero: {Default: false, PreRelease: featuregate.Alpha}, - - AllowDNSOnlyNodeCSR: {Default: false, PreRelease: featuregate.Deprecated}, // remove after 1.33 - - AllowInsecureKubeletCertificateSigningRequests: {Default: false, PreRelease: featuregate.Deprecated}, // remove in 1.33 - - DisableNodeKubeProxyVersion: {Default: false, PreRelease: featuregate.Deprecated}, // default on in 1.33 - - StorageNamespaceIndex: {Default: true, PreRelease: featuregate.Beta}, - - RecursiveReadOnlyMounts: {Default: true, PreRelease: featuregate.Beta}, -} +var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{} diff --git a/pkg/features/versioned_kube_features.go b/pkg/features/versioned_kube_features.go index 925f61dc1e3..dfe88a07582 100644 --- a/pkg/features/versioned_kube_features.go +++ b/pkg/features/versioned_kube_features.go @@ -21,6 +21,7 @@ import ( "k8s.io/apimachinery/pkg/util/version" genericfeatures "k8s.io/apiserver/pkg/features" "k8s.io/component-base/featuregate" + kcmfeatures "k8s.io/controller-manager/pkg/features" ) // defaultVersionedKubernetesFeatureGates consists of all known Kubernetes-specific feature keys with VersionedSpecs. @@ -34,6 +35,14 @@ import ( // // Entries are alphabetized. var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate.VersionedSpecs{ + AllowDNSOnlyNodeCSR: { + {Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Deprecated}, + }, + + AllowInsecureKubeletCertificateSigningRequests: { + {Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Deprecated}, + }, + AllowOverwriteTerminationGracePeriodSeconds: { {Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Deprecated}, }, @@ -56,6 +65,10 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate {Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Alpha}, }, + kcmfeatures.CloudControllerManagerWebhook: { + {Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha}, + }, + ClusterTrustBundle: { {Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha}, }, @@ -146,6 +159,11 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate {Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, }, + DisableNodeKubeProxyVersion: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: featuregate.Alpha}, + {Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Deprecated}, + }, + DRAControlPlaneController: { {Version: version.MustParse("1.26"), Default: false, PreRelease: featuregate.Alpha}, }, @@ -232,6 +250,11 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate {Version: version.MustParse("1.24"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, }, + genericfeatures.KMSv1: { + {Version: version.MustParse("1.28"), Default: true, PreRelease: featuregate.Deprecated}, + {Version: version.MustParse("1.29"), Default: false, PreRelease: featuregate.Deprecated}, + }, + genericfeatures.MutatingAdmissionPolicy: { {Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha}, }, @@ -251,6 +274,12 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate {Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.Beta}, }, + genericfeatures.RetryGenerateName: { + {Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha}, + {Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.Beta}, + {Version: version.MustParse("1.32"), Default: true, LockToDefault: true, PreRelease: featuregate.GA}, + }, + genericfeatures.SeparateCacheWatchRPC: { {Version: version.MustParse("1.28"), Default: true, PreRelease: featuregate.Beta}, }, @@ -331,6 +360,10 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate {Version: version.MustParse("1.30"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.32 }, + HPAScaleToZero: { + {Version: version.MustParse("1.16"), Default: false, PreRelease: featuregate.Alpha}, + }, + ImageMaximumGCAge: { {Version: version.MustParse("1.29"), Default: false, PreRelease: featuregate.Alpha}, {Version: version.MustParse("1.30"), Default: true, PreRelease: featuregate.Beta}, @@ -340,6 +373,10 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate {Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Alpha}, }, + InPlacePodVerticalScaling: { + {Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha}, + }, + InTreePluginPortworxUnregister: { {Version: version.MustParse("1.23"), Default: false, PreRelease: featuregate.Alpha}, }, @@ -541,6 +578,10 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate RecoverVolumeExpansionFailure: { {Version: version.MustParse("1.23"), Default: false, PreRelease: featuregate.Alpha}, }, + RecursiveReadOnlyMounts: { + {Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha}, + {Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.Beta}, + }, RelaxedDNSSearchValidation: { {Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Alpha}, @@ -585,6 +626,10 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate {Version: version.MustParse("1.29"), Default: true, PreRelease: featuregate.Beta}, }, + StorageNamespaceIndex: { + {Version: version.MustParse("1.30"), Default: true, PreRelease: featuregate.Beta}, + }, + ServiceAccountTokenJTI: { {Version: version.MustParse("1.29"), Default: false, PreRelease: featuregate.Alpha}, {Version: version.MustParse("1.30"), Default: true, PreRelease: featuregate.Beta}, @@ -698,9 +743,4 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate WindowsHostNetwork: { {Version: version.MustParse("1.26"), Default: true, PreRelease: featuregate.Alpha}, }, - genericfeatures.RetryGenerateName: { - {Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha}, - {Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.Beta}, - {Version: version.MustParse("1.32"), Default: true, LockToDefault: true, PreRelease: featuregate.GA}, - }, } diff --git a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go index ced24a03d45..50ac6930b2e 100644 --- a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go +++ b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go @@ -363,6 +363,11 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate {Version: version.MustParse("1.24"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, }, + KMSv1: { + {Version: version.MustParse("1.28"), Default: true, PreRelease: featuregate.Deprecated}, + {Version: version.MustParse("1.29"), Default: false, PreRelease: featuregate.Deprecated}, + }, + MutatingAdmissionPolicy: { {Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha}, }, @@ -448,9 +453,6 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate }, } -// defaultKubernetesFeatureGates consists of all known Kubernetes-specific feature keys. -// To add a new feature, define a key for it above and add it here. The features will be -// available throughout Kubernetes binaries. -var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ - KMSv1: {Default: false, PreRelease: featuregate.Deprecated}, -} +// defaultKubernetesFeatureGates consists of legacy unversioned Kubernetes-specific feature keys. +// Please do not add to this struct and use defaultVersionedKubernetesFeatureGates instead. +var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{} diff --git a/staging/src/k8s.io/controller-manager/pkg/features/kube_features.go b/staging/src/k8s.io/controller-manager/pkg/features/kube_features.go index 639ca19a9f0..a28256d47e6 100644 --- a/staging/src/k8s.io/controller-manager/pkg/features/kube_features.go +++ b/staging/src/k8s.io/controller-manager/pkg/features/kube_features.go @@ -17,6 +17,7 @@ limitations under the License. package features import ( + "k8s.io/apimachinery/pkg/util/version" "k8s.io/component-base/featuregate" ) @@ -39,12 +40,14 @@ const ( CloudControllerManagerWebhook featuregate.Feature = "CloudControllerManagerWebhook" ) -func SetupCurrentKubernetesSpecificFeatureGates(featuregates featuregate.MutableFeatureGate) error { - return featuregates.Add(cloudPublicFeatureGates) +func SetupCurrentKubernetesSpecificFeatureGates(featuregates featuregate.MutableVersionedFeatureGate) error { + return featuregates.AddVersioned(versionedCloudPublicFeatureGates) } -// cloudPublicFeatureGates consists of cloud-specific feature keys. -// To add a new feature, define a key for it at k8s.io/api/pkg/features and add it here. -var cloudPublicFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ - CloudControllerManagerWebhook: {Default: false, PreRelease: featuregate.Alpha}, +// versionedCloudPublicFeatureGates consists of versioned cloud-specific feature keys. +// To add a new feature, define a key for it above and add it here. +var versionedCloudPublicFeatureGates = map[featuregate.Feature]featuregate.VersionedSpecs{ + CloudControllerManagerWebhook: { + {Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha}, + }, } diff --git a/test/featuregates_linter/test_data/unversioned_feature_list.yaml b/test/featuregates_linter/test_data/unversioned_feature_list.yaml index 07d839aea76..4296ca06761 100644 --- a/test/featuregates_linter/test_data/unversioned_feature_list.yaml +++ b/test/featuregates_linter/test_data/unversioned_feature_list.yaml @@ -1,21 +1,3 @@ -- name: AllowDNSOnlyNodeCSR - versionedSpecs: - - default: false - lockToDefault: false - preRelease: Deprecated - version: "" -- name: AllowInsecureKubeletCertificateSigningRequests - versionedSpecs: - - default: false - lockToDefault: false - preRelease: Deprecated - version: "" -- name: CloudControllerManagerWebhook - versionedSpecs: - - default: false - lockToDefault: false - preRelease: Alpha - version: "" - name: ComponentSLIs versionedSpecs: - default: true @@ -28,30 +10,6 @@ lockToDefault: false preRelease: Beta version: "" -- name: DisableNodeKubeProxyVersion - versionedSpecs: - - default: false - lockToDefault: false - preRelease: Deprecated - version: "" -- name: HPAScaleToZero - versionedSpecs: - - default: false - lockToDefault: false - preRelease: Alpha - version: "" -- name: InPlacePodVerticalScaling - versionedSpecs: - - default: false - lockToDefault: false - preRelease: Alpha - version: "" -- name: KMSv1 - versionedSpecs: - - default: false - lockToDefault: false - preRelease: Deprecated - version: "" - name: LoggingAlphaOptions versionedSpecs: - default: false @@ -64,15 +22,3 @@ lockToDefault: false preRelease: Beta version: "" -- name: RecursiveReadOnlyMounts - versionedSpecs: - - default: true - lockToDefault: false - preRelease: Beta - version: "" -- name: StorageNamespaceIndex - versionedSpecs: - - default: true - lockToDefault: false - preRelease: Beta - version: "" diff --git a/test/featuregates_linter/test_data/versioned_feature_list.yaml b/test/featuregates_linter/test_data/versioned_feature_list.yaml index 62199c5265f..e7de7567a78 100644 --- a/test/featuregates_linter/test_data/versioned_feature_list.yaml +++ b/test/featuregates_linter/test_data/versioned_feature_list.yaml @@ -26,6 +26,18 @@ lockToDefault: true preRelease: GA version: "1.30" +- name: AllowDNSOnlyNodeCSR + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Deprecated + version: "1.31" +- name: AllowInsecureKubeletCertificateSigningRequests + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Deprecated + version: "1.31" - name: AllowOverwriteTerminationGracePeriodSeconds versionedSpecs: - default: false @@ -134,6 +146,12 @@ lockToDefault: false preRelease: Alpha version: "1.31" +- name: CloudControllerManagerWebhook + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.27" - name: ClusterTrustBundle versionedSpecs: - default: false @@ -320,6 +338,16 @@ lockToDefault: true preRelease: GA version: "1.31" +- name: DisableNodeKubeProxyVersion + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.29" + - default: false + lockToDefault: false + preRelease: Deprecated + version: "1.31" - name: DRAControlPlaneController versionedSpecs: - default: false @@ -412,6 +440,12 @@ lockToDefault: true preRelease: GA version: "1.30" +- name: HPAScaleToZero + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.16" - name: ImageMaximumGCAge versionedSpecs: - default: false @@ -428,6 +462,12 @@ lockToDefault: false preRelease: Alpha version: "1.31" +- name: InPlacePodVerticalScaling + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.27" - name: InTreePluginPortworxUnregister versionedSpecs: - default: false @@ -484,6 +524,16 @@ lockToDefault: false preRelease: Beta version: "1.31" +- name: KMSv1 + versionedSpecs: + - default: true + lockToDefault: false + preRelease: Deprecated + version: "1.28" + - default: false + lockToDefault: false + preRelease: Deprecated + version: "1.29" - name: KubeletCgroupDriverFromCRI versionedSpecs: - default: false @@ -842,6 +892,16 @@ lockToDefault: false preRelease: Alpha version: "1.23" +- name: RecursiveReadOnlyMounts + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.30" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.31" - name: RelaxedDNSSearchValidation versionedSpecs: - default: false @@ -1048,6 +1108,12 @@ lockToDefault: true preRelease: GA version: "1.31" +- name: StorageNamespaceIndex + versionedSpecs: + - default: true + lockToDefault: false + preRelease: Beta + version: "1.30" - name: StorageVersionAPI versionedSpecs: - default: false