From 28d359beec1907a7e1ef344d0028e294a4b9ad4c Mon Sep 17 00:00:00 2001 From: carlory Date: Mon, 13 Jan 2025 16:04:20 +0800 Subject: [PATCH] promote HonorPVReclaimPolicy to GA Signed-off-by: carlory --- .../volume/persistentvolume/delete_test.go | 3 ++ .../volume/persistentvolume/provision_test.go | 5 ++ .../persistentvolume/pv_controller_test.go | 5 ++ pkg/features/versioned_kube_features.go | 1 + test/e2e/feature/feature.go | 14 ----- test/e2e/storage/csimock/base.go | 2 - .../csimock/csi_honor_pv_reclaim_policy.go | 54 ++++++++----------- test/e2e/storage/drivers/csi.go | 7 --- .../test_data/versioned_feature_list.yaml | 4 ++ 9 files changed, 39 insertions(+), 56 deletions(-) diff --git a/pkg/controller/volume/persistentvolume/delete_test.go b/pkg/controller/volume/persistentvolume/delete_test.go index c520d255111..062c077931b 100644 --- a/pkg/controller/volume/persistentvolume/delete_test.go +++ b/pkg/controller/volume/persistentvolume/delete_test.go @@ -22,6 +22,7 @@ import ( v1 "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/util/version" utilfeature "k8s.io/apiserver/pkg/util/feature" featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/component-helpers/storage/volume" @@ -37,6 +38,8 @@ import ( func TestDeleteSync(t *testing.T) { const gceDriver = "pd.csi.storage.gke.io" // Default enable the HonorPVReclaimPolicy feature gate. + // TODO: this will be removed in 1.36 + featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.32")) featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HonorPVReclaimPolicy, true) _, ctx := ktesting.NewTestContext(t) tests := []controllerTest{ diff --git a/pkg/controller/volume/persistentvolume/provision_test.go b/pkg/controller/volume/persistentvolume/provision_test.go index af76a0683d9..47281b75045 100644 --- a/pkg/controller/volume/persistentvolume/provision_test.go +++ b/pkg/controller/volume/persistentvolume/provision_test.go @@ -29,6 +29,7 @@ import ( storage "k8s.io/api/storage/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/version" corelisters "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/component-helpers/storage/volume" @@ -172,6 +173,8 @@ var provision2Success = provisionCall{ // 3. Compare resulting volumes with expected volumes. func TestProvisionSync(t *testing.T) { // Default enable the HonorPVReclaimPolicy feature gate. + // TODO: this will be removed in 1.36 + featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.32")) featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HonorPVReclaimPolicy, true) _, ctx := ktesting.NewTestContext(t) tests := []controllerTest{ @@ -599,6 +602,8 @@ func TestProvisionSync(t *testing.T) { // Some limit of calls in enforced to prevent endless loops. func TestProvisionMultiSync(t *testing.T) { // Default enable the HonorPVReclaimPolicy feature gate. + // TODO: this will be removed in 1.36 + featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.32")) featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HonorPVReclaimPolicy, true) _, ctx := ktesting.NewTestContext(t) diff --git a/pkg/controller/volume/persistentvolume/pv_controller_test.go b/pkg/controller/volume/persistentvolume/pv_controller_test.go index 2296f7aca20..8414c28fc57 100644 --- a/pkg/controller/volume/persistentvolume/pv_controller_test.go +++ b/pkg/controller/volume/persistentvolume/pv_controller_test.go @@ -26,6 +26,7 @@ import ( v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/version" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" utilfeature "k8s.io/apiserver/pkg/util/feature" @@ -52,6 +53,8 @@ import ( // either very timing-sensitive or slow to wait for real periodic sync. func TestControllerSync(t *testing.T) { // Default enable the HonorPVReclaimPolicy feature gate. + // TODO: this will be removed in 1.36 + featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.32")) featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HonorPVReclaimPolicy, true) tests := []controllerTest{ // [Unit test set 5] - controller tests. @@ -602,6 +605,8 @@ func TestModifyDeletionFinalizers(t *testing.T) { // in-tree plugin is used as migration is disabled. When that plugin is migrated, a different // non-migrated one should be used. If all plugins are migrated this test can be removed. The // gce in-tree plugin is used for a migrated driver as it is feature-locked as of 1.25. + // TODO: this will be removed in 1.36 + featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.32")) featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HonorPVReclaimPolicy, true) const nonmigratedDriver = "rbd.csi.ceph.com" const migratedPlugin = "kubernetes.io/gce-pd" diff --git a/pkg/features/versioned_kube_features.go b/pkg/features/versioned_kube_features.go index 7a75518397b..0002c18d07a 100644 --- a/pkg/features/versioned_kube_features.go +++ b/pkg/features/versioned_kube_features.go @@ -380,6 +380,7 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate HonorPVReclaimPolicy: { {Version: version.MustParse("1.23"), Default: false, PreRelease: featuregate.Alpha}, {Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.Beta}, + {Version: version.MustParse("1.33"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.36 }, HPAScaleToZero: { diff --git a/test/e2e/feature/feature.go b/test/e2e/feature/feature.go index 9fa1da34425..9df5edabfab 100644 --- a/test/e2e/feature/feature.go +++ b/test/e2e/feature/feature.go @@ -192,20 +192,6 @@ var ( // TODO: document the feature (owning SIG, when to use this feature for a test) HPA = framework.WithFeature(framework.ValidFeatures.Add("HPA")) - // owning-sig: sig-storage - // kep: https://kep.k8s.io/2680 - // test-infra jobs: - // - pull-kubernetes-e2e-storage-kind-alpha-features (need manual trigger) - // - ci-kubernetes-e2e-storage-kind-alpha-features - // - // When this label is added to a test, it means that the cluster must be created - // with the feature-gate "HonorPVReclaimPolicy=true". - // - // Once the feature are stable, this label should be removed and these tests will - // be run by default on any cluster. The test-infra job also should be updated to - // not focus on this feature anymore. - HonorPVReclaimPolicy = framework.WithFeature(framework.ValidFeatures.Add("HonorPVReclaimPolicy")) - // owner: sig-node HostAccess = framework.WithFeature(framework.ValidFeatures.Add("HostAccess")) diff --git a/test/e2e/storage/csimock/base.go b/test/e2e/storage/csimock/base.go index 2c2fdd70590..d58a7b33c57 100644 --- a/test/e2e/storage/csimock/base.go +++ b/test/e2e/storage/csimock/base.go @@ -106,7 +106,6 @@ type testParameters struct { fsGroupPolicy *storagev1.FSGroupPolicy enableSELinuxMount *bool enableRecoverExpansionFailure bool - enableHonorPVReclaimPolicy bool enableCSINodeExpandSecret bool reclaimPolicy *v1.PersistentVolumeReclaimPolicy } @@ -181,7 +180,6 @@ func (m *mockDriverSetup) init(ctx context.Context, tp testParameters) { FSGroupPolicy: tp.fsGroupPolicy, EnableSELinuxMount: tp.enableSELinuxMount, EnableRecoverExpansionFailure: tp.enableRecoverExpansionFailure, - EnableHonorPVReclaimPolicy: tp.enableHonorPVReclaimPolicy, } // At the moment, only tests which need hooks are diff --git a/test/e2e/storage/csimock/csi_honor_pv_reclaim_policy.go b/test/e2e/storage/csimock/csi_honor_pv_reclaim_policy.go index 17dbabf06f5..07767bd7b2b 100644 --- a/test/e2e/storage/csimock/csi_honor_pv_reclaim_policy.go +++ b/test/e2e/storage/csimock/csi_honor_pv_reclaim_policy.go @@ -28,8 +28,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/retry" storagehelpers "k8s.io/component-helpers/storage/volume" - "k8s.io/kubernetes/pkg/features" - "k8s.io/kubernetes/test/e2e/feature" "k8s.io/kubernetes/test/e2e/framework" e2epv "k8s.io/kubernetes/test/e2e/framework/pv" "k8s.io/kubernetes/test/e2e/storage/utils" @@ -37,7 +35,7 @@ import ( "k8s.io/utils/ptr" ) -var _ = utils.SIGDescribe("CSI Mock honor pv reclaim policy", feature.HonorPVReclaimPolicy, framework.WithFeatureGate(features.HonorPVReclaimPolicy), func() { +var _ = utils.SIGDescribe("CSI Mock honor pv reclaim policy", func() { f := framework.NewDefaultFramework("csi-mock-honor-pv-reclaim-policy") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged m := newMockDriverSetup(f) @@ -45,9 +43,8 @@ var _ = utils.SIGDescribe("CSI Mock honor pv reclaim policy", feature.HonorPVRec ginkgo.Context("CSI honor pv reclaim policy using mock driver", func() { ginkgo.It("Dynamic provisioning should honor pv delete reclaim policy when deleting pvc", func(ctx context.Context) { m.init(ctx, testParameters{ - registerDriver: true, - enableHonorPVReclaimPolicy: true, - reclaimPolicy: ptr.To(v1.PersistentVolumeReclaimDelete), + registerDriver: true, + reclaimPolicy: ptr.To(v1.PersistentVolumeReclaimDelete), }) ginkgo.DeferCleanup(m.cleanup) @@ -80,9 +77,8 @@ var _ = utils.SIGDescribe("CSI Mock honor pv reclaim policy", feature.HonorPVRec ginkgo.It("Dynamic provisioning should honor pv delete reclaim policy when deleting pv then pvc", func(ctx context.Context) { m.init(ctx, testParameters{ - registerDriver: true, - enableHonorPVReclaimPolicy: true, - reclaimPolicy: ptr.To(v1.PersistentVolumeReclaimDelete), + registerDriver: true, + reclaimPolicy: ptr.To(v1.PersistentVolumeReclaimDelete), }) ginkgo.DeferCleanup(m.cleanup) @@ -119,9 +115,8 @@ var _ = utils.SIGDescribe("CSI Mock honor pv reclaim policy", feature.HonorPVRec ginkgo.It("Dynamic provisioning should honor pv retain reclaim policy when deleting pvc then pv", func(ctx context.Context) { m.init(ctx, testParameters{ - registerDriver: true, - enableHonorPVReclaimPolicy: true, - reclaimPolicy: ptr.To(v1.PersistentVolumeReclaimRetain), + registerDriver: true, + reclaimPolicy: ptr.To(v1.PersistentVolumeReclaimRetain), }) ginkgo.DeferCleanup(m.cleanup) @@ -166,9 +161,8 @@ var _ = utils.SIGDescribe("CSI Mock honor pv reclaim policy", feature.HonorPVRec ginkgo.It("Dynamic provisioning should honor pv retain reclaim policy when deleting pv then pvc", func(ctx context.Context) { m.init(ctx, testParameters{ - registerDriver: true, - enableHonorPVReclaimPolicy: true, - reclaimPolicy: ptr.To(v1.PersistentVolumeReclaimRetain), + registerDriver: true, + reclaimPolicy: ptr.To(v1.PersistentVolumeReclaimRetain), }) ginkgo.DeferCleanup(m.cleanup) @@ -207,9 +201,8 @@ var _ = utils.SIGDescribe("CSI Mock honor pv reclaim policy", feature.HonorPVRec ginkgo.It("Static provisioning should honor pv delete reclaim policy when deleting pvc", func(ctx context.Context) { m.init(ctx, testParameters{ - registerDriver: true, - enableHonorPVReclaimPolicy: true, - reclaimPolicy: ptr.To(v1.PersistentVolumeReclaimDelete), + registerDriver: true, + reclaimPolicy: ptr.To(v1.PersistentVolumeReclaimDelete), }) ginkgo.DeferCleanup(m.cleanup) @@ -237,9 +230,8 @@ var _ = utils.SIGDescribe("CSI Mock honor pv reclaim policy", feature.HonorPVRec ginkgo.It("Static provisioning should honor pv delete reclaim policy when deleting pv then pvc", func(ctx context.Context) { m.init(ctx, testParameters{ - registerDriver: true, - enableHonorPVReclaimPolicy: true, - reclaimPolicy: ptr.To(v1.PersistentVolumeReclaimDelete), + registerDriver: true, + reclaimPolicy: ptr.To(v1.PersistentVolumeReclaimDelete), }) ginkgo.DeferCleanup(m.cleanup) @@ -271,9 +263,8 @@ var _ = utils.SIGDescribe("CSI Mock honor pv reclaim policy", feature.HonorPVRec ginkgo.It("Static provisioning should honor pv retain reclaim policy when deleting pvc then pv", func(ctx context.Context) { m.init(ctx, testParameters{ - registerDriver: true, - enableHonorPVReclaimPolicy: true, - reclaimPolicy: ptr.To(v1.PersistentVolumeReclaimRetain), + registerDriver: true, + reclaimPolicy: ptr.To(v1.PersistentVolumeReclaimRetain), }) ginkgo.DeferCleanup(m.cleanup) @@ -311,9 +302,8 @@ var _ = utils.SIGDescribe("CSI Mock honor pv reclaim policy", feature.HonorPVRec ginkgo.It("Static provisioning should honor pv retain reclaim policy when deleting pv then pvc", func(ctx context.Context) { m.init(ctx, testParameters{ - registerDriver: true, - enableHonorPVReclaimPolicy: true, - reclaimPolicy: ptr.To(v1.PersistentVolumeReclaimRetain), + registerDriver: true, + reclaimPolicy: ptr.To(v1.PersistentVolumeReclaimRetain), }) ginkgo.DeferCleanup(m.cleanup) @@ -347,9 +337,8 @@ var _ = utils.SIGDescribe("CSI Mock honor pv reclaim policy", feature.HonorPVRec ginkgo.Context("CSI honor pv reclaim policy changes using mock driver", func() { ginkgo.It("should honor pv reclaim policy after it is changed from retain to deleted", func(ctx context.Context) { m.init(ctx, testParameters{ - registerDriver: true, - enableHonorPVReclaimPolicy: true, - reclaimPolicy: ptr.To(v1.PersistentVolumeReclaimRetain), + registerDriver: true, + reclaimPolicy: ptr.To(v1.PersistentVolumeReclaimRetain), }) ginkgo.DeferCleanup(m.cleanup) @@ -405,9 +394,8 @@ var _ = utils.SIGDescribe("CSI Mock honor pv reclaim policy", feature.HonorPVRec ginkgo.It("should honor pv reclaim policy after it is changed from deleted to retain", func(ctx context.Context) { m.init(ctx, testParameters{ - registerDriver: true, - enableHonorPVReclaimPolicy: true, - reclaimPolicy: ptr.To(v1.PersistentVolumeReclaimDelete), + registerDriver: true, + reclaimPolicy: ptr.To(v1.PersistentVolumeReclaimDelete), }) ginkgo.DeferCleanup(m.cleanup) diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index d2e99b724f3..75e757b7728 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -62,7 +62,6 @@ import ( "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" - "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/test/e2e/feature" "k8s.io/kubernetes/test/e2e/framework" e2enode "k8s.io/kubernetes/test/e2e/framework/node" @@ -356,7 +355,6 @@ type mockCSIDriver struct { enableSELinuxMount *bool enableRecoverExpansionFailure bool disableControllerExpansion bool - enableHonorPVReclaimPolicy bool // Additional values set during PrepareTest clientSet clientset.Interface @@ -407,7 +405,6 @@ type CSIMockDriverOpts struct { FSGroupPolicy *storagev1.FSGroupPolicy EnableSELinuxMount *bool EnableRecoverExpansionFailure bool - EnableHonorPVReclaimPolicy bool // Embedded defines whether the CSI mock driver runs // inside the cluster (false, the default) or just a proxy @@ -564,7 +561,6 @@ func InitMockCSIDriver(driverOpts CSIMockDriverOpts) MockCSITestDriver { enableVolumeMountGroup: driverOpts.EnableVolumeMountGroup, enableSELinuxMount: driverOpts.EnableSELinuxMount, enableRecoverExpansionFailure: driverOpts.EnableRecoverExpansionFailure, - enableHonorPVReclaimPolicy: driverOpts.EnableHonorPVReclaimPolicy, embedded: driverOpts.Embedded, hooks: driverOpts.Hooks, } @@ -726,9 +722,6 @@ func (m *mockCSIDriver) PrepareTest(ctx context.Context, f *framework.Framework) if m.enableRecoverExpansionFailure { o.Features["csi-resizer"] = []string{"RecoverVolumeExpansionFailure=true"} } - if m.enableHonorPVReclaimPolicy { - o.Features["csi-provisioner"] = append(o.Features["csi-provisioner"], fmt.Sprintf("%s=true", features.HonorPVReclaimPolicy)) - } err = utils.CreateFromManifests(ctx, f, m.driverNamespace, func(item interface{}) error { if err := utils.PatchCSIDeployment(config.Framework, o, item); err != nil { diff --git a/test/featuregates_linter/test_data/versioned_feature_list.yaml b/test/featuregates_linter/test_data/versioned_feature_list.yaml index 975224f7011..deb494360aa 100644 --- a/test/featuregates_linter/test_data/versioned_feature_list.yaml +++ b/test/featuregates_linter/test_data/versioned_feature_list.yaml @@ -500,6 +500,10 @@ lockToDefault: false preRelease: Beta version: "1.31" + - default: true + lockToDefault: true + preRelease: GA + version: "1.33" - name: HPAScaleToZero versionedSpecs: - default: false