From bda557b4bc3f5ec655d32a9ec23345ed9c77147b Mon Sep 17 00:00:00 2001 From: Jiawei Wang Date: Wed, 20 Jan 2021 14:51:16 -0800 Subject: [PATCH] Disable in-tree plugin without enabling CSI migration This commit replaces the CSIMigrationXXXComplete flag with InTreePluginXXUnregister flag. This new flag will be a superset of the CSIMigrationXXXComplete. But this decouple the plugin unregister from CSI migration. So if a K8s distribution want to go directly with CSI and do not support in-tree, they can use this flag directly. Testing: 1. Enable the InTreePluginXXUnregister and not CSIMigrationXXX, verify that the PVC using old plugin name will have error saying cannot find the plugin 2. Enable both the InTreePluginXXUnregister and CSIMigrationXXX verify that the PVC using old plugin name will start to use the migrated CSI plugin --- .../app/plugins_providers.go | 62 +++-- cmd/kubelet/app/plugins_providers.go | 36 ++- pkg/features/kube_features.go | 164 +++++++------ pkg/volume/csimigration/plugin_manager.go | 47 ++-- .../csimigration/plugin_manager_test.go | 231 ++++++++++-------- 5 files changed, 304 insertions(+), 236 deletions(-) diff --git a/cmd/kube-controller-manager/app/plugins_providers.go b/cmd/kube-controller-manager/app/plugins_providers.go index 093e236dc23..915e1b715cb 100644 --- a/cmd/kube-controller-manager/app/plugins_providers.go +++ b/cmd/kube-controller-manager/app/plugins_providers.go @@ -35,40 +35,48 @@ import ( type probeFn func() []volume.VolumePlugin -func appendPluginBasedOnMigrationFeatureFlags(plugins []volume.VolumePlugin, inTreePluginName string, featureGate featuregate.FeatureGate, pluginMigration, pluginMigrationComplete featuregate.Feature, fn probeFn) ([]volume.VolumePlugin, error) { +func appendPluginBasedOnFeatureFlags(plugins []volume.VolumePlugin, inTreePluginName string, featureGate featuregate.FeatureGate, pluginInfo pluginInfo) ([]volume.VolumePlugin, error) { // Skip appending the in-tree plugin to the list of plugins to be probed/initialized // if the CSIMigration feature flag and plugin specific feature flag indicating // CSI migration is complete - err := csimigration.CheckMigrationFeatureFlags(featureGate, pluginMigration, pluginMigrationComplete) + migrationComplete, err := csimigration.CheckMigrationFeatureFlags(featureGate, pluginInfo.pluginMigrationFeature, + pluginInfo.pluginMigrationCompleteFeature, pluginInfo.pluginUnregisterFeature) if err != nil { klog.Warningf("Unexpected CSI Migration Feature Flags combination detected: %v. CSI Migration may not take effect", err) // TODO: fail and return here once alpha only tests can set the feature flags for a plugin correctly } - if featureGate.Enabled(features.CSIMigration) && featureGate.Enabled(pluginMigration) && featureGate.Enabled(pluginMigrationComplete) { - klog.Infof("Skip registration of plugin %s since feature flag %v is enabled", inTreePluginName, pluginMigrationComplete) + // TODO: This can be removed after feature flag CSIMigrationvSphereComplete is removed. + if migrationComplete { + klog.Infof("Skip registration of plugin %s since migration is complete", inTreePluginName) return plugins, nil } - plugins = append(plugins, fn()...) + if featureGate.Enabled(pluginInfo.pluginUnregisterFeature) { + klog.Infof("Skip registration of plugin %s since feature flag %v is enabled", inTreePluginName, pluginInfo.pluginUnregisterFeature) + return plugins, nil + } + plugins = append(plugins, pluginInfo.pluginProbeFunction()...) return plugins, nil } type pluginInfo struct { - pluginMigrationFeature featuregate.Feature + pluginMigrationFeature featuregate.Feature + // deprecated, only to keep here for vSphere pluginMigrationCompleteFeature featuregate.Feature + pluginUnregisterFeature featuregate.Feature pluginProbeFunction probeFn } func appendAttachableLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) { pluginMigrationStatus := make(map[string]pluginInfo) - pluginMigrationStatus[plugins.AWSEBSInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAWS, pluginMigrationCompleteFeature: features.CSIMigrationAWSComplete, pluginProbeFunction: awsebs.ProbeVolumePlugins} - pluginMigrationStatus[plugins.GCEPDInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationGCE, pluginMigrationCompleteFeature: features.CSIMigrationGCEComplete, pluginProbeFunction: gcepd.ProbeVolumePlugins} - pluginMigrationStatus[plugins.CinderInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationOpenStack, pluginMigrationCompleteFeature: features.CSIMigrationOpenStackComplete, pluginProbeFunction: cinder.ProbeVolumePlugins} - pluginMigrationStatus[plugins.AzureDiskInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureDisk, pluginMigrationCompleteFeature: features.CSIMigrationAzureDiskComplete, pluginProbeFunction: azuredd.ProbeVolumePlugins} - pluginMigrationStatus[plugins.VSphereInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationvSphere, pluginMigrationCompleteFeature: features.CSIMigrationvSphereComplete, pluginProbeFunction: vsphere_volume.ProbeVolumePlugins} + pluginMigrationStatus[plugins.AWSEBSInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAWS, pluginUnregisterFeature: features.InTreePluginAWSUnregister, pluginProbeFunction: awsebs.ProbeVolumePlugins} + pluginMigrationStatus[plugins.GCEPDInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationGCE, pluginUnregisterFeature: features.InTreePluginGCEUnregister, pluginProbeFunction: gcepd.ProbeVolumePlugins} + pluginMigrationStatus[plugins.CinderInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationOpenStack, pluginUnregisterFeature: features.InTreePluginOpenStackUnregister, pluginProbeFunction: cinder.ProbeVolumePlugins} + pluginMigrationStatus[plugins.AzureDiskInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureDisk, pluginUnregisterFeature: features.InTreePluginAzureDiskUnregister, pluginProbeFunction: azuredd.ProbeVolumePlugins} + pluginMigrationStatus[plugins.VSphereInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationvSphere, pluginMigrationCompleteFeature: features.CSIMigrationvSphereComplete, pluginUnregisterFeature: features.InTreePluginvSphereUnregister, pluginProbeFunction: vsphere_volume.ProbeVolumePlugins} var err error for pluginName, pluginInfo := range pluginMigrationStatus { - allPlugins, err = appendPluginBasedOnMigrationFeatureFlags(allPlugins, pluginName, featureGate, pluginInfo.pluginMigrationFeature, pluginInfo.pluginMigrationCompleteFeature, pluginInfo.pluginProbeFunction) + allPlugins, err = appendPluginBasedOnFeatureFlags(allPlugins, pluginName, featureGate, pluginInfo) if err != nil { return allPlugins, err } @@ -81,20 +89,24 @@ func appendExpandableLegacyProviderVolumes(allPlugins []volume.VolumePlugin, fea } func appendLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) { - pluginMigrationStatus := make(map[string]pluginInfo) - pluginMigrationStatus[plugins.AWSEBSInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAWS, pluginMigrationCompleteFeature: features.CSIMigrationAWSComplete, pluginProbeFunction: awsebs.ProbeVolumePlugins} - pluginMigrationStatus[plugins.GCEPDInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationGCE, pluginMigrationCompleteFeature: features.CSIMigrationGCEComplete, pluginProbeFunction: gcepd.ProbeVolumePlugins} - pluginMigrationStatus[plugins.CinderInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationOpenStack, pluginMigrationCompleteFeature: features.CSIMigrationOpenStackComplete, pluginProbeFunction: cinder.ProbeVolumePlugins} - pluginMigrationStatus[plugins.AzureDiskInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureDisk, pluginMigrationCompleteFeature: features.CSIMigrationAzureDiskComplete, pluginProbeFunction: azuredd.ProbeVolumePlugins} - pluginMigrationStatus[plugins.AzureFileInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureFile, pluginMigrationCompleteFeature: features.CSIMigrationAzureFileComplete, pluginProbeFunction: azure_file.ProbeVolumePlugins} - pluginMigrationStatus[plugins.VSphereInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationvSphere, pluginMigrationCompleteFeature: features.CSIMigrationvSphereComplete, pluginProbeFunction: vsphere_volume.ProbeVolumePlugins} - var err error - for pluginName, pluginInfo := range pluginMigrationStatus { - allPlugins, err = appendPluginBasedOnMigrationFeatureFlags(allPlugins, pluginName, featureGate, pluginInfo.pluginMigrationFeature, pluginInfo.pluginMigrationCompleteFeature, pluginInfo.pluginProbeFunction) - if err != nil { - return allPlugins, err - } + // First append attachable volumes + allPlugins, err = appendAttachableLegacyProviderVolumes(allPlugins, featureGate) + if err != nil { + return allPlugins, err } + + // Then append non-attachable volumes + pluginName := plugins.AzureFileInTreePluginName + pluginInfo := pluginInfo{ + pluginMigrationFeature: features.CSIMigrationAzureFile, + pluginUnregisterFeature: features.InTreePluginAzureFileUnregister, + pluginProbeFunction: azure_file.ProbeVolumePlugins, + } + allPlugins, err = appendPluginBasedOnFeatureFlags(allPlugins, pluginName, featureGate, pluginInfo) + if err != nil { + return allPlugins, err + } + return allPlugins, nil } diff --git a/cmd/kubelet/app/plugins_providers.go b/cmd/kubelet/app/plugins_providers.go index 9f8ce86ff0b..bd71645de23 100644 --- a/cmd/kubelet/app/plugins_providers.go +++ b/cmd/kubelet/app/plugins_providers.go @@ -35,41 +35,51 @@ import ( type probeFn func() []volume.VolumePlugin -func appendPluginBasedOnMigrationFeatureFlags(plugins []volume.VolumePlugin, inTreePluginName string, featureGate featuregate.FeatureGate, pluginMigration, pluginMigrationComplete featuregate.Feature, fn probeFn) ([]volume.VolumePlugin, error) { +func appendPluginBasedOnFeatureFlags(plugins []volume.VolumePlugin, inTreePluginName string, + featureGate featuregate.FeatureGate, pluginInfo pluginInfo) ([]volume.VolumePlugin, error) { // Skip appending the in-tree plugin to the list of plugins to be probed/initialized // if the CSIMigration feature flag and plugin specific feature flag indicating // CSI migration is complete - err := csimigration.CheckMigrationFeatureFlags(featureGate, pluginMigration, pluginMigrationComplete) + migrationComplete, err := csimigration.CheckMigrationFeatureFlags(featureGate, pluginInfo.pluginMigrationFeature, + pluginInfo.pluginMigrationCompleteFeature, pluginInfo.pluginUnregisterFeature) if err != nil { klog.Warningf("Unexpected CSI Migration Feature Flags combination detected: %v. CSI Migration may not take effect", err) // TODO: fail and return here once alpha only tests can set the feature flags for a plugin correctly } - if featureGate.Enabled(features.CSIMigration) && featureGate.Enabled(pluginMigration) && featureGate.Enabled(pluginMigrationComplete) { - klog.Infof("Skip registration of plugin %s since feature flag %v is enabled", inTreePluginName, pluginMigrationComplete) + // TODO: This can be removed after feature flag CSIMigrationvSphereComplete is removed. + if migrationComplete { + klog.Infof("Skip registration of plugin %s since migration is complete", inTreePluginName) return plugins, nil } - plugins = append(plugins, fn()...) + if featureGate.Enabled(pluginInfo.pluginUnregisterFeature) { + klog.Infof("Skip registration of plugin %s since feature flag %v is enabled", inTreePluginName, pluginInfo.pluginUnregisterFeature) + return plugins, nil + } + + plugins = append(plugins, pluginInfo.pluginProbeFunction()...) return plugins, nil } type pluginInfo struct { - pluginMigrationFeature featuregate.Feature + pluginMigrationFeature featuregate.Feature + // deprecated, only to keep here for vSphere pluginMigrationCompleteFeature featuregate.Feature + pluginUnregisterFeature featuregate.Feature pluginProbeFunction probeFn } func appendLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) { pluginMigrationStatus := make(map[string]pluginInfo) - pluginMigrationStatus[plugins.AWSEBSInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAWS, pluginMigrationCompleteFeature: features.CSIMigrationAWSComplete, pluginProbeFunction: awsebs.ProbeVolumePlugins} - pluginMigrationStatus[plugins.GCEPDInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationGCE, pluginMigrationCompleteFeature: features.CSIMigrationGCEComplete, pluginProbeFunction: gcepd.ProbeVolumePlugins} - pluginMigrationStatus[plugins.CinderInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationOpenStack, pluginMigrationCompleteFeature: features.CSIMigrationOpenStackComplete, pluginProbeFunction: cinder.ProbeVolumePlugins} - pluginMigrationStatus[plugins.AzureDiskInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureDisk, pluginMigrationCompleteFeature: features.CSIMigrationAzureDiskComplete, pluginProbeFunction: azuredd.ProbeVolumePlugins} - pluginMigrationStatus[plugins.AzureFileInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureFile, pluginMigrationCompleteFeature: features.CSIMigrationAzureFileComplete, pluginProbeFunction: azure_file.ProbeVolumePlugins} - pluginMigrationStatus[plugins.VSphereInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationvSphere, pluginMigrationCompleteFeature: features.CSIMigrationvSphereComplete, pluginProbeFunction: vsphere_volume.ProbeVolumePlugins} + pluginMigrationStatus[plugins.AWSEBSInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAWS, pluginUnregisterFeature: features.InTreePluginAWSUnregister, pluginProbeFunction: awsebs.ProbeVolumePlugins} + pluginMigrationStatus[plugins.GCEPDInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationGCE, pluginUnregisterFeature: features.InTreePluginGCEUnregister, pluginProbeFunction: gcepd.ProbeVolumePlugins} + pluginMigrationStatus[plugins.CinderInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationOpenStack, pluginUnregisterFeature: features.InTreePluginOpenStackUnregister, pluginProbeFunction: cinder.ProbeVolumePlugins} + pluginMigrationStatus[plugins.AzureDiskInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureDisk, pluginUnregisterFeature: features.InTreePluginAzureDiskUnregister, pluginProbeFunction: azuredd.ProbeVolumePlugins} + pluginMigrationStatus[plugins.AzureFileInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureFile, pluginUnregisterFeature: features.InTreePluginAzureFileUnregister, pluginProbeFunction: azure_file.ProbeVolumePlugins} + pluginMigrationStatus[plugins.VSphereInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationvSphere, pluginMigrationCompleteFeature: features.CSIMigrationvSphereComplete, pluginUnregisterFeature: features.InTreePluginvSphereUnregister, pluginProbeFunction: vsphere_volume.ProbeVolumePlugins} var err error for pluginName, pluginInfo := range pluginMigrationStatus { - allPlugins, err = appendPluginBasedOnMigrationFeatureFlags(allPlugins, pluginName, featureGate, pluginInfo.pluginMigrationFeature, pluginInfo.pluginMigrationCompleteFeature, pluginInfo.pluginProbeFunction) + allPlugins, err = appendPluginBasedOnFeatureFlags(allPlugins, pluginName, featureGate, pluginInfo) if err != nil { return allPlugins, err } diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 0f51df87b0c..ddba2f4ccce 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -328,12 +328,11 @@ const ( // Enables the GCE PD in-tree driver to GCE CSI Driver migration feature. CSIMigrationGCE featuregate.Feature = "CSIMigrationGCE" - // owner: @davidz627 - // alpha: v1.17 + // owner: @Jiawei0227 + // alpha: v1.21 // // Disables the GCE PD in-tree driver. - // Expects GCE PD CSI Driver to be installed and configured on all nodes. - CSIMigrationGCEComplete featuregate.Feature = "CSIMigrationGCEComplete" + InTreePluginGCEUnregister featuregate.Feature = "InTreePluginGCEUnregister" // owner: @leakingtapan // alpha: v1.14 @@ -343,11 +342,10 @@ const ( CSIMigrationAWS featuregate.Feature = "CSIMigrationAWS" // owner: @leakingtapan - // alpha: v1.17 + // alpha: v1.21 // // Disables the AWS EBS in-tree driver. - // Expects AWS EBS CSI Driver to be installed and configured on all nodes. - CSIMigrationAWSComplete featuregate.Feature = "CSIMigrationAWSComplete" + InTreePluginAWSUnregister featuregate.Feature = "InTreePluginAWSUnregister" // owner: @andyzhangx // alpha: v1.15 @@ -357,11 +355,10 @@ const ( CSIMigrationAzureDisk featuregate.Feature = "CSIMigrationAzureDisk" // owner: @andyzhangx - // alpha: v1.17 + // alpha: v1.21 // // Disables the Azure Disk in-tree driver. - // Expects Azure Disk CSI Driver to be installed and configured on all nodes. - CSIMigrationAzureDiskComplete featuregate.Feature = "CSIMigrationAzureDiskComplete" + InTreePluginAzureDiskUnregister featuregate.Feature = "InTreePluginAzureDiskUnregister" // owner: @andyzhangx // alpha: v1.15 @@ -370,11 +367,10 @@ const ( CSIMigrationAzureFile featuregate.Feature = "CSIMigrationAzureFile" // owner: @andyzhangx - // alpha: v1.17 + // alpha: v1.21 // // Disables the Azure File in-tree driver. - // Expects Azure File CSI Driver to be installed and configured on all nodes. - CSIMigrationAzureFileComplete featuregate.Feature = "CSIMigrationAzureFileComplete" + InTreePluginAzureFileUnregister featuregate.Feature = "InTreePluginAzureFileUnregister" // owner: @divyenpatel // beta: v1.19 (requires: vSphere vCenter/ESXi Version: 7.0u1, HW Version: VM version 15) @@ -389,6 +385,25 @@ const ( // Expects vSphere CSI Driver to be installed and configured on all nodes. CSIMigrationvSphereComplete featuregate.Feature = "CSIMigrationvSphereComplete" + // owner: @divyenpatel + // alpha: v1.21 + // + // Disables the vSphere in-tree driver. + InTreePluginvSphereUnregister featuregate.Feature = "InTreePluginvSphereUnregister" + + // owner: @adisky + // alpha: v1.14 + // beta: v1.18 + // + // Enables the OpenStack Cinder in-tree driver to OpenStack Cinder CSI Driver migration feature. + CSIMigrationOpenStack featuregate.Feature = "CSIMigrationOpenStack" + + // owner: @adisky + // alpha: v1.21 + // + // Disables the OpenStack Cinder in-tree driver. + InTreePluginOpenStackUnregister featuregate.Feature = "InTreePluginOpenStackUnregister" + // owner: @huffmanca // alpha: v1.19 // beta: v1.20 @@ -425,20 +440,6 @@ const ( // Enables support for running container entrypoints as different usernames than their default ones. WindowsRunAsUserName featuregate.Feature = "WindowsRunAsUserName" - // owner: @adisky - // alpha: v1.14 - // beta: v1.18 - // - // Enables the OpenStack Cinder in-tree driver to OpenStack Cinder CSI Driver migration feature. - CSIMigrationOpenStack featuregate.Feature = "CSIMigrationOpenStack" - - // owner: @adisky - // alpha: v1.17 - // - // Disables the OpenStack Cinder in-tree driver. - // Expects the OpenStack Cinder CSI Driver to be installed and configured on all nodes. - CSIMigrationOpenStackComplete featuregate.Feature = "CSIMigrationOpenStackComplete" - // owner: @RobertKrawitz // alpha: v1.15 // @@ -683,60 +684,61 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS AppArmor: {Default: true, PreRelease: featuregate.Beta}, DynamicKubeletConfig: {Default: true, PreRelease: featuregate.Beta}, ExperimentalHostUserNamespaceDefaultingGate: {Default: false, PreRelease: featuregate.Beta}, - DevicePlugins: {Default: true, PreRelease: featuregate.Beta}, - RotateKubeletServerCertificate: {Default: true, PreRelease: featuregate.Beta}, - LocalStorageCapacityIsolation: {Default: true, PreRelease: featuregate.Beta}, - Sysctls: {Default: true, PreRelease: featuregate.Beta}, - EphemeralContainers: {Default: false, PreRelease: featuregate.Alpha}, - QOSReserved: {Default: false, PreRelease: featuregate.Alpha}, - ExpandPersistentVolumes: {Default: true, PreRelease: featuregate.Beta}, - ExpandInUsePersistentVolumes: {Default: true, PreRelease: featuregate.Beta}, - ExpandCSIVolumes: {Default: true, PreRelease: featuregate.Beta}, - CPUManager: {Default: true, PreRelease: featuregate.Beta}, - CPUCFSQuotaPeriod: {Default: false, PreRelease: featuregate.Alpha}, - TopologyManager: {Default: true, PreRelease: featuregate.Beta}, - ServiceNodeExclusion: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.22 - NodeDisruptionExclusion: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.22 - CSIDriverRegistry: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.20 - BlockVolume: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.20 - StorageObjectInUseProtection: {Default: true, PreRelease: featuregate.GA}, - SupportPodPidsLimit: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.23 - SupportNodePidsLimit: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.23 - BoundServiceAccountTokenVolume: {Default: false, PreRelease: featuregate.Alpha}, - ServiceAccountIssuerDiscovery: {Default: true, PreRelease: featuregate.Beta}, - CRIContainerLogRotation: {Default: true, PreRelease: featuregate.Beta}, - CSIMigration: {Default: true, PreRelease: featuregate.Beta}, - CSIMigrationGCE: {Default: false, PreRelease: featuregate.Beta}, // Off by default (requires GCE PD CSI Driver) - CSIMigrationGCEComplete: {Default: false, PreRelease: featuregate.Alpha}, - CSIMigrationAWS: {Default: false, PreRelease: featuregate.Beta}, // Off by default (requires AWS EBS CSI driver) - CSIMigrationAWSComplete: {Default: false, PreRelease: featuregate.Alpha}, - CSIMigrationAzureDisk: {Default: false, PreRelease: featuregate.Beta}, // Off by default (requires Azure Disk CSI driver) - CSIMigrationAzureDiskComplete: {Default: false, PreRelease: featuregate.Alpha}, - CSIMigrationAzureFile: {Default: false, PreRelease: featuregate.Alpha}, - CSIMigrationAzureFileComplete: {Default: false, PreRelease: featuregate.Alpha}, - CSIMigrationvSphere: {Default: false, PreRelease: featuregate.Beta}, - CSIMigrationvSphereComplete: {Default: false, PreRelease: featuregate.Beta}, - RunAsGroup: {Default: true, PreRelease: featuregate.Beta}, - CSIMigrationOpenStack: {Default: false, PreRelease: featuregate.Beta}, // Off by default (requires OpenStack Cinder CSI driver) - CSIMigrationOpenStackComplete: {Default: false, PreRelease: featuregate.Alpha}, - VolumeSubpath: {Default: true, PreRelease: featuregate.GA}, - ConfigurableFSGroupPolicy: {Default: true, PreRelease: featuregate.Beta}, - BalanceAttachedNodeVolumes: {Default: false, PreRelease: featuregate.Alpha}, - CSIBlockVolume: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.20 - CSIInlineVolume: {Default: true, PreRelease: featuregate.Beta}, - CSIStorageCapacity: {Default: false, PreRelease: featuregate.Alpha}, - CSIServiceAccountToken: {Default: false, PreRelease: featuregate.Alpha}, - GenericEphemeralVolume: {Default: false, PreRelease: featuregate.Alpha}, - CSIVolumeFSGroupPolicy: {Default: true, PreRelease: featuregate.Beta}, - RuntimeClass: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.23 - NodeLease: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, - SCTPSupport: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.22 - VolumeSnapshotDataSource: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.21 - ProcMountType: {Default: false, PreRelease: featuregate.Alpha}, - TTLAfterFinished: {Default: false, PreRelease: featuregate.Alpha}, - KubeletPodResources: {Default: true, PreRelease: featuregate.Beta}, - WindowsGMSA: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.20 - WindowsRunAsUserName: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.20 + DevicePlugins: {Default: true, PreRelease: featuregate.Beta}, + RotateKubeletServerCertificate: {Default: true, PreRelease: featuregate.Beta}, + LocalStorageCapacityIsolation: {Default: true, PreRelease: featuregate.Beta}, + Sysctls: {Default: true, PreRelease: featuregate.Beta}, + EphemeralContainers: {Default: false, PreRelease: featuregate.Alpha}, + QOSReserved: {Default: false, PreRelease: featuregate.Alpha}, + ExpandPersistentVolumes: {Default: true, PreRelease: featuregate.Beta}, + ExpandInUsePersistentVolumes: {Default: true, PreRelease: featuregate.Beta}, + ExpandCSIVolumes: {Default: true, PreRelease: featuregate.Beta}, + CPUManager: {Default: true, PreRelease: featuregate.Beta}, + CPUCFSQuotaPeriod: {Default: false, PreRelease: featuregate.Alpha}, + TopologyManager: {Default: true, PreRelease: featuregate.Beta}, + ServiceNodeExclusion: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.22 + NodeDisruptionExclusion: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.22 + CSIDriverRegistry: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.20 + BlockVolume: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.20 + StorageObjectInUseProtection: {Default: true, PreRelease: featuregate.GA}, + SupportPodPidsLimit: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.23 + SupportNodePidsLimit: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.23 + BoundServiceAccountTokenVolume: {Default: false, PreRelease: featuregate.Alpha}, + ServiceAccountIssuerDiscovery: {Default: true, PreRelease: featuregate.Beta}, + CRIContainerLogRotation: {Default: true, PreRelease: featuregate.Beta}, + CSIMigration: {Default: true, PreRelease: featuregate.Beta}, + CSIMigrationGCE: {Default: false, PreRelease: featuregate.Beta}, // Off by default (requires GCE PD CSI Driver) + InTreePluginGCEUnregister: {Default: false, PreRelease: featuregate.Alpha}, + CSIMigrationAWS: {Default: false, PreRelease: featuregate.Beta}, // Off by default (requires AWS EBS CSI driver) + InTreePluginAWSUnregister: {Default: false, PreRelease: featuregate.Alpha}, + CSIMigrationAzureDisk: {Default: false, PreRelease: featuregate.Beta}, // Off by default (requires Azure Disk CSI driver) + InTreePluginAzureDiskUnregister: {Default: false, PreRelease: featuregate.Alpha}, + CSIMigrationAzureFile: {Default: false, PreRelease: featuregate.Alpha}, // Off by default (requires Azure File CSI driver) + InTreePluginAzureFileUnregister: {Default: false, PreRelease: featuregate.Alpha}, + CSIMigrationvSphere: {Default: false, PreRelease: featuregate.Beta}, // Off by default (requires vSphere CSI driver) + CSIMigrationvSphereComplete: {Default: false, PreRelease: featuregate.Beta}, // remove in 1.22 + InTreePluginvSphereUnregister: {Default: false, PreRelease: featuregate.Alpha}, + RunAsGroup: {Default: true, PreRelease: featuregate.Beta}, + CSIMigrationOpenStack: {Default: false, PreRelease: featuregate.Beta}, // Off by default (requires OpenStack Cinder CSI driver) + InTreePluginOpenStackUnregister: {Default: false, PreRelease: featuregate.Alpha}, + VolumeSubpath: {Default: true, PreRelease: featuregate.GA}, + ConfigurableFSGroupPolicy: {Default: true, PreRelease: featuregate.Beta}, + BalanceAttachedNodeVolumes: {Default: false, PreRelease: featuregate.Alpha}, + CSIBlockVolume: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.20 + CSIInlineVolume: {Default: true, PreRelease: featuregate.Beta}, + CSIStorageCapacity: {Default: false, PreRelease: featuregate.Alpha}, + CSIServiceAccountToken: {Default: false, PreRelease: featuregate.Alpha}, + GenericEphemeralVolume: {Default: false, PreRelease: featuregate.Alpha}, + CSIVolumeFSGroupPolicy: {Default: true, PreRelease: featuregate.Beta}, + RuntimeClass: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.23 + NodeLease: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, + SCTPSupport: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.22 + VolumeSnapshotDataSource: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.21 + ProcMountType: {Default: false, PreRelease: featuregate.Alpha}, + TTLAfterFinished: {Default: false, PreRelease: featuregate.Alpha}, + KubeletPodResources: {Default: true, PreRelease: featuregate.Beta}, + WindowsGMSA: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.20 + WindowsRunAsUserName: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.20 LocalStorageCapacityIsolationFSQuotaMonitoring: {Default: false, PreRelease: featuregate.Alpha}, NonPreemptingPriority: {Default: true, PreRelease: featuregate.Beta}, PodOverhead: {Default: true, PreRelease: featuregate.Beta}, diff --git a/pkg/volume/csimigration/plugin_manager.go b/pkg/volume/csimigration/plugin_manager.go index 1d635166aed..d7ca1db228d 100644 --- a/pkg/volume/csimigration/plugin_manager.go +++ b/pkg/volume/csimigration/plugin_manager.go @@ -20,7 +20,7 @@ import ( "errors" "fmt" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/component-base/featuregate" csilibplugins "k8s.io/csi-translation-lib/plugins" @@ -48,28 +48,29 @@ func NewPluginManager(m PluginNameMapper) PluginManager { } // IsMigrationCompleteForPlugin indicates whether CSI migration has been completed -// for a particular storage plugin +// for a particular storage plugin. A complete migration will need to: +// 1. Enable CSIMigrationXX for the plugin +// 2. Unregister the in-tree plugin by setting the InTreePluginXXUnregister feature gate func (pm PluginManager) IsMigrationCompleteForPlugin(pluginName string) bool { - // CSIMigration feature and plugin specific migration feature flags should - // be enabled for plugin specific migration completion feature flags to be - // take effect + // CSIMigration feature and plugin specific InTreePluginUnregister feature flags should + // be enabled for plugin specific migration completion to be take effect if !pm.IsMigrationEnabledForPlugin(pluginName) { return false } switch pluginName { case csilibplugins.AWSEBSInTreePluginName: - return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationAWSComplete) + return utilfeature.DefaultFeatureGate.Enabled(features.InTreePluginAWSUnregister) case csilibplugins.GCEPDInTreePluginName: - return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationGCEComplete) + return utilfeature.DefaultFeatureGate.Enabled(features.InTreePluginGCEUnregister) case csilibplugins.AzureFileInTreePluginName: - return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationAzureFileComplete) + return utilfeature.DefaultFeatureGate.Enabled(features.InTreePluginAzureFileUnregister) case csilibplugins.AzureDiskInTreePluginName: - return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationAzureDiskComplete) + return utilfeature.DefaultFeatureGate.Enabled(features.InTreePluginAzureDiskUnregister) case csilibplugins.CinderInTreePluginName: - return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationOpenStackComplete) + return utilfeature.DefaultFeatureGate.Enabled(features.InTreePluginOpenStackUnregister) case csilibplugins.VSphereInTreePluginName: - return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationvSphereComplete) + return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationvSphereComplete) || utilfeature.DefaultFeatureGate.Enabled(features.InTreePluginvSphereUnregister) default: return false } @@ -148,13 +149,25 @@ func TranslateInTreeSpecToCSI(spec *volume.Spec, translator InTreeToCSITranslato } // CheckMigrationFeatureFlags checks the configuration of feature flags related -// to CSI Migration is valid -func CheckMigrationFeatureFlags(f featuregate.FeatureGate, pluginMigration, pluginMigrationComplete featuregate.Feature) error { +// to CSI Migration is valid. It will return whether the migration is complete +// by looking up the pluginMigrationComplete and pluginUnregister flag +func CheckMigrationFeatureFlags(f featuregate.FeatureGate, pluginMigration, + pluginMigrationComplete, pluginUnregister featuregate.Feature) (migrationComplete bool, err error) { if f.Enabled(pluginMigration) && !f.Enabled(features.CSIMigration) { - return fmt.Errorf("enabling %q requires CSIMigration to be enabled", pluginMigration) + return false, fmt.Errorf("enabling %q requires CSIMigration to be enabled", pluginMigration) } - if f.Enabled(pluginMigrationComplete) && !f.Enabled(pluginMigration) { - return fmt.Errorf("enabling %q requires %q to be enabled", pluginMigrationComplete, pluginMigration) + // TODO: Remove the following two checks once the CSIMigrationXXComplete flag is removed + if pluginMigrationComplete != "" && f.Enabled(pluginMigrationComplete) && !f.Enabled(pluginMigration) { + return false, fmt.Errorf("enabling %q requires %q to be enabled", pluginMigrationComplete, pluginMigration) } - return nil + // This is only needed for vSphere since we will deprecate the CSIMigrationvSphereComplete flag soon + if pluginMigrationComplete != "" && f.Enabled(features.CSIMigration) && + f.Enabled(pluginMigration) && f.Enabled(pluginMigrationComplete) { + return true, nil + } + // This is for other in-tree plugin that get migration finished + if f.Enabled(pluginMigration) && f.Enabled(pluginUnregister) { + return true, nil + } + return false, nil } diff --git a/pkg/volume/csimigration/plugin_manager_test.go b/pkg/volume/csimigration/plugin_manager_test.go index 51621db9896..cad4d5c1029 100644 --- a/pkg/volume/csimigration/plugin_manager_test.go +++ b/pkg/volume/csimigration/plugin_manager_test.go @@ -20,7 +20,7 @@ import ( "fmt" "testing" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/component-base/featuregate" featuregatetesting "k8s.io/component-base/featuregate/testing" @@ -148,160 +148,191 @@ func TestCheckMigrationFeatureFlags(t *testing.T) { csiMigrationEnabled bool pluginFeatureComplete featuregate.Feature pluginFeatureCompleteEnabled bool - result bool + pluginUnregsiterFeature featuregate.Feature + pluginUnregsiterEnabled bool + expectMigrationComplete bool + expectErr bool }{ { name: "plugin specific feature flag enabled with migration flag disabled", - pluginFeature: features.CSIMigrationGCE, + pluginFeature: features.CSIMigrationvSphere, pluginFeatureEnabled: true, csiMigrationEnabled: false, - pluginFeatureComplete: features.CSIMigrationGCEComplete, + pluginFeatureComplete: features.CSIMigrationvSphereComplete, pluginFeatureCompleteEnabled: false, - result: false, + pluginUnregsiterFeature: features.InTreePluginvSphereUnregister, + pluginUnregsiterEnabled: false, + expectMigrationComplete: false, + expectErr: false, }, { name: "plugin specific complete flag enabled but plugin specific feature flag disabled", - pluginFeature: features.CSIMigrationAWS, + pluginFeature: features.CSIMigrationvSphere, pluginFeatureEnabled: false, csiMigrationEnabled: true, - pluginFeatureComplete: features.CSIMigrationAWSComplete, + pluginFeatureComplete: features.CSIMigrationvSphereComplete, pluginFeatureCompleteEnabled: true, - result: false, + pluginUnregsiterFeature: features.InTreePluginvSphereUnregister, + pluginUnregsiterEnabled: false, + expectMigrationComplete: false, + expectErr: false, }, { name: "plugin specific complete feature disabled but plugin specific migration feature and CSI migration flag enabled", - pluginFeature: features.CSIMigrationGCE, + pluginFeature: features.CSIMigrationvSphere, pluginFeatureEnabled: true, csiMigrationEnabled: true, - pluginFeatureComplete: features.CSIMigrationGCEComplete, + pluginFeatureComplete: features.CSIMigrationvSphereComplete, pluginFeatureCompleteEnabled: false, - result: true, + pluginUnregsiterFeature: features.InTreePluginvSphereUnregister, + pluginUnregsiterEnabled: false, + expectMigrationComplete: false, + expectErr: true, }, { name: "all features enabled", - pluginFeature: features.CSIMigrationAWS, + pluginFeature: features.CSIMigrationvSphere, pluginFeatureEnabled: true, csiMigrationEnabled: true, - pluginFeatureComplete: features.CSIMigrationAWSComplete, + pluginFeatureComplete: features.CSIMigrationvSphereComplete, pluginFeatureCompleteEnabled: true, - result: true, + pluginUnregsiterFeature: features.InTreePluginvSphereUnregister, + pluginUnregsiterEnabled: true, + expectMigrationComplete: true, + expectErr: true, + }, + { + name: "no plugin feature complete set", + pluginFeature: features.CSIMigrationGCE, + pluginFeatureEnabled: true, + csiMigrationEnabled: true, + pluginUnregsiterFeature: features.InTreePluginGCEUnregister, + pluginUnregsiterEnabled: true, + expectMigrationComplete: true, + expectErr: true, }, } for _, test := range testCases { t.Run(fmt.Sprintf("Testing %v", test.name), func(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIMigration, test.csiMigrationEnabled)() defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, test.pluginFeature, test.pluginFeatureEnabled)() - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, test.pluginFeatureComplete, test.pluginFeatureCompleteEnabled)() - err := CheckMigrationFeatureFlags(utilfeature.DefaultFeatureGate, test.pluginFeature, test.pluginFeatureComplete) - if err != nil && test.result == true { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, test.pluginUnregsiterFeature, test.pluginUnregsiterEnabled)() + if test.pluginFeatureComplete != "" { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, test.pluginFeatureComplete, test.pluginFeatureCompleteEnabled)() + } + migrationComplete, err := CheckMigrationFeatureFlags(utilfeature.DefaultFeatureGate, test.pluginFeature, test.pluginFeatureComplete, test.pluginUnregsiterFeature) + if err != nil && test.expectErr == true { t.Errorf("Unexpected error: %v", err) } - if err == nil && test.result == false { + if err == nil && test.expectErr == false { t.Errorf("Unexpected validation pass") } + if migrationComplete != test.expectMigrationComplete { + t.Errorf("Unexpected migrationComplete result. Exp: %v, got %v", test.expectMigrationComplete, migrationComplete) + } }) } } func TestMigrationFeatureFlagStatus(t *testing.T) { testCases := []struct { - name string - pluginName string - csiMigrationEnabled bool - pluginFeature featuregate.Feature - pluginFeatureEnabled bool - pluginFeatureComplete featuregate.Feature - pluginFeatureCompleteEnabled bool - csiMigrationResult bool - csiMigrationCompleteResult bool + name string + pluginName string + csiMigrationEnabled bool + pluginFeature featuregate.Feature + pluginFeatureEnabled bool + inTreePluginUnregister featuregate.Feature + inTreePluginUnregisterEnabled bool + csiMigrationResult bool + csiMigrationCompleteResult bool }{ { - name: "gce-pd migration flag disabled and migration-complete flag disabled with CSI migration flag disabled", - pluginName: "kubernetes.io/gce-pd", - pluginFeature: features.CSIMigrationGCE, - pluginFeatureEnabled: false, - csiMigrationEnabled: false, - pluginFeatureComplete: features.CSIMigrationGCEComplete, - pluginFeatureCompleteEnabled: false, - csiMigrationResult: false, - csiMigrationCompleteResult: false, + name: "gce-pd migration flag disabled and migration-complete flag disabled with CSI migration flag disabled", + pluginName: "kubernetes.io/gce-pd", + pluginFeature: features.CSIMigrationGCE, + pluginFeatureEnabled: false, + csiMigrationEnabled: false, + inTreePluginUnregister: features.InTreePluginGCEUnregister, + inTreePluginUnregisterEnabled: false, + csiMigrationResult: false, + csiMigrationCompleteResult: false, }, { - name: "gce-pd migration flag disabled and migration-complete flag disabled with CSI migration flag enabled", - pluginName: "kubernetes.io/gce-pd", - pluginFeature: features.CSIMigrationGCE, - pluginFeatureEnabled: false, - csiMigrationEnabled: true, - pluginFeatureComplete: features.CSIMigrationGCEComplete, - pluginFeatureCompleteEnabled: false, - csiMigrationResult: false, - csiMigrationCompleteResult: false, + name: "gce-pd migration flag disabled and migration-complete flag disabled with CSI migration flag enabled", + pluginName: "kubernetes.io/gce-pd", + pluginFeature: features.CSIMigrationGCE, + pluginFeatureEnabled: false, + csiMigrationEnabled: true, + inTreePluginUnregister: features.InTreePluginGCEUnregister, + inTreePluginUnregisterEnabled: false, + csiMigrationResult: false, + csiMigrationCompleteResult: false, }, { - name: "gce-pd migration flag enabled and migration-complete flag disabled with CSI migration flag enabled", - pluginName: "kubernetes.io/gce-pd", - pluginFeature: features.CSIMigrationGCE, - pluginFeatureEnabled: true, - csiMigrationEnabled: true, - pluginFeatureComplete: features.CSIMigrationGCEComplete, - pluginFeatureCompleteEnabled: false, - csiMigrationResult: true, - csiMigrationCompleteResult: false, + name: "gce-pd migration flag enabled and migration-complete flag disabled with CSI migration flag enabled", + pluginName: "kubernetes.io/gce-pd", + pluginFeature: features.CSIMigrationGCE, + pluginFeatureEnabled: true, + csiMigrationEnabled: true, + inTreePluginUnregister: features.InTreePluginGCEUnregister, + inTreePluginUnregisterEnabled: false, + csiMigrationResult: true, + csiMigrationCompleteResult: false, }, { - name: "gce-pd migration flag enabled and migration-complete flag enabled with CSI migration flag enabled", - pluginName: "kubernetes.io/gce-pd", - pluginFeature: features.CSIMigrationGCE, - pluginFeatureEnabled: true, - csiMigrationEnabled: true, - pluginFeatureComplete: features.CSIMigrationGCEComplete, - pluginFeatureCompleteEnabled: true, - csiMigrationResult: true, - csiMigrationCompleteResult: true, + name: "gce-pd migration flag enabled and migration-complete flag enabled with CSI migration flag enabled", + pluginName: "kubernetes.io/gce-pd", + pluginFeature: features.CSIMigrationGCE, + pluginFeatureEnabled: true, + csiMigrationEnabled: true, + inTreePluginUnregister: features.InTreePluginGCEUnregister, + inTreePluginUnregisterEnabled: true, + csiMigrationResult: true, + csiMigrationCompleteResult: true, }, { - name: "aws-ebs migration flag disabled and migration-complete flag disabled with CSI migration flag disabled", - pluginName: "kubernetes.io/aws-ebs", - pluginFeature: features.CSIMigrationAWS, - pluginFeatureEnabled: false, - csiMigrationEnabled: false, - pluginFeatureComplete: features.CSIMigrationAWSComplete, - pluginFeatureCompleteEnabled: false, - csiMigrationResult: false, - csiMigrationCompleteResult: false, + name: "aws-ebs migration flag disabled and migration-complete flag disabled with CSI migration flag disabled", + pluginName: "kubernetes.io/aws-ebs", + pluginFeature: features.CSIMigrationAWS, + pluginFeatureEnabled: false, + csiMigrationEnabled: false, + inTreePluginUnregister: features.InTreePluginAWSUnregister, + inTreePluginUnregisterEnabled: false, + csiMigrationResult: false, + csiMigrationCompleteResult: false, }, { - name: "aws-ebs migration flag disabled and migration-complete flag disabled with CSI migration flag enabled", - pluginName: "kubernetes.io/aws-ebs", - pluginFeature: features.CSIMigrationAWS, - pluginFeatureEnabled: false, - csiMigrationEnabled: true, - pluginFeatureComplete: features.CSIMigrationAWSComplete, - pluginFeatureCompleteEnabled: false, - csiMigrationResult: false, - csiMigrationCompleteResult: false, + name: "aws-ebs migration flag disabled and migration-complete flag disabled with CSI migration flag enabled", + pluginName: "kubernetes.io/aws-ebs", + pluginFeature: features.CSIMigrationAWS, + pluginFeatureEnabled: false, + csiMigrationEnabled: true, + inTreePluginUnregister: features.InTreePluginAWSUnregister, + inTreePluginUnregisterEnabled: false, + csiMigrationResult: false, + csiMigrationCompleteResult: false, }, { - name: "aws-ebs migration flag enabled and migration-complete flag disabled with CSI migration flag enabled", - pluginName: "kubernetes.io/aws-ebs", - pluginFeature: features.CSIMigrationAWS, - pluginFeatureEnabled: true, - csiMigrationEnabled: true, - pluginFeatureComplete: features.CSIMigrationAWSComplete, - pluginFeatureCompleteEnabled: false, - csiMigrationResult: true, - csiMigrationCompleteResult: false, + name: "aws-ebs migration flag enabled and migration-complete flag disabled with CSI migration flag enabled", + pluginName: "kubernetes.io/aws-ebs", + pluginFeature: features.CSIMigrationAWS, + pluginFeatureEnabled: true, + csiMigrationEnabled: true, + inTreePluginUnregister: features.InTreePluginAWSUnregister, + inTreePluginUnregisterEnabled: false, + csiMigrationResult: true, + csiMigrationCompleteResult: false, }, { - name: "aws-ebs migration flag enabled and migration-complete flag enabled with CSI migration flag enabled", - pluginName: "kubernetes.io/aws-ebs", - pluginFeature: features.CSIMigrationAWS, - pluginFeatureEnabled: true, - csiMigrationEnabled: true, - pluginFeatureComplete: features.CSIMigrationAWSComplete, - pluginFeatureCompleteEnabled: true, - csiMigrationResult: true, - csiMigrationCompleteResult: true, + name: "aws-ebs migration flag enabled and migration-complete flag enabled with CSI migration flag enabled", + pluginName: "kubernetes.io/aws-ebs", + pluginFeature: features.CSIMigrationAWS, + pluginFeatureEnabled: true, + csiMigrationEnabled: true, + inTreePluginUnregister: features.InTreePluginAWSUnregister, + inTreePluginUnregisterEnabled: true, + csiMigrationResult: true, + csiMigrationCompleteResult: true, }, } csiTranslator := csitrans.New() @@ -310,7 +341,7 @@ func TestMigrationFeatureFlagStatus(t *testing.T) { t.Run(fmt.Sprintf("Testing %v", test.name), func(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIMigration, test.csiMigrationEnabled)() defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, test.pluginFeature, test.pluginFeatureEnabled)() - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, test.pluginFeatureComplete, test.pluginFeatureCompleteEnabled)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, test.inTreePluginUnregister, test.inTreePluginUnregisterEnabled)() csiMigrationResult := pm.IsMigrationEnabledForPlugin(test.pluginName) if csiMigrationResult != test.csiMigrationResult {