diff --git a/cmd/kube-controller-manager/app/plugins_providers.go b/cmd/kube-controller-manager/app/plugins_providers.go index 093e236dc23..915e1b715cb 100644 --- a/cmd/kube-controller-manager/app/plugins_providers.go +++ b/cmd/kube-controller-manager/app/plugins_providers.go @@ -35,40 +35,48 @@ import ( type probeFn func() []volume.VolumePlugin -func appendPluginBasedOnMigrationFeatureFlags(plugins []volume.VolumePlugin, inTreePluginName string, featureGate featuregate.FeatureGate, pluginMigration, pluginMigrationComplete featuregate.Feature, fn probeFn) ([]volume.VolumePlugin, error) { +func appendPluginBasedOnFeatureFlags(plugins []volume.VolumePlugin, inTreePluginName string, featureGate featuregate.FeatureGate, pluginInfo pluginInfo) ([]volume.VolumePlugin, error) { // Skip appending the in-tree plugin to the list of plugins to be probed/initialized // if the CSIMigration feature flag and plugin specific feature flag indicating // CSI migration is complete - err := csimigration.CheckMigrationFeatureFlags(featureGate, pluginMigration, pluginMigrationComplete) + migrationComplete, err := csimigration.CheckMigrationFeatureFlags(featureGate, pluginInfo.pluginMigrationFeature, + pluginInfo.pluginMigrationCompleteFeature, pluginInfo.pluginUnregisterFeature) if err != nil { klog.Warningf("Unexpected CSI Migration Feature Flags combination detected: %v. CSI Migration may not take effect", err) // TODO: fail and return here once alpha only tests can set the feature flags for a plugin correctly } - if featureGate.Enabled(features.CSIMigration) && featureGate.Enabled(pluginMigration) && featureGate.Enabled(pluginMigrationComplete) { - klog.Infof("Skip registration of plugin %s since feature flag %v is enabled", inTreePluginName, pluginMigrationComplete) + // TODO: This can be removed after feature flag CSIMigrationvSphereComplete is removed. + if migrationComplete { + klog.Infof("Skip registration of plugin %s since migration is complete", inTreePluginName) return plugins, nil } - plugins = append(plugins, fn()...) + if featureGate.Enabled(pluginInfo.pluginUnregisterFeature) { + klog.Infof("Skip registration of plugin %s since feature flag %v is enabled", inTreePluginName, pluginInfo.pluginUnregisterFeature) + return plugins, nil + } + plugins = append(plugins, pluginInfo.pluginProbeFunction()...) return plugins, nil } type pluginInfo struct { - pluginMigrationFeature featuregate.Feature + pluginMigrationFeature featuregate.Feature + // deprecated, only to keep here for vSphere pluginMigrationCompleteFeature featuregate.Feature + pluginUnregisterFeature featuregate.Feature pluginProbeFunction probeFn } func appendAttachableLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) { pluginMigrationStatus := make(map[string]pluginInfo) - pluginMigrationStatus[plugins.AWSEBSInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAWS, pluginMigrationCompleteFeature: features.CSIMigrationAWSComplete, pluginProbeFunction: awsebs.ProbeVolumePlugins} - pluginMigrationStatus[plugins.GCEPDInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationGCE, pluginMigrationCompleteFeature: features.CSIMigrationGCEComplete, pluginProbeFunction: gcepd.ProbeVolumePlugins} - pluginMigrationStatus[plugins.CinderInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationOpenStack, pluginMigrationCompleteFeature: features.CSIMigrationOpenStackComplete, pluginProbeFunction: cinder.ProbeVolumePlugins} - pluginMigrationStatus[plugins.AzureDiskInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureDisk, pluginMigrationCompleteFeature: features.CSIMigrationAzureDiskComplete, pluginProbeFunction: azuredd.ProbeVolumePlugins} - pluginMigrationStatus[plugins.VSphereInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationvSphere, pluginMigrationCompleteFeature: features.CSIMigrationvSphereComplete, pluginProbeFunction: vsphere_volume.ProbeVolumePlugins} + pluginMigrationStatus[plugins.AWSEBSInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAWS, pluginUnregisterFeature: features.InTreePluginAWSUnregister, pluginProbeFunction: awsebs.ProbeVolumePlugins} + pluginMigrationStatus[plugins.GCEPDInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationGCE, pluginUnregisterFeature: features.InTreePluginGCEUnregister, pluginProbeFunction: gcepd.ProbeVolumePlugins} + pluginMigrationStatus[plugins.CinderInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationOpenStack, pluginUnregisterFeature: features.InTreePluginOpenStackUnregister, pluginProbeFunction: cinder.ProbeVolumePlugins} + pluginMigrationStatus[plugins.AzureDiskInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureDisk, pluginUnregisterFeature: features.InTreePluginAzureDiskUnregister, pluginProbeFunction: azuredd.ProbeVolumePlugins} + pluginMigrationStatus[plugins.VSphereInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationvSphere, pluginMigrationCompleteFeature: features.CSIMigrationvSphereComplete, pluginUnregisterFeature: features.InTreePluginvSphereUnregister, pluginProbeFunction: vsphere_volume.ProbeVolumePlugins} var err error for pluginName, pluginInfo := range pluginMigrationStatus { - allPlugins, err = appendPluginBasedOnMigrationFeatureFlags(allPlugins, pluginName, featureGate, pluginInfo.pluginMigrationFeature, pluginInfo.pluginMigrationCompleteFeature, pluginInfo.pluginProbeFunction) + allPlugins, err = appendPluginBasedOnFeatureFlags(allPlugins, pluginName, featureGate, pluginInfo) if err != nil { return allPlugins, err } @@ -81,20 +89,24 @@ func appendExpandableLegacyProviderVolumes(allPlugins []volume.VolumePlugin, fea } func appendLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) { - pluginMigrationStatus := make(map[string]pluginInfo) - pluginMigrationStatus[plugins.AWSEBSInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAWS, pluginMigrationCompleteFeature: features.CSIMigrationAWSComplete, pluginProbeFunction: awsebs.ProbeVolumePlugins} - pluginMigrationStatus[plugins.GCEPDInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationGCE, pluginMigrationCompleteFeature: features.CSIMigrationGCEComplete, pluginProbeFunction: gcepd.ProbeVolumePlugins} - pluginMigrationStatus[plugins.CinderInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationOpenStack, pluginMigrationCompleteFeature: features.CSIMigrationOpenStackComplete, pluginProbeFunction: cinder.ProbeVolumePlugins} - pluginMigrationStatus[plugins.AzureDiskInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureDisk, pluginMigrationCompleteFeature: features.CSIMigrationAzureDiskComplete, pluginProbeFunction: azuredd.ProbeVolumePlugins} - pluginMigrationStatus[plugins.AzureFileInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureFile, pluginMigrationCompleteFeature: features.CSIMigrationAzureFileComplete, pluginProbeFunction: azure_file.ProbeVolumePlugins} - pluginMigrationStatus[plugins.VSphereInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationvSphere, pluginMigrationCompleteFeature: features.CSIMigrationvSphereComplete, pluginProbeFunction: vsphere_volume.ProbeVolumePlugins} - var err error - for pluginName, pluginInfo := range pluginMigrationStatus { - allPlugins, err = appendPluginBasedOnMigrationFeatureFlags(allPlugins, pluginName, featureGate, pluginInfo.pluginMigrationFeature, pluginInfo.pluginMigrationCompleteFeature, pluginInfo.pluginProbeFunction) - if err != nil { - return allPlugins, err - } + // First append attachable volumes + allPlugins, err = appendAttachableLegacyProviderVolumes(allPlugins, featureGate) + if err != nil { + return allPlugins, err } + + // Then append non-attachable volumes + pluginName := plugins.AzureFileInTreePluginName + pluginInfo := pluginInfo{ + pluginMigrationFeature: features.CSIMigrationAzureFile, + pluginUnregisterFeature: features.InTreePluginAzureFileUnregister, + pluginProbeFunction: azure_file.ProbeVolumePlugins, + } + allPlugins, err = appendPluginBasedOnFeatureFlags(allPlugins, pluginName, featureGate, pluginInfo) + if err != nil { + return allPlugins, err + } + return allPlugins, nil } diff --git a/cmd/kubelet/app/plugins_providers.go b/cmd/kubelet/app/plugins_providers.go index 9f8ce86ff0b..bd71645de23 100644 --- a/cmd/kubelet/app/plugins_providers.go +++ b/cmd/kubelet/app/plugins_providers.go @@ -35,41 +35,51 @@ import ( type probeFn func() []volume.VolumePlugin -func appendPluginBasedOnMigrationFeatureFlags(plugins []volume.VolumePlugin, inTreePluginName string, featureGate featuregate.FeatureGate, pluginMigration, pluginMigrationComplete featuregate.Feature, fn probeFn) ([]volume.VolumePlugin, error) { +func appendPluginBasedOnFeatureFlags(plugins []volume.VolumePlugin, inTreePluginName string, + featureGate featuregate.FeatureGate, pluginInfo pluginInfo) ([]volume.VolumePlugin, error) { // Skip appending the in-tree plugin to the list of plugins to be probed/initialized // if the CSIMigration feature flag and plugin specific feature flag indicating // CSI migration is complete - err := csimigration.CheckMigrationFeatureFlags(featureGate, pluginMigration, pluginMigrationComplete) + migrationComplete, err := csimigration.CheckMigrationFeatureFlags(featureGate, pluginInfo.pluginMigrationFeature, + pluginInfo.pluginMigrationCompleteFeature, pluginInfo.pluginUnregisterFeature) if err != nil { klog.Warningf("Unexpected CSI Migration Feature Flags combination detected: %v. CSI Migration may not take effect", err) // TODO: fail and return here once alpha only tests can set the feature flags for a plugin correctly } - if featureGate.Enabled(features.CSIMigration) && featureGate.Enabled(pluginMigration) && featureGate.Enabled(pluginMigrationComplete) { - klog.Infof("Skip registration of plugin %s since feature flag %v is enabled", inTreePluginName, pluginMigrationComplete) + // TODO: This can be removed after feature flag CSIMigrationvSphereComplete is removed. + if migrationComplete { + klog.Infof("Skip registration of plugin %s since migration is complete", inTreePluginName) return plugins, nil } - plugins = append(plugins, fn()...) + if featureGate.Enabled(pluginInfo.pluginUnregisterFeature) { + klog.Infof("Skip registration of plugin %s since feature flag %v is enabled", inTreePluginName, pluginInfo.pluginUnregisterFeature) + return plugins, nil + } + + plugins = append(plugins, pluginInfo.pluginProbeFunction()...) return plugins, nil } type pluginInfo struct { - pluginMigrationFeature featuregate.Feature + pluginMigrationFeature featuregate.Feature + // deprecated, only to keep here for vSphere pluginMigrationCompleteFeature featuregate.Feature + pluginUnregisterFeature featuregate.Feature pluginProbeFunction probeFn } func appendLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) { pluginMigrationStatus := make(map[string]pluginInfo) - pluginMigrationStatus[plugins.AWSEBSInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAWS, pluginMigrationCompleteFeature: features.CSIMigrationAWSComplete, pluginProbeFunction: awsebs.ProbeVolumePlugins} - pluginMigrationStatus[plugins.GCEPDInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationGCE, pluginMigrationCompleteFeature: features.CSIMigrationGCEComplete, pluginProbeFunction: gcepd.ProbeVolumePlugins} - pluginMigrationStatus[plugins.CinderInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationOpenStack, pluginMigrationCompleteFeature: features.CSIMigrationOpenStackComplete, pluginProbeFunction: cinder.ProbeVolumePlugins} - pluginMigrationStatus[plugins.AzureDiskInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureDisk, pluginMigrationCompleteFeature: features.CSIMigrationAzureDiskComplete, pluginProbeFunction: azuredd.ProbeVolumePlugins} - pluginMigrationStatus[plugins.AzureFileInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureFile, pluginMigrationCompleteFeature: features.CSIMigrationAzureFileComplete, pluginProbeFunction: azure_file.ProbeVolumePlugins} - pluginMigrationStatus[plugins.VSphereInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationvSphere, pluginMigrationCompleteFeature: features.CSIMigrationvSphereComplete, pluginProbeFunction: vsphere_volume.ProbeVolumePlugins} + pluginMigrationStatus[plugins.AWSEBSInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAWS, pluginUnregisterFeature: features.InTreePluginAWSUnregister, pluginProbeFunction: awsebs.ProbeVolumePlugins} + pluginMigrationStatus[plugins.GCEPDInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationGCE, pluginUnregisterFeature: features.InTreePluginGCEUnregister, pluginProbeFunction: gcepd.ProbeVolumePlugins} + pluginMigrationStatus[plugins.CinderInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationOpenStack, pluginUnregisterFeature: features.InTreePluginOpenStackUnregister, pluginProbeFunction: cinder.ProbeVolumePlugins} + pluginMigrationStatus[plugins.AzureDiskInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureDisk, pluginUnregisterFeature: features.InTreePluginAzureDiskUnregister, pluginProbeFunction: azuredd.ProbeVolumePlugins} + pluginMigrationStatus[plugins.AzureFileInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureFile, pluginUnregisterFeature: features.InTreePluginAzureFileUnregister, pluginProbeFunction: azure_file.ProbeVolumePlugins} + pluginMigrationStatus[plugins.VSphereInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationvSphere, pluginMigrationCompleteFeature: features.CSIMigrationvSphereComplete, pluginUnregisterFeature: features.InTreePluginvSphereUnregister, pluginProbeFunction: vsphere_volume.ProbeVolumePlugins} var err error for pluginName, pluginInfo := range pluginMigrationStatus { - allPlugins, err = appendPluginBasedOnMigrationFeatureFlags(allPlugins, pluginName, featureGate, pluginInfo.pluginMigrationFeature, pluginInfo.pluginMigrationCompleteFeature, pluginInfo.pluginProbeFunction) + allPlugins, err = appendPluginBasedOnFeatureFlags(allPlugins, pluginName, featureGate, pluginInfo) if err != nil { return allPlugins, err } diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 0f51df87b0c..ddba2f4ccce 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -328,12 +328,11 @@ const ( // Enables the GCE PD in-tree driver to GCE CSI Driver migration feature. CSIMigrationGCE featuregate.Feature = "CSIMigrationGCE" - // owner: @davidz627 - // alpha: v1.17 + // owner: @Jiawei0227 + // alpha: v1.21 // // Disables the GCE PD in-tree driver. - // Expects GCE PD CSI Driver to be installed and configured on all nodes. - CSIMigrationGCEComplete featuregate.Feature = "CSIMigrationGCEComplete" + InTreePluginGCEUnregister featuregate.Feature = "InTreePluginGCEUnregister" // owner: @leakingtapan // alpha: v1.14 @@ -343,11 +342,10 @@ const ( CSIMigrationAWS featuregate.Feature = "CSIMigrationAWS" // owner: @leakingtapan - // alpha: v1.17 + // alpha: v1.21 // // Disables the AWS EBS in-tree driver. - // Expects AWS EBS CSI Driver to be installed and configured on all nodes. - CSIMigrationAWSComplete featuregate.Feature = "CSIMigrationAWSComplete" + InTreePluginAWSUnregister featuregate.Feature = "InTreePluginAWSUnregister" // owner: @andyzhangx // alpha: v1.15 @@ -357,11 +355,10 @@ const ( CSIMigrationAzureDisk featuregate.Feature = "CSIMigrationAzureDisk" // owner: @andyzhangx - // alpha: v1.17 + // alpha: v1.21 // // Disables the Azure Disk in-tree driver. - // Expects Azure Disk CSI Driver to be installed and configured on all nodes. - CSIMigrationAzureDiskComplete featuregate.Feature = "CSIMigrationAzureDiskComplete" + InTreePluginAzureDiskUnregister featuregate.Feature = "InTreePluginAzureDiskUnregister" // owner: @andyzhangx // alpha: v1.15 @@ -370,11 +367,10 @@ const ( CSIMigrationAzureFile featuregate.Feature = "CSIMigrationAzureFile" // owner: @andyzhangx - // alpha: v1.17 + // alpha: v1.21 // // Disables the Azure File in-tree driver. - // Expects Azure File CSI Driver to be installed and configured on all nodes. - CSIMigrationAzureFileComplete featuregate.Feature = "CSIMigrationAzureFileComplete" + InTreePluginAzureFileUnregister featuregate.Feature = "InTreePluginAzureFileUnregister" // owner: @divyenpatel // beta: v1.19 (requires: vSphere vCenter/ESXi Version: 7.0u1, HW Version: VM version 15) @@ -389,6 +385,25 @@ const ( // Expects vSphere CSI Driver to be installed and configured on all nodes. CSIMigrationvSphereComplete featuregate.Feature = "CSIMigrationvSphereComplete" + // owner: @divyenpatel + // alpha: v1.21 + // + // Disables the vSphere in-tree driver. + InTreePluginvSphereUnregister featuregate.Feature = "InTreePluginvSphereUnregister" + + // owner: @adisky + // alpha: v1.14 + // beta: v1.18 + // + // Enables the OpenStack Cinder in-tree driver to OpenStack Cinder CSI Driver migration feature. + CSIMigrationOpenStack featuregate.Feature = "CSIMigrationOpenStack" + + // owner: @adisky + // alpha: v1.21 + // + // Disables the OpenStack Cinder in-tree driver. + InTreePluginOpenStackUnregister featuregate.Feature = "InTreePluginOpenStackUnregister" + // owner: @huffmanca // alpha: v1.19 // beta: v1.20 @@ -425,20 +440,6 @@ const ( // Enables support for running container entrypoints as different usernames than their default ones. WindowsRunAsUserName featuregate.Feature = "WindowsRunAsUserName" - // owner: @adisky - // alpha: v1.14 - // beta: v1.18 - // - // Enables the OpenStack Cinder in-tree driver to OpenStack Cinder CSI Driver migration feature. - CSIMigrationOpenStack featuregate.Feature = "CSIMigrationOpenStack" - - // owner: @adisky - // alpha: v1.17 - // - // Disables the OpenStack Cinder in-tree driver. - // Expects the OpenStack Cinder CSI Driver to be installed and configured on all nodes. - CSIMigrationOpenStackComplete featuregate.Feature = "CSIMigrationOpenStackComplete" - // owner: @RobertKrawitz // alpha: v1.15 // @@ -683,60 +684,61 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS AppArmor: {Default: true, PreRelease: featuregate.Beta}, DynamicKubeletConfig: {Default: true, PreRelease: featuregate.Beta}, ExperimentalHostUserNamespaceDefaultingGate: {Default: false, PreRelease: featuregate.Beta}, - DevicePlugins: {Default: true, PreRelease: featuregate.Beta}, - RotateKubeletServerCertificate: {Default: true, PreRelease: featuregate.Beta}, - LocalStorageCapacityIsolation: {Default: true, PreRelease: featuregate.Beta}, - Sysctls: {Default: true, PreRelease: featuregate.Beta}, - EphemeralContainers: {Default: false, PreRelease: featuregate.Alpha}, - QOSReserved: {Default: false, PreRelease: featuregate.Alpha}, - ExpandPersistentVolumes: {Default: true, PreRelease: featuregate.Beta}, - ExpandInUsePersistentVolumes: {Default: true, PreRelease: featuregate.Beta}, - ExpandCSIVolumes: {Default: true, PreRelease: featuregate.Beta}, - CPUManager: {Default: true, PreRelease: featuregate.Beta}, - CPUCFSQuotaPeriod: {Default: false, PreRelease: featuregate.Alpha}, - TopologyManager: {Default: true, PreRelease: featuregate.Beta}, - ServiceNodeExclusion: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.22 - NodeDisruptionExclusion: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.22 - CSIDriverRegistry: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.20 - BlockVolume: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.20 - StorageObjectInUseProtection: {Default: true, PreRelease: featuregate.GA}, - SupportPodPidsLimit: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.23 - SupportNodePidsLimit: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.23 - BoundServiceAccountTokenVolume: {Default: false, PreRelease: featuregate.Alpha}, - ServiceAccountIssuerDiscovery: {Default: true, PreRelease: featuregate.Beta}, - CRIContainerLogRotation: {Default: true, PreRelease: featuregate.Beta}, - CSIMigration: {Default: true, PreRelease: featuregate.Beta}, - CSIMigrationGCE: {Default: false, PreRelease: featuregate.Beta}, // Off by default (requires GCE PD CSI Driver) - CSIMigrationGCEComplete: {Default: false, PreRelease: featuregate.Alpha}, - CSIMigrationAWS: {Default: false, PreRelease: featuregate.Beta}, // Off by default (requires AWS EBS CSI driver) - CSIMigrationAWSComplete: {Default: false, PreRelease: featuregate.Alpha}, - CSIMigrationAzureDisk: {Default: false, PreRelease: featuregate.Beta}, // Off by default (requires Azure Disk CSI driver) - CSIMigrationAzureDiskComplete: {Default: false, PreRelease: featuregate.Alpha}, - CSIMigrationAzureFile: {Default: false, PreRelease: featuregate.Alpha}, - CSIMigrationAzureFileComplete: {Default: false, PreRelease: featuregate.Alpha}, - CSIMigrationvSphere: {Default: false, PreRelease: featuregate.Beta}, - CSIMigrationvSphereComplete: {Default: false, PreRelease: featuregate.Beta}, - RunAsGroup: {Default: true, PreRelease: featuregate.Beta}, - CSIMigrationOpenStack: {Default: false, PreRelease: featuregate.Beta}, // Off by default (requires OpenStack Cinder CSI driver) - CSIMigrationOpenStackComplete: {Default: false, PreRelease: featuregate.Alpha}, - VolumeSubpath: {Default: true, PreRelease: featuregate.GA}, - ConfigurableFSGroupPolicy: {Default: true, PreRelease: featuregate.Beta}, - BalanceAttachedNodeVolumes: {Default: false, PreRelease: featuregate.Alpha}, - CSIBlockVolume: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.20 - CSIInlineVolume: {Default: true, PreRelease: featuregate.Beta}, - CSIStorageCapacity: {Default: false, PreRelease: featuregate.Alpha}, - CSIServiceAccountToken: {Default: false, PreRelease: featuregate.Alpha}, - GenericEphemeralVolume: {Default: false, PreRelease: featuregate.Alpha}, - CSIVolumeFSGroupPolicy: {Default: true, PreRelease: featuregate.Beta}, - RuntimeClass: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.23 - NodeLease: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, - SCTPSupport: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.22 - VolumeSnapshotDataSource: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.21 - ProcMountType: {Default: false, PreRelease: featuregate.Alpha}, - TTLAfterFinished: {Default: false, PreRelease: featuregate.Alpha}, - KubeletPodResources: {Default: true, PreRelease: featuregate.Beta}, - WindowsGMSA: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.20 - WindowsRunAsUserName: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.20 + DevicePlugins: {Default: true, PreRelease: featuregate.Beta}, + RotateKubeletServerCertificate: {Default: true, PreRelease: featuregate.Beta}, + LocalStorageCapacityIsolation: {Default: true, PreRelease: featuregate.Beta}, + Sysctls: {Default: true, PreRelease: featuregate.Beta}, + EphemeralContainers: {Default: false, PreRelease: featuregate.Alpha}, + QOSReserved: {Default: false, PreRelease: featuregate.Alpha}, + ExpandPersistentVolumes: {Default: true, PreRelease: featuregate.Beta}, + ExpandInUsePersistentVolumes: {Default: true, PreRelease: featuregate.Beta}, + ExpandCSIVolumes: {Default: true, PreRelease: featuregate.Beta}, + CPUManager: {Default: true, PreRelease: featuregate.Beta}, + CPUCFSQuotaPeriod: {Default: false, PreRelease: featuregate.Alpha}, + TopologyManager: {Default: true, PreRelease: featuregate.Beta}, + ServiceNodeExclusion: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.22 + NodeDisruptionExclusion: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.22 + CSIDriverRegistry: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.20 + BlockVolume: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.20 + StorageObjectInUseProtection: {Default: true, PreRelease: featuregate.GA}, + SupportPodPidsLimit: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.23 + SupportNodePidsLimit: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.23 + BoundServiceAccountTokenVolume: {Default: false, PreRelease: featuregate.Alpha}, + ServiceAccountIssuerDiscovery: {Default: true, PreRelease: featuregate.Beta}, + CRIContainerLogRotation: {Default: true, PreRelease: featuregate.Beta}, + CSIMigration: {Default: true, PreRelease: featuregate.Beta}, + CSIMigrationGCE: {Default: false, PreRelease: featuregate.Beta}, // Off by default (requires GCE PD CSI Driver) + InTreePluginGCEUnregister: {Default: false, PreRelease: featuregate.Alpha}, + CSIMigrationAWS: {Default: false, PreRelease: featuregate.Beta}, // Off by default (requires AWS EBS CSI driver) + InTreePluginAWSUnregister: {Default: false, PreRelease: featuregate.Alpha}, + CSIMigrationAzureDisk: {Default: false, PreRelease: featuregate.Beta}, // Off by default (requires Azure Disk CSI driver) + InTreePluginAzureDiskUnregister: {Default: false, PreRelease: featuregate.Alpha}, + CSIMigrationAzureFile: {Default: false, PreRelease: featuregate.Alpha}, // Off by default (requires Azure File CSI driver) + InTreePluginAzureFileUnregister: {Default: false, PreRelease: featuregate.Alpha}, + CSIMigrationvSphere: {Default: false, PreRelease: featuregate.Beta}, // Off by default (requires vSphere CSI driver) + CSIMigrationvSphereComplete: {Default: false, PreRelease: featuregate.Beta}, // remove in 1.22 + InTreePluginvSphereUnregister: {Default: false, PreRelease: featuregate.Alpha}, + RunAsGroup: {Default: true, PreRelease: featuregate.Beta}, + CSIMigrationOpenStack: {Default: false, PreRelease: featuregate.Beta}, // Off by default (requires OpenStack Cinder CSI driver) + InTreePluginOpenStackUnregister: {Default: false, PreRelease: featuregate.Alpha}, + VolumeSubpath: {Default: true, PreRelease: featuregate.GA}, + ConfigurableFSGroupPolicy: {Default: true, PreRelease: featuregate.Beta}, + BalanceAttachedNodeVolumes: {Default: false, PreRelease: featuregate.Alpha}, + CSIBlockVolume: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.20 + CSIInlineVolume: {Default: true, PreRelease: featuregate.Beta}, + CSIStorageCapacity: {Default: false, PreRelease: featuregate.Alpha}, + CSIServiceAccountToken: {Default: false, PreRelease: featuregate.Alpha}, + GenericEphemeralVolume: {Default: false, PreRelease: featuregate.Alpha}, + CSIVolumeFSGroupPolicy: {Default: true, PreRelease: featuregate.Beta}, + RuntimeClass: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.23 + NodeLease: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, + SCTPSupport: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.22 + VolumeSnapshotDataSource: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.21 + ProcMountType: {Default: false, PreRelease: featuregate.Alpha}, + TTLAfterFinished: {Default: false, PreRelease: featuregate.Alpha}, + KubeletPodResources: {Default: true, PreRelease: featuregate.Beta}, + WindowsGMSA: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.20 + WindowsRunAsUserName: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.20 LocalStorageCapacityIsolationFSQuotaMonitoring: {Default: false, PreRelease: featuregate.Alpha}, NonPreemptingPriority: {Default: true, PreRelease: featuregate.Beta}, PodOverhead: {Default: true, PreRelease: featuregate.Beta}, diff --git a/pkg/volume/csimigration/plugin_manager.go b/pkg/volume/csimigration/plugin_manager.go index 1d635166aed..d7ca1db228d 100644 --- a/pkg/volume/csimigration/plugin_manager.go +++ b/pkg/volume/csimigration/plugin_manager.go @@ -20,7 +20,7 @@ import ( "errors" "fmt" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/component-base/featuregate" csilibplugins "k8s.io/csi-translation-lib/plugins" @@ -48,28 +48,29 @@ func NewPluginManager(m PluginNameMapper) PluginManager { } // IsMigrationCompleteForPlugin indicates whether CSI migration has been completed -// for a particular storage plugin +// for a particular storage plugin. A complete migration will need to: +// 1. Enable CSIMigrationXX for the plugin +// 2. Unregister the in-tree plugin by setting the InTreePluginXXUnregister feature gate func (pm PluginManager) IsMigrationCompleteForPlugin(pluginName string) bool { - // CSIMigration feature and plugin specific migration feature flags should - // be enabled for plugin specific migration completion feature flags to be - // take effect + // CSIMigration feature and plugin specific InTreePluginUnregister feature flags should + // be enabled for plugin specific migration completion to be take effect if !pm.IsMigrationEnabledForPlugin(pluginName) { return false } switch pluginName { case csilibplugins.AWSEBSInTreePluginName: - return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationAWSComplete) + return utilfeature.DefaultFeatureGate.Enabled(features.InTreePluginAWSUnregister) case csilibplugins.GCEPDInTreePluginName: - return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationGCEComplete) + return utilfeature.DefaultFeatureGate.Enabled(features.InTreePluginGCEUnregister) case csilibplugins.AzureFileInTreePluginName: - return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationAzureFileComplete) + return utilfeature.DefaultFeatureGate.Enabled(features.InTreePluginAzureFileUnregister) case csilibplugins.AzureDiskInTreePluginName: - return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationAzureDiskComplete) + return utilfeature.DefaultFeatureGate.Enabled(features.InTreePluginAzureDiskUnregister) case csilibplugins.CinderInTreePluginName: - return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationOpenStackComplete) + return utilfeature.DefaultFeatureGate.Enabled(features.InTreePluginOpenStackUnregister) case csilibplugins.VSphereInTreePluginName: - return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationvSphereComplete) + return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationvSphereComplete) || utilfeature.DefaultFeatureGate.Enabled(features.InTreePluginvSphereUnregister) default: return false } @@ -148,13 +149,25 @@ func TranslateInTreeSpecToCSI(spec *volume.Spec, translator InTreeToCSITranslato } // CheckMigrationFeatureFlags checks the configuration of feature flags related -// to CSI Migration is valid -func CheckMigrationFeatureFlags(f featuregate.FeatureGate, pluginMigration, pluginMigrationComplete featuregate.Feature) error { +// to CSI Migration is valid. It will return whether the migration is complete +// by looking up the pluginMigrationComplete and pluginUnregister flag +func CheckMigrationFeatureFlags(f featuregate.FeatureGate, pluginMigration, + pluginMigrationComplete, pluginUnregister featuregate.Feature) (migrationComplete bool, err error) { if f.Enabled(pluginMigration) && !f.Enabled(features.CSIMigration) { - return fmt.Errorf("enabling %q requires CSIMigration to be enabled", pluginMigration) + return false, fmt.Errorf("enabling %q requires CSIMigration to be enabled", pluginMigration) } - if f.Enabled(pluginMigrationComplete) && !f.Enabled(pluginMigration) { - return fmt.Errorf("enabling %q requires %q to be enabled", pluginMigrationComplete, pluginMigration) + // TODO: Remove the following two checks once the CSIMigrationXXComplete flag is removed + if pluginMigrationComplete != "" && f.Enabled(pluginMigrationComplete) && !f.Enabled(pluginMigration) { + return false, fmt.Errorf("enabling %q requires %q to be enabled", pluginMigrationComplete, pluginMigration) } - return nil + // This is only needed for vSphere since we will deprecate the CSIMigrationvSphereComplete flag soon + if pluginMigrationComplete != "" && f.Enabled(features.CSIMigration) && + f.Enabled(pluginMigration) && f.Enabled(pluginMigrationComplete) { + return true, nil + } + // This is for other in-tree plugin that get migration finished + if f.Enabled(pluginMigration) && f.Enabled(pluginUnregister) { + return true, nil + } + return false, nil } diff --git a/pkg/volume/csimigration/plugin_manager_test.go b/pkg/volume/csimigration/plugin_manager_test.go index 51621db9896..cad4d5c1029 100644 --- a/pkg/volume/csimigration/plugin_manager_test.go +++ b/pkg/volume/csimigration/plugin_manager_test.go @@ -20,7 +20,7 @@ import ( "fmt" "testing" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/component-base/featuregate" featuregatetesting "k8s.io/component-base/featuregate/testing" @@ -148,160 +148,191 @@ func TestCheckMigrationFeatureFlags(t *testing.T) { csiMigrationEnabled bool pluginFeatureComplete featuregate.Feature pluginFeatureCompleteEnabled bool - result bool + pluginUnregsiterFeature featuregate.Feature + pluginUnregsiterEnabled bool + expectMigrationComplete bool + expectErr bool }{ { name: "plugin specific feature flag enabled with migration flag disabled", - pluginFeature: features.CSIMigrationGCE, + pluginFeature: features.CSIMigrationvSphere, pluginFeatureEnabled: true, csiMigrationEnabled: false, - pluginFeatureComplete: features.CSIMigrationGCEComplete, + pluginFeatureComplete: features.CSIMigrationvSphereComplete, pluginFeatureCompleteEnabled: false, - result: false, + pluginUnregsiterFeature: features.InTreePluginvSphereUnregister, + pluginUnregsiterEnabled: false, + expectMigrationComplete: false, + expectErr: false, }, { name: "plugin specific complete flag enabled but plugin specific feature flag disabled", - pluginFeature: features.CSIMigrationAWS, + pluginFeature: features.CSIMigrationvSphere, pluginFeatureEnabled: false, csiMigrationEnabled: true, - pluginFeatureComplete: features.CSIMigrationAWSComplete, + pluginFeatureComplete: features.CSIMigrationvSphereComplete, pluginFeatureCompleteEnabled: true, - result: false, + pluginUnregsiterFeature: features.InTreePluginvSphereUnregister, + pluginUnregsiterEnabled: false, + expectMigrationComplete: false, + expectErr: false, }, { name: "plugin specific complete feature disabled but plugin specific migration feature and CSI migration flag enabled", - pluginFeature: features.CSIMigrationGCE, + pluginFeature: features.CSIMigrationvSphere, pluginFeatureEnabled: true, csiMigrationEnabled: true, - pluginFeatureComplete: features.CSIMigrationGCEComplete, + pluginFeatureComplete: features.CSIMigrationvSphereComplete, pluginFeatureCompleteEnabled: false, - result: true, + pluginUnregsiterFeature: features.InTreePluginvSphereUnregister, + pluginUnregsiterEnabled: false, + expectMigrationComplete: false, + expectErr: true, }, { name: "all features enabled", - pluginFeature: features.CSIMigrationAWS, + pluginFeature: features.CSIMigrationvSphere, pluginFeatureEnabled: true, csiMigrationEnabled: true, - pluginFeatureComplete: features.CSIMigrationAWSComplete, + pluginFeatureComplete: features.CSIMigrationvSphereComplete, pluginFeatureCompleteEnabled: true, - result: true, + pluginUnregsiterFeature: features.InTreePluginvSphereUnregister, + pluginUnregsiterEnabled: true, + expectMigrationComplete: true, + expectErr: true, + }, + { + name: "no plugin feature complete set", + pluginFeature: features.CSIMigrationGCE, + pluginFeatureEnabled: true, + csiMigrationEnabled: true, + pluginUnregsiterFeature: features.InTreePluginGCEUnregister, + pluginUnregsiterEnabled: true, + expectMigrationComplete: true, + expectErr: true, }, } for _, test := range testCases { t.Run(fmt.Sprintf("Testing %v", test.name), func(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIMigration, test.csiMigrationEnabled)() defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, test.pluginFeature, test.pluginFeatureEnabled)() - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, test.pluginFeatureComplete, test.pluginFeatureCompleteEnabled)() - err := CheckMigrationFeatureFlags(utilfeature.DefaultFeatureGate, test.pluginFeature, test.pluginFeatureComplete) - if err != nil && test.result == true { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, test.pluginUnregsiterFeature, test.pluginUnregsiterEnabled)() + if test.pluginFeatureComplete != "" { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, test.pluginFeatureComplete, test.pluginFeatureCompleteEnabled)() + } + migrationComplete, err := CheckMigrationFeatureFlags(utilfeature.DefaultFeatureGate, test.pluginFeature, test.pluginFeatureComplete, test.pluginUnregsiterFeature) + if err != nil && test.expectErr == true { t.Errorf("Unexpected error: %v", err) } - if err == nil && test.result == false { + if err == nil && test.expectErr == false { t.Errorf("Unexpected validation pass") } + if migrationComplete != test.expectMigrationComplete { + t.Errorf("Unexpected migrationComplete result. Exp: %v, got %v", test.expectMigrationComplete, migrationComplete) + } }) } } func TestMigrationFeatureFlagStatus(t *testing.T) { testCases := []struct { - name string - pluginName string - csiMigrationEnabled bool - pluginFeature featuregate.Feature - pluginFeatureEnabled bool - pluginFeatureComplete featuregate.Feature - pluginFeatureCompleteEnabled bool - csiMigrationResult bool - csiMigrationCompleteResult bool + name string + pluginName string + csiMigrationEnabled bool + pluginFeature featuregate.Feature + pluginFeatureEnabled bool + inTreePluginUnregister featuregate.Feature + inTreePluginUnregisterEnabled bool + csiMigrationResult bool + csiMigrationCompleteResult bool }{ { - name: "gce-pd migration flag disabled and migration-complete flag disabled with CSI migration flag disabled", - pluginName: "kubernetes.io/gce-pd", - pluginFeature: features.CSIMigrationGCE, - pluginFeatureEnabled: false, - csiMigrationEnabled: false, - pluginFeatureComplete: features.CSIMigrationGCEComplete, - pluginFeatureCompleteEnabled: false, - csiMigrationResult: false, - csiMigrationCompleteResult: false, + name: "gce-pd migration flag disabled and migration-complete flag disabled with CSI migration flag disabled", + pluginName: "kubernetes.io/gce-pd", + pluginFeature: features.CSIMigrationGCE, + pluginFeatureEnabled: false, + csiMigrationEnabled: false, + inTreePluginUnregister: features.InTreePluginGCEUnregister, + inTreePluginUnregisterEnabled: false, + csiMigrationResult: false, + csiMigrationCompleteResult: false, }, { - name: "gce-pd migration flag disabled and migration-complete flag disabled with CSI migration flag enabled", - pluginName: "kubernetes.io/gce-pd", - pluginFeature: features.CSIMigrationGCE, - pluginFeatureEnabled: false, - csiMigrationEnabled: true, - pluginFeatureComplete: features.CSIMigrationGCEComplete, - pluginFeatureCompleteEnabled: false, - csiMigrationResult: false, - csiMigrationCompleteResult: false, + name: "gce-pd migration flag disabled and migration-complete flag disabled with CSI migration flag enabled", + pluginName: "kubernetes.io/gce-pd", + pluginFeature: features.CSIMigrationGCE, + pluginFeatureEnabled: false, + csiMigrationEnabled: true, + inTreePluginUnregister: features.InTreePluginGCEUnregister, + inTreePluginUnregisterEnabled: false, + csiMigrationResult: false, + csiMigrationCompleteResult: false, }, { - name: "gce-pd migration flag enabled and migration-complete flag disabled with CSI migration flag enabled", - pluginName: "kubernetes.io/gce-pd", - pluginFeature: features.CSIMigrationGCE, - pluginFeatureEnabled: true, - csiMigrationEnabled: true, - pluginFeatureComplete: features.CSIMigrationGCEComplete, - pluginFeatureCompleteEnabled: false, - csiMigrationResult: true, - csiMigrationCompleteResult: false, + name: "gce-pd migration flag enabled and migration-complete flag disabled with CSI migration flag enabled", + pluginName: "kubernetes.io/gce-pd", + pluginFeature: features.CSIMigrationGCE, + pluginFeatureEnabled: true, + csiMigrationEnabled: true, + inTreePluginUnregister: features.InTreePluginGCEUnregister, + inTreePluginUnregisterEnabled: false, + csiMigrationResult: true, + csiMigrationCompleteResult: false, }, { - name: "gce-pd migration flag enabled and migration-complete flag enabled with CSI migration flag enabled", - pluginName: "kubernetes.io/gce-pd", - pluginFeature: features.CSIMigrationGCE, - pluginFeatureEnabled: true, - csiMigrationEnabled: true, - pluginFeatureComplete: features.CSIMigrationGCEComplete, - pluginFeatureCompleteEnabled: true, - csiMigrationResult: true, - csiMigrationCompleteResult: true, + name: "gce-pd migration flag enabled and migration-complete flag enabled with CSI migration flag enabled", + pluginName: "kubernetes.io/gce-pd", + pluginFeature: features.CSIMigrationGCE, + pluginFeatureEnabled: true, + csiMigrationEnabled: true, + inTreePluginUnregister: features.InTreePluginGCEUnregister, + inTreePluginUnregisterEnabled: true, + csiMigrationResult: true, + csiMigrationCompleteResult: true, }, { - name: "aws-ebs migration flag disabled and migration-complete flag disabled with CSI migration flag disabled", - pluginName: "kubernetes.io/aws-ebs", - pluginFeature: features.CSIMigrationAWS, - pluginFeatureEnabled: false, - csiMigrationEnabled: false, - pluginFeatureComplete: features.CSIMigrationAWSComplete, - pluginFeatureCompleteEnabled: false, - csiMigrationResult: false, - csiMigrationCompleteResult: false, + name: "aws-ebs migration flag disabled and migration-complete flag disabled with CSI migration flag disabled", + pluginName: "kubernetes.io/aws-ebs", + pluginFeature: features.CSIMigrationAWS, + pluginFeatureEnabled: false, + csiMigrationEnabled: false, + inTreePluginUnregister: features.InTreePluginAWSUnregister, + inTreePluginUnregisterEnabled: false, + csiMigrationResult: false, + csiMigrationCompleteResult: false, }, { - name: "aws-ebs migration flag disabled and migration-complete flag disabled with CSI migration flag enabled", - pluginName: "kubernetes.io/aws-ebs", - pluginFeature: features.CSIMigrationAWS, - pluginFeatureEnabled: false, - csiMigrationEnabled: true, - pluginFeatureComplete: features.CSIMigrationAWSComplete, - pluginFeatureCompleteEnabled: false, - csiMigrationResult: false, - csiMigrationCompleteResult: false, + name: "aws-ebs migration flag disabled and migration-complete flag disabled with CSI migration flag enabled", + pluginName: "kubernetes.io/aws-ebs", + pluginFeature: features.CSIMigrationAWS, + pluginFeatureEnabled: false, + csiMigrationEnabled: true, + inTreePluginUnregister: features.InTreePluginAWSUnregister, + inTreePluginUnregisterEnabled: false, + csiMigrationResult: false, + csiMigrationCompleteResult: false, }, { - name: "aws-ebs migration flag enabled and migration-complete flag disabled with CSI migration flag enabled", - pluginName: "kubernetes.io/aws-ebs", - pluginFeature: features.CSIMigrationAWS, - pluginFeatureEnabled: true, - csiMigrationEnabled: true, - pluginFeatureComplete: features.CSIMigrationAWSComplete, - pluginFeatureCompleteEnabled: false, - csiMigrationResult: true, - csiMigrationCompleteResult: false, + name: "aws-ebs migration flag enabled and migration-complete flag disabled with CSI migration flag enabled", + pluginName: "kubernetes.io/aws-ebs", + pluginFeature: features.CSIMigrationAWS, + pluginFeatureEnabled: true, + csiMigrationEnabled: true, + inTreePluginUnregister: features.InTreePluginAWSUnregister, + inTreePluginUnregisterEnabled: false, + csiMigrationResult: true, + csiMigrationCompleteResult: false, }, { - name: "aws-ebs migration flag enabled and migration-complete flag enabled with CSI migration flag enabled", - pluginName: "kubernetes.io/aws-ebs", - pluginFeature: features.CSIMigrationAWS, - pluginFeatureEnabled: true, - csiMigrationEnabled: true, - pluginFeatureComplete: features.CSIMigrationAWSComplete, - pluginFeatureCompleteEnabled: true, - csiMigrationResult: true, - csiMigrationCompleteResult: true, + name: "aws-ebs migration flag enabled and migration-complete flag enabled with CSI migration flag enabled", + pluginName: "kubernetes.io/aws-ebs", + pluginFeature: features.CSIMigrationAWS, + pluginFeatureEnabled: true, + csiMigrationEnabled: true, + inTreePluginUnregister: features.InTreePluginAWSUnregister, + inTreePluginUnregisterEnabled: true, + csiMigrationResult: true, + csiMigrationCompleteResult: true, }, } csiTranslator := csitrans.New() @@ -310,7 +341,7 @@ func TestMigrationFeatureFlagStatus(t *testing.T) { t.Run(fmt.Sprintf("Testing %v", test.name), func(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIMigration, test.csiMigrationEnabled)() defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, test.pluginFeature, test.pluginFeatureEnabled)() - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, test.pluginFeatureComplete, test.pluginFeatureCompleteEnabled)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, test.inTreePluginUnregister, test.inTreePluginUnregisterEnabled)() csiMigrationResult := pm.IsMigrationEnabledForPlugin(test.pluginName) if csiMigrationResult != test.csiMigrationResult {