github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-01 07:47:56 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #98243 from Jiawei0227/unregister-plugin

Browse Source 
Disable in-tree plugin without enabling CSI migration
This commit is contained in:
Kubernetes Prow Robot 2021-01-29 12:59:48 -08:00 committed by GitHub
commit c04058418f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 304 additions and 236 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
cmd/kube-controller-manager/app/plugins_providers.go
View File

@ -35,40 +35,48 @@ import (
type probeFn func() []volume.VolumePlugin type probeFn func() []volume.VolumePlugin
func appendPluginBasedOnMigrationFeatureFlags(plugins []volume.VolumePlugin, inTreePluginName string, featureGate featuregate.FeatureGate, pluginMigration, pluginMigrationComplete featuregate.Feature, fn probeFn) ([]volume.VolumePlugin, error) { func appendPluginBasedOnFeatureFlags(plugins []volume.VolumePlugin, inTreePluginName string, featureGate featuregate.FeatureGate, pluginInfo pluginInfo) ([]volume.VolumePlugin, error) {
// Skip appending the in-tree plugin to the list of plugins to be probed/initialized // Skip appending the in-tree plugin to the list of plugins to be probed/initialized
// if the CSIMigration feature flag and plugin specific feature flag indicating // if the CSIMigration feature flag and plugin specific feature flag indicating
// CSI migration is complete // CSI migration is complete
err := csimigration.CheckMigrationFeatureFlags(featureGate, pluginMigration, pluginMigrationComplete) migrationComplete, err := csimigration.CheckMigrationFeatureFlags(featureGate, pluginInfo.pluginMigrationFeature,
pluginInfo.pluginMigrationCompleteFeature, pluginInfo.pluginUnregisterFeature)
if err != nil { if err != nil {
klog.Warningf("Unexpected CSI Migration Feature Flags combination detected: %v. CSI Migration may not take effect", err) klog.Warningf("Unexpected CSI Migration Feature Flags combination detected: %v. CSI Migration may not take effect", err)
// TODO: fail and return here once alpha only tests can set the feature flags for a plugin correctly // TODO: fail and return here once alpha only tests can set the feature flags for a plugin correctly
} }
if featureGate.Enabled(features.CSIMigration) && featureGate.Enabled(pluginMigration) && featureGate.Enabled(pluginMigrationComplete) { // TODO: This can be removed after feature flag CSIMigrationvSphereComplete is removed.
klog.Infof("Skip registration of plugin %s since feature flag %v is enabled", inTreePluginName, pluginMigrationComplete) if migrationComplete {
klog.Infof("Skip registration of plugin %s since migration is complete", inTreePluginName)
return plugins, nil return plugins, nil
} }
plugins = append(plugins, fn()...) if featureGate.Enabled(pluginInfo.pluginUnregisterFeature) {
klog.Infof("Skip registration of plugin %s since feature flag %v is enabled", inTreePluginName, pluginInfo.pluginUnregisterFeature)
return plugins, nil
}
plugins = append(plugins, pluginInfo.pluginProbeFunction()...)
return plugins, nil return plugins, nil
} }
type pluginInfo struct { type pluginInfo struct {
pluginMigrationFeature featuregate.Feature pluginMigrationFeature featuregate.Feature
// deprecated, only to keep here for vSphere
pluginMigrationCompleteFeature featuregate.Feature pluginMigrationCompleteFeature featuregate.Feature
pluginUnregisterFeature featuregate.Feature
pluginProbeFunction probeFn pluginProbeFunction probeFn
} }
func appendAttachableLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) { func appendAttachableLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) {
pluginMigrationStatus := make(map[string]pluginInfo) pluginMigrationStatus := make(map[string]pluginInfo)
pluginMigrationStatus[plugins.AWSEBSInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAWS, pluginMigrationCompleteFeature: features.CSIMigrationAWSComplete, pluginProbeFunction: awsebs.ProbeVolumePlugins} pluginMigrationStatus[plugins.AWSEBSInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAWS, pluginUnregisterFeature: features.InTreePluginAWSUnregister, pluginProbeFunction: awsebs.ProbeVolumePlugins}
pluginMigrationStatus[plugins.GCEPDInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationGCE, pluginMigrationCompleteFeature: features.CSIMigrationGCEComplete, pluginProbeFunction: gcepd.ProbeVolumePlugins} pluginMigrationStatus[plugins.GCEPDInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationGCE, pluginUnregisterFeature: features.InTreePluginGCEUnregister, pluginProbeFunction: gcepd.ProbeVolumePlugins}
pluginMigrationStatus[plugins.CinderInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationOpenStack, pluginMigrationCompleteFeature: features.CSIMigrationOpenStackComplete, pluginProbeFunction: cinder.ProbeVolumePlugins} pluginMigrationStatus[plugins.CinderInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationOpenStack, pluginUnregisterFeature: features.InTreePluginOpenStackUnregister, pluginProbeFunction: cinder.ProbeVolumePlugins}
pluginMigrationStatus[plugins.AzureDiskInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureDisk, pluginMigrationCompleteFeature: features.CSIMigrationAzureDiskComplete, pluginProbeFunction: azuredd.ProbeVolumePlugins} pluginMigrationStatus[plugins.AzureDiskInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureDisk, pluginUnregisterFeature: features.InTreePluginAzureDiskUnregister, pluginProbeFunction: azuredd.ProbeVolumePlugins}
pluginMigrationStatus[plugins.VSphereInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationvSphere, pluginMigrationCompleteFeature: features.CSIMigrationvSphereComplete, pluginProbeFunction: vsphere_volume.ProbeVolumePlugins} pluginMigrationStatus[plugins.VSphereInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationvSphere, pluginMigrationCompleteFeature: features.CSIMigrationvSphereComplete, pluginUnregisterFeature: features.InTreePluginvSphereUnregister, pluginProbeFunction: vsphere_volume.ProbeVolumePlugins}
var err error var err error
for pluginName, pluginInfo := range pluginMigrationStatus { for pluginName, pluginInfo := range pluginMigrationStatus {
allPlugins, err = appendPluginBasedOnMigrationFeatureFlags(allPlugins, pluginName, featureGate, pluginInfo.pluginMigrationFeature, pluginInfo.pluginMigrationCompleteFeature, pluginInfo.pluginProbeFunction) allPlugins, err = appendPluginBasedOnFeatureFlags(allPlugins, pluginName, featureGate, pluginInfo)
if err != nil { if err != nil {
return allPlugins, err return allPlugins, err
} }
@ -81,20 +89,24 @@ func appendExpandableLegacyProviderVolumes(allPlugins []volume.VolumePlugin, fea
} }
func appendLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) { func appendLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) {
pluginMigrationStatus := make(map[string]pluginInfo)
pluginMigrationStatus[plugins.AWSEBSInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAWS, pluginMigrationCompleteFeature: features.CSIMigrationAWSComplete, pluginProbeFunction: awsebs.ProbeVolumePlugins}
pluginMigrationStatus[plugins.GCEPDInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationGCE, pluginMigrationCompleteFeature: features.CSIMigrationGCEComplete, pluginProbeFunction: gcepd.ProbeVolumePlugins}
pluginMigrationStatus[plugins.CinderInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationOpenStack, pluginMigrationCompleteFeature: features.CSIMigrationOpenStackComplete, pluginProbeFunction: cinder.ProbeVolumePlugins}
pluginMigrationStatus[plugins.AzureDiskInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureDisk, pluginMigrationCompleteFeature: features.CSIMigrationAzureDiskComplete, pluginProbeFunction: azuredd.ProbeVolumePlugins}
pluginMigrationStatus[plugins.AzureFileInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureFile, pluginMigrationCompleteFeature: features.CSIMigrationAzureFileComplete, pluginProbeFunction: azure_file.ProbeVolumePlugins}
pluginMigrationStatus[plugins.VSphereInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationvSphere, pluginMigrationCompleteFeature: features.CSIMigrationvSphereComplete, pluginProbeFunction: vsphere_volume.ProbeVolumePlugins}
var err error var err error
for pluginName, pluginInfo := range pluginMigrationStatus { // First append attachable volumes
allPlugins, err = appendPluginBasedOnMigrationFeatureFlags(allPlugins, pluginName, featureGate, pluginInfo.pluginMigrationFeature, pluginInfo.pluginMigrationCompleteFeature, pluginInfo.pluginProbeFunction) allPlugins, err = appendAttachableLegacyProviderVolumes(allPlugins, featureGate)
if err != nil { if err != nil {
return allPlugins, err return allPlugins, err
} }
// Then append non-attachable volumes
pluginName := plugins.AzureFileInTreePluginName
pluginInfo := pluginInfo{
pluginMigrationFeature: features.CSIMigrationAzureFile,
pluginUnregisterFeature: features.InTreePluginAzureFileUnregister,
pluginProbeFunction: azure_file.ProbeVolumePlugins,
} }
allPlugins, err = appendPluginBasedOnFeatureFlags(allPlugins, pluginName, featureGate, pluginInfo)
if err != nil {
return allPlugins, err
}
return allPlugins, nil return allPlugins, nil
} }

34
cmd/kubelet/app/plugins_providers.go
View File

@ -35,41 +35,51 @@ import (
type probeFn func() []volume.VolumePlugin type probeFn func() []volume.VolumePlugin
func appendPluginBasedOnMigrationFeatureFlags(plugins []volume.VolumePlugin, inTreePluginName string, featureGate featuregate.FeatureGate, pluginMigration, pluginMigrationComplete featuregate.Feature, fn probeFn) ([]volume.VolumePlugin, error) { func appendPluginBasedOnFeatureFlags(plugins []volume.VolumePlugin, inTreePluginName string,
featureGate featuregate.FeatureGate, pluginInfo pluginInfo) ([]volume.VolumePlugin, error) {
// Skip appending the in-tree plugin to the list of plugins to be probed/initialized // Skip appending the in-tree plugin to the list of plugins to be probed/initialized
// if the CSIMigration feature flag and plugin specific feature flag indicating // if the CSIMigration feature flag and plugin specific feature flag indicating
// CSI migration is complete // CSI migration is complete
err := csimigration.CheckMigrationFeatureFlags(featureGate, pluginMigration, pluginMigrationComplete) migrationComplete, err := csimigration.CheckMigrationFeatureFlags(featureGate, pluginInfo.pluginMigrationFeature,
pluginInfo.pluginMigrationCompleteFeature, pluginInfo.pluginUnregisterFeature)
if err != nil { if err != nil {
klog.Warningf("Unexpected CSI Migration Feature Flags combination detected: %v. CSI Migration may not take effect", err) klog.Warningf("Unexpected CSI Migration Feature Flags combination detected: %v. CSI Migration may not take effect", err)
// TODO: fail and return here once alpha only tests can set the feature flags for a plugin correctly // TODO: fail and return here once alpha only tests can set the feature flags for a plugin correctly
} }
if featureGate.Enabled(features.CSIMigration) && featureGate.Enabled(pluginMigration) && featureGate.Enabled(pluginMigrationComplete) { // TODO: This can be removed after feature flag CSIMigrationvSphereComplete is removed.
klog.Infof("Skip registration of plugin %s since feature flag %v is enabled", inTreePluginName, pluginMigrationComplete) if migrationComplete {
klog.Infof("Skip registration of plugin %s since migration is complete", inTreePluginName)
return plugins, nil return plugins, nil
} }
plugins = append(plugins, fn()...) if featureGate.Enabled(pluginInfo.pluginUnregisterFeature) {
klog.Infof("Skip registration of plugin %s since feature flag %v is enabled", inTreePluginName, pluginInfo.pluginUnregisterFeature)
return plugins, nil
}
plugins = append(plugins, pluginInfo.pluginProbeFunction()...)
return plugins, nil return plugins, nil
} }
type pluginInfo struct { type pluginInfo struct {
pluginMigrationFeature featuregate.Feature pluginMigrationFeature featuregate.Feature
// deprecated, only to keep here for vSphere
pluginMigrationCompleteFeature featuregate.Feature pluginMigrationCompleteFeature featuregate.Feature
pluginUnregisterFeature featuregate.Feature
pluginProbeFunction probeFn pluginProbeFunction probeFn
} }
func appendLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) { func appendLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) {
pluginMigrationStatus := make(map[string]pluginInfo) pluginMigrationStatus := make(map[string]pluginInfo)
pluginMigrationStatus[plugins.AWSEBSInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAWS, pluginMigrationCompleteFeature: features.CSIMigrationAWSComplete, pluginProbeFunction: awsebs.ProbeVolumePlugins} pluginMigrationStatus[plugins.AWSEBSInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAWS, pluginUnregisterFeature: features.InTreePluginAWSUnregister, pluginProbeFunction: awsebs.ProbeVolumePlugins}
pluginMigrationStatus[plugins.GCEPDInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationGCE, pluginMigrationCompleteFeature: features.CSIMigrationGCEComplete, pluginProbeFunction: gcepd.ProbeVolumePlugins} pluginMigrationStatus[plugins.GCEPDInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationGCE, pluginUnregisterFeature: features.InTreePluginGCEUnregister, pluginProbeFunction: gcepd.ProbeVolumePlugins}
pluginMigrationStatus[plugins.CinderInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationOpenStack, pluginMigrationCompleteFeature: features.CSIMigrationOpenStackComplete, pluginProbeFunction: cinder.ProbeVolumePlugins} pluginMigrationStatus[plugins.CinderInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationOpenStack, pluginUnregisterFeature: features.InTreePluginOpenStackUnregister, pluginProbeFunction: cinder.ProbeVolumePlugins}
pluginMigrationStatus[plugins.AzureDiskInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureDisk, pluginMigrationCompleteFeature: features.CSIMigrationAzureDiskComplete, pluginProbeFunction: azuredd.ProbeVolumePlugins} pluginMigrationStatus[plugins.AzureDiskInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureDisk, pluginUnregisterFeature: features.InTreePluginAzureDiskUnregister, pluginProbeFunction: azuredd.ProbeVolumePlugins}
pluginMigrationStatus[plugins.AzureFileInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureFile, pluginMigrationCompleteFeature: features.CSIMigrationAzureFileComplete, pluginProbeFunction: azure_file.ProbeVolumePlugins} pluginMigrationStatus[plugins.AzureFileInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureFile, pluginUnregisterFeature: features.InTreePluginAzureFileUnregister, pluginProbeFunction: azure_file.ProbeVolumePlugins}
pluginMigrationStatus[plugins.VSphereInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationvSphere, pluginMigrationCompleteFeature: features.CSIMigrationvSphereComplete, pluginProbeFunction: vsphere_volume.ProbeVolumePlugins} pluginMigrationStatus[plugins.VSphereInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationvSphere, pluginMigrationCompleteFeature: features.CSIMigrationvSphereComplete, pluginUnregisterFeature: features.InTreePluginvSphereUnregister, pluginProbeFunction: vsphere_volume.ProbeVolumePlugins}
var err error var err error
for pluginName, pluginInfo := range pluginMigrationStatus { for pluginName, pluginInfo := range pluginMigrationStatus {
allPlugins, err = appendPluginBasedOnMigrationFeatureFlags(allPlugins, pluginName, featureGate, pluginInfo.pluginMigrationFeature, pluginInfo.pluginMigrationCompleteFeature, pluginInfo.pluginProbeFunction) allPlugins, err = appendPluginBasedOnFeatureFlags(allPlugins, pluginName, featureGate, pluginInfo)
if err != nil { if err != nil {
return allPlugins, err return allPlugins, err
} }

72
pkg/features/kube_features.go
View File

@ -338,12 +338,11 @@ const (
// Enables the GCE PD in-tree driver to GCE CSI Driver migration feature. // Enables the GCE PD in-tree driver to GCE CSI Driver migration feature.
CSIMigrationGCE featuregate.Feature = "CSIMigrationGCE" CSIMigrationGCE featuregate.Feature = "CSIMigrationGCE"
// owner: @davidz627 // owner: @Jiawei0227
// alpha: v1.17 // alpha: v1.21
// //
// Disables the GCE PD in-tree driver. // Disables the GCE PD in-tree driver.
// Expects GCE PD CSI Driver to be installed and configured on all nodes. InTreePluginGCEUnregister featuregate.Feature = "InTreePluginGCEUnregister"
CSIMigrationGCEComplete featuregate.Feature = "CSIMigrationGCEComplete"
// owner: @leakingtapan // owner: @leakingtapan
// alpha: v1.14 // alpha: v1.14
@ -353,11 +352,10 @@ const (
CSIMigrationAWS featuregate.Feature = "CSIMigrationAWS" CSIMigrationAWS featuregate.Feature = "CSIMigrationAWS"
// owner: @leakingtapan // owner: @leakingtapan
// alpha: v1.17 // alpha: v1.21
// //
// Disables the AWS EBS in-tree driver. // Disables the AWS EBS in-tree driver.
// Expects AWS EBS CSI Driver to be installed and configured on all nodes. InTreePluginAWSUnregister featuregate.Feature = "InTreePluginAWSUnregister"
CSIMigrationAWSComplete featuregate.Feature = "CSIMigrationAWSComplete"
// owner: @andyzhangx // owner: @andyzhangx
// alpha: v1.15 // alpha: v1.15
@ -367,11 +365,10 @@ const (
CSIMigrationAzureDisk featuregate.Feature = "CSIMigrationAzureDisk" CSIMigrationAzureDisk featuregate.Feature = "CSIMigrationAzureDisk"
// owner: @andyzhangx // owner: @andyzhangx
// alpha: v1.17 // alpha: v1.21
// //
// Disables the Azure Disk in-tree driver. // Disables the Azure Disk in-tree driver.
// Expects Azure Disk CSI Driver to be installed and configured on all nodes. InTreePluginAzureDiskUnregister featuregate.Feature = "InTreePluginAzureDiskUnregister"
CSIMigrationAzureDiskComplete featuregate.Feature = "CSIMigrationAzureDiskComplete"
// owner: @andyzhangx // owner: @andyzhangx
// alpha: v1.15 // alpha: v1.15
@ -380,11 +377,10 @@ const (
CSIMigrationAzureFile featuregate.Feature = "CSIMigrationAzureFile" CSIMigrationAzureFile featuregate.Feature = "CSIMigrationAzureFile"
// owner: @andyzhangx // owner: @andyzhangx
// alpha: v1.17 // alpha: v1.21
// //
// Disables the Azure File in-tree driver. // Disables the Azure File in-tree driver.
// Expects Azure File CSI Driver to be installed and configured on all nodes. InTreePluginAzureFileUnregister featuregate.Feature = "InTreePluginAzureFileUnregister"
CSIMigrationAzureFileComplete featuregate.Feature = "CSIMigrationAzureFileComplete"
// owner: @divyenpatel // owner: @divyenpatel
// beta: v1.19 (requires: vSphere vCenter/ESXi Version: 7.0u1, HW Version: VM version 15) // beta: v1.19 (requires: vSphere vCenter/ESXi Version: 7.0u1, HW Version: VM version 15)
@ -399,6 +395,25 @@ const (
// Expects vSphere CSI Driver to be installed and configured on all nodes. // Expects vSphere CSI Driver to be installed and configured on all nodes.
CSIMigrationvSphereComplete featuregate.Feature = "CSIMigrationvSphereComplete" CSIMigrationvSphereComplete featuregate.Feature = "CSIMigrationvSphereComplete"
// owner: @divyenpatel
// alpha: v1.21
//
// Disables the vSphere in-tree driver.
InTreePluginvSphereUnregister featuregate.Feature = "InTreePluginvSphereUnregister"
// owner: @adisky
// alpha: v1.14
// beta: v1.18
//
// Enables the OpenStack Cinder in-tree driver to OpenStack Cinder CSI Driver migration feature.
CSIMigrationOpenStack featuregate.Feature = "CSIMigrationOpenStack"
// owner: @adisky
// alpha: v1.21
//
// Disables the OpenStack Cinder in-tree driver.
InTreePluginOpenStackUnregister featuregate.Feature = "InTreePluginOpenStackUnregister"
// owner: @huffmanca // owner: @huffmanca
// alpha: v1.19 // alpha: v1.19
// beta: v1.20 // beta: v1.20
@ -435,20 +450,6 @@ const (
// Enables support for running container entrypoints as different usernames than their default ones. // Enables support for running container entrypoints as different usernames than their default ones.
WindowsRunAsUserName featuregate.Feature = "WindowsRunAsUserName" WindowsRunAsUserName featuregate.Feature = "WindowsRunAsUserName"
// owner: @adisky
// alpha: v1.14
// beta: v1.18
//
// Enables the OpenStack Cinder in-tree driver to OpenStack Cinder CSI Driver migration feature.
CSIMigrationOpenStack featuregate.Feature = "CSIMigrationOpenStack"
// owner: @adisky
// alpha: v1.17
//
// Disables the OpenStack Cinder in-tree driver.
// Expects the OpenStack Cinder CSI Driver to be installed and configured on all nodes.
CSIMigrationOpenStackComplete featuregate.Feature = "CSIMigrationOpenStackComplete"
// owner: @RobertKrawitz // owner: @RobertKrawitz
// alpha: v1.15 // alpha: v1.15
// //
@ -717,18 +718,19 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
CRIContainerLogRotation: {Default: true, PreRelease: featuregate.Beta}, CRIContainerLogRotation: {Default: true, PreRelease: featuregate.Beta},
CSIMigration: {Default: true, PreRelease: featuregate.Beta}, CSIMigration: {Default: true, PreRelease: featuregate.Beta},
CSIMigrationGCE: {Default: false, PreRelease: featuregate.Beta}, // Off by default (requires GCE PD CSI Driver) CSIMigrationGCE: {Default: false, PreRelease: featuregate.Beta}, // Off by default (requires GCE PD CSI Driver)
CSIMigrationGCEComplete: {Default: false, PreRelease: featuregate.Alpha}, InTreePluginGCEUnregister: {Default: false, PreRelease: featuregate.Alpha},
CSIMigrationAWS: {Default: false, PreRelease: featuregate.Beta}, // Off by default (requires AWS EBS CSI driver) CSIMigrationAWS: {Default: false, PreRelease: featuregate.Beta}, // Off by default (requires AWS EBS CSI driver)
CSIMigrationAWSComplete: {Default: false, PreRelease: featuregate.Alpha}, InTreePluginAWSUnregister: {Default: false, PreRelease: featuregate.Alpha},
CSIMigrationAzureDisk: {Default: false, PreRelease: featuregate.Beta}, // Off by default (requires Azure Disk CSI driver) CSIMigrationAzureDisk: {Default: false, PreRelease: featuregate.Beta}, // Off by default (requires Azure Disk CSI driver)
CSIMigrationAzureDiskComplete: {Default: false, PreRelease: featuregate.Alpha}, InTreePluginAzureDiskUnregister: {Default: false, PreRelease: featuregate.Alpha},
CSIMigrationAzureFile: {Default: false, PreRelease: featuregate.Alpha}, CSIMigrationAzureFile: {Default: false, PreRelease: featuregate.Alpha}, // Off by default (requires Azure File CSI driver)
CSIMigrationAzureFileComplete: {Default: false, PreRelease: featuregate.Alpha}, InTreePluginAzureFileUnregister: {Default: false, PreRelease: featuregate.Alpha},
CSIMigrationvSphere: {Default: false, PreRelease: featuregate.Beta}, CSIMigrationvSphere: {Default: false, PreRelease: featuregate.Beta}, // Off by default (requires vSphere CSI driver)
CSIMigrationvSphereComplete: {Default: false, PreRelease: featuregate.Beta}, CSIMigrationvSphereComplete: {Default: false, PreRelease: featuregate.Beta}, // remove in 1.22
InTreePluginvSphereUnregister: {Default: false, PreRelease: featuregate.Alpha},
RunAsGroup: {Default: true, PreRelease: featuregate.Beta}, RunAsGroup: {Default: true, PreRelease: featuregate.Beta},
CSIMigrationOpenStack: {Default: false, PreRelease: featuregate.Beta}, // Off by default (requires OpenStack Cinder CSI driver) CSIMigrationOpenStack: {Default: false, PreRelease: featuregate.Beta}, // Off by default (requires OpenStack Cinder CSI driver)
CSIMigrationOpenStackComplete: {Default: false, PreRelease: featuregate.Alpha}, InTreePluginOpenStackUnregister: {Default: false, PreRelease: featuregate.Alpha},
VolumeSubpath: {Default: true, PreRelease: featuregate.GA}, VolumeSubpath: {Default: true, PreRelease: featuregate.GA},
ConfigurableFSGroupPolicy: {Default: true, PreRelease: featuregate.Beta}, ConfigurableFSGroupPolicy: {Default: true, PreRelease: featuregate.Beta},
BalanceAttachedNodeVolumes: {Default: false, PreRelease: featuregate.Alpha}, BalanceAttachedNodeVolumes: {Default: false, PreRelease: featuregate.Alpha},

47
pkg/volume/csimigration/plugin_manager.go
View File

@ -20,7 +20,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"k8s.io/api/core/v1" v1 "k8s.io/api/core/v1"
utilfeature "k8s.io/apiserver/pkg/util/feature" utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/component-base/featuregate" "k8s.io/component-base/featuregate"
csilibplugins "k8s.io/csi-translation-lib/plugins" csilibplugins "k8s.io/csi-translation-lib/plugins"
@ -48,28 +48,29 @@ func NewPluginManager(m PluginNameMapper) PluginManager {
} }
// IsMigrationCompleteForPlugin indicates whether CSI migration has been completed // IsMigrationCompleteForPlugin indicates whether CSI migration has been completed
// for a particular storage plugin // for a particular storage plugin. A complete migration will need to:
// 1. Enable CSIMigrationXX for the plugin
// 2. Unregister the in-tree plugin by setting the InTreePluginXXUnregister feature gate
func (pm PluginManager) IsMigrationCompleteForPlugin(pluginName string) bool { func (pm PluginManager) IsMigrationCompleteForPlugin(pluginName string) bool {
// CSIMigration feature and plugin specific migration feature flags should // CSIMigration feature and plugin specific InTreePluginUnregister feature flags should
// be enabled for plugin specific migration completion feature flags to be // be enabled for plugin specific migration completion to be take effect
// take effect
if !pm.IsMigrationEnabledForPlugin(pluginName) { if !pm.IsMigrationEnabledForPlugin(pluginName) {
return false return false
} }
switch pluginName { switch pluginName {
case csilibplugins.AWSEBSInTreePluginName: case csilibplugins.AWSEBSInTreePluginName:
return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationAWSComplete) return utilfeature.DefaultFeatureGate.Enabled(features.InTreePluginAWSUnregister)
case csilibplugins.GCEPDInTreePluginName: case csilibplugins.GCEPDInTreePluginName:
return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationGCEComplete) return utilfeature.DefaultFeatureGate.Enabled(features.InTreePluginGCEUnregister)
case csilibplugins.AzureFileInTreePluginName: case csilibplugins.AzureFileInTreePluginName:
return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationAzureFileComplete) return utilfeature.DefaultFeatureGate.Enabled(features.InTreePluginAzureFileUnregister)
case csilibplugins.AzureDiskInTreePluginName: case csilibplugins.AzureDiskInTreePluginName:
return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationAzureDiskComplete) return utilfeature.DefaultFeatureGate.Enabled(features.InTreePluginAzureDiskUnregister)
case csilibplugins.CinderInTreePluginName: case csilibplugins.CinderInTreePluginName:
return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationOpenStackComplete) return utilfeature.DefaultFeatureGate.Enabled(features.InTreePluginOpenStackUnregister)
case csilibplugins.VSphereInTreePluginName: case csilibplugins.VSphereInTreePluginName:
return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationvSphereComplete) return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationvSphereComplete) || utilfeature.DefaultFeatureGate.Enabled(features.InTreePluginvSphereUnregister)
default: default:
return false return false
} }
@ -148,13 +149,25 @@ func TranslateInTreeSpecToCSI(spec *volume.Spec, translator InTreeToCSITranslato
} }
// CheckMigrationFeatureFlags checks the configuration of feature flags related // CheckMigrationFeatureFlags checks the configuration of feature flags related
// to CSI Migration is valid // to CSI Migration is valid. It will return whether the migration is complete
func CheckMigrationFeatureFlags(f featuregate.FeatureGate, pluginMigration, pluginMigrationComplete featuregate.Feature) error { // by looking up the pluginMigrationComplete and pluginUnregister flag
func CheckMigrationFeatureFlags(f featuregate.FeatureGate, pluginMigration,
pluginMigrationComplete, pluginUnregister featuregate.Feature) (migrationComplete bool, err error) {
if f.Enabled(pluginMigration) && !f.Enabled(features.CSIMigration) { if f.Enabled(pluginMigration) && !f.Enabled(features.CSIMigration) {
return fmt.Errorf("enabling %q requires CSIMigration to be enabled", pluginMigration) return false, fmt.Errorf("enabling %q requires CSIMigration to be enabled", pluginMigration)
} }
if f.Enabled(pluginMigrationComplete) && !f.Enabled(pluginMigration) { // TODO: Remove the following two checks once the CSIMigrationXXComplete flag is removed
return fmt.Errorf("enabling %q requires %q to be enabled", pluginMigrationComplete, pluginMigration) if pluginMigrationComplete != "" && f.Enabled(pluginMigrationComplete) && !f.Enabled(pluginMigration) {
return false, fmt.Errorf("enabling %q requires %q to be enabled", pluginMigrationComplete, pluginMigration)
} }
return nil // This is only needed for vSphere since we will deprecate the CSIMigrationvSphereComplete flag soon
if pluginMigrationComplete != "" && f.Enabled(features.CSIMigration) &&
f.Enabled(pluginMigration) && f.Enabled(pluginMigrationComplete) {
return true, nil
}
// This is for other in-tree plugin that get migration finished
if f.Enabled(pluginMigration) && f.Enabled(pluginUnregister) {
return true, nil
}
return false, nil
} }

103
pkg/volume/csimigration/plugin_manager_test.go
View File

@ -20,7 +20,7 @@ import (
"fmt" "fmt"
"testing" "testing"
"k8s.io/api/core/v1" v1 "k8s.io/api/core/v1"
utilfeature "k8s.io/apiserver/pkg/util/feature" utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/component-base/featuregate" "k8s.io/component-base/featuregate"
featuregatetesting "k8s.io/component-base/featuregate/testing" featuregatetesting "k8s.io/component-base/featuregate/testing"
@ -148,57 +148,88 @@ func TestCheckMigrationFeatureFlags(t *testing.T) {
csiMigrationEnabled bool csiMigrationEnabled bool
pluginFeatureComplete featuregate.Feature pluginFeatureComplete featuregate.Feature
pluginFeatureCompleteEnabled bool pluginFeatureCompleteEnabled bool
result bool pluginUnregsiterFeature featuregate.Feature
pluginUnregsiterEnabled bool
expectMigrationComplete bool
expectErr bool
}{ }{
{ {
name: "plugin specific feature flag enabled with migration flag disabled", name: "plugin specific feature flag enabled with migration flag disabled",
pluginFeature: features.CSIMigrationGCE, pluginFeature: features.CSIMigrationvSphere,
pluginFeatureEnabled: true, pluginFeatureEnabled: true,
csiMigrationEnabled: false, csiMigrationEnabled: false,
pluginFeatureComplete: features.CSIMigrationGCEComplete, pluginFeatureComplete: features.CSIMigrationvSphereComplete,
pluginFeatureCompleteEnabled: false, pluginFeatureCompleteEnabled: false,
result: false, pluginUnregsiterFeature: features.InTreePluginvSphereUnregister,
pluginUnregsiterEnabled: false,
expectMigrationComplete: false,
expectErr: false,
}, },
{ {
name: "plugin specific complete flag enabled but plugin specific feature flag disabled", name: "plugin specific complete flag enabled but plugin specific feature flag disabled",
pluginFeature: features.CSIMigrationAWS, pluginFeature: features.CSIMigrationvSphere,
pluginFeatureEnabled: false, pluginFeatureEnabled: false,
csiMigrationEnabled: true, csiMigrationEnabled: true,
pluginFeatureComplete: features.CSIMigrationAWSComplete, pluginFeatureComplete: features.CSIMigrationvSphereComplete,
pluginFeatureCompleteEnabled: true, pluginFeatureCompleteEnabled: true,
result: false, pluginUnregsiterFeature: features.InTreePluginvSphereUnregister,
pluginUnregsiterEnabled: false,
expectMigrationComplete: false,
expectErr: false,
}, },
{ {
name: "plugin specific complete feature disabled but plugin specific migration feature and CSI migration flag enabled", name: "plugin specific complete feature disabled but plugin specific migration feature and CSI migration flag enabled",
pluginFeature: features.CSIMigrationGCE, pluginFeature: features.CSIMigrationvSphere,
pluginFeatureEnabled: true, pluginFeatureEnabled: true,
csiMigrationEnabled: true, csiMigrationEnabled: true,
pluginFeatureComplete: features.CSIMigrationGCEComplete, pluginFeatureComplete: features.CSIMigrationvSphereComplete,
pluginFeatureCompleteEnabled: false, pluginFeatureCompleteEnabled: false,
result: true, pluginUnregsiterFeature: features.InTreePluginvSphereUnregister,
pluginUnregsiterEnabled: false,
expectMigrationComplete: false,
expectErr: true,
}, },
{ {
name: "all features enabled", name: "all features enabled",
pluginFeature: features.CSIMigrationAWS, pluginFeature: features.CSIMigrationvSphere,
pluginFeatureEnabled: true, pluginFeatureEnabled: true,
csiMigrationEnabled: true, csiMigrationEnabled: true,
pluginFeatureComplete: features.CSIMigrationAWSComplete, pluginFeatureComplete: features.CSIMigrationvSphereComplete,
pluginFeatureCompleteEnabled: true, pluginFeatureCompleteEnabled: true,
result: true, pluginUnregsiterFeature: features.InTreePluginvSphereUnregister,
pluginUnregsiterEnabled: true,
expectMigrationComplete: true,
expectErr: true,
},
{
name: "no plugin feature complete set",
pluginFeature: features.CSIMigrationGCE,
pluginFeatureEnabled: true,
csiMigrationEnabled: true,
pluginUnregsiterFeature: features.InTreePluginGCEUnregister,
pluginUnregsiterEnabled: true,
expectMigrationComplete: true,
expectErr: true,
}, },
} }
for _, test := range testCases { for _, test := range testCases {
t.Run(fmt.Sprintf("Testing %v", test.name), func(t *testing.T) { t.Run(fmt.Sprintf("Testing %v", test.name), func(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIMigration, test.csiMigrationEnabled)() defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIMigration, test.csiMigrationEnabled)()
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, test.pluginFeature, test.pluginFeatureEnabled)() defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, test.pluginFeature, test.pluginFeatureEnabled)()
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, test.pluginUnregsiterFeature, test.pluginUnregsiterEnabled)()
if test.pluginFeatureComplete != "" {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, test.pluginFeatureComplete, test.pluginFeatureCompleteEnabled)() defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, test.pluginFeatureComplete, test.pluginFeatureCompleteEnabled)()
err := CheckMigrationFeatureFlags(utilfeature.DefaultFeatureGate, test.pluginFeature, test.pluginFeatureComplete) }
if err != nil && test.result == true { migrationComplete, err := CheckMigrationFeatureFlags(utilfeature.DefaultFeatureGate, test.pluginFeature, test.pluginFeatureComplete, test.pluginUnregsiterFeature)
if err != nil && test.expectErr == true {
t.Errorf("Unexpected error: %v", err) t.Errorf("Unexpected error: %v", err)
} }
if err == nil && test.result == false { if err == nil && test.expectErr == false {
t.Errorf("Unexpected validation pass") t.Errorf("Unexpected validation pass")
} }
if migrationComplete != test.expectMigrationComplete {
t.Errorf("Unexpected migrationComplete result. Exp: %v, got %v", test.expectMigrationComplete, migrationComplete)
}
}) })
} }
} }
@ -210,8 +241,8 @@ func TestMigrationFeatureFlagStatus(t *testing.T) {
csiMigrationEnabled bool csiMigrationEnabled bool
pluginFeature featuregate.Feature pluginFeature featuregate.Feature
pluginFeatureEnabled bool pluginFeatureEnabled bool
pluginFeatureComplete featuregate.Feature inTreePluginUnregister featuregate.Feature
pluginFeatureCompleteEnabled bool inTreePluginUnregisterEnabled bool
csiMigrationResult bool csiMigrationResult bool
csiMigrationCompleteResult bool csiMigrationCompleteResult bool
}{ }{
@ -221,8 +252,8 @@ func TestMigrationFeatureFlagStatus(t *testing.T) {
pluginFeature: features.CSIMigrationGCE, pluginFeature: features.CSIMigrationGCE,
pluginFeatureEnabled: false, pluginFeatureEnabled: false,
csiMigrationEnabled: false, csiMigrationEnabled: false,
pluginFeatureComplete: features.CSIMigrationGCEComplete, inTreePluginUnregister: features.InTreePluginGCEUnregister,
pluginFeatureCompleteEnabled: false, inTreePluginUnregisterEnabled: false,
csiMigrationResult: false, csiMigrationResult: false,
csiMigrationCompleteResult: false, csiMigrationCompleteResult: false,
}, },
@ -232,8 +263,8 @@ func TestMigrationFeatureFlagStatus(t *testing.T) {
pluginFeature: features.CSIMigrationGCE, pluginFeature: features.CSIMigrationGCE,
pluginFeatureEnabled: false, pluginFeatureEnabled: false,
csiMigrationEnabled: true, csiMigrationEnabled: true,
pluginFeatureComplete: features.CSIMigrationGCEComplete, inTreePluginUnregister: features.InTreePluginGCEUnregister,
pluginFeatureCompleteEnabled: false, inTreePluginUnregisterEnabled: false,
csiMigrationResult: false, csiMigrationResult: false,
csiMigrationCompleteResult: false, csiMigrationCompleteResult: false,
}, },
@ -243,8 +274,8 @@ func TestMigrationFeatureFlagStatus(t *testing.T) {
pluginFeature: features.CSIMigrationGCE, pluginFeature: features.CSIMigrationGCE,
pluginFeatureEnabled: true, pluginFeatureEnabled: true,
csiMigrationEnabled: true, csiMigrationEnabled: true,
pluginFeatureComplete: features.CSIMigrationGCEComplete, inTreePluginUnregister: features.InTreePluginGCEUnregister,
pluginFeatureCompleteEnabled: false, inTreePluginUnregisterEnabled: false,
csiMigrationResult: true, csiMigrationResult: true,
csiMigrationCompleteResult: false, csiMigrationCompleteResult: false,
}, },
@ -254,8 +285,8 @@ func TestMigrationFeatureFlagStatus(t *testing.T) {
pluginFeature: features.CSIMigrationGCE, pluginFeature: features.CSIMigrationGCE,
pluginFeatureEnabled: true, pluginFeatureEnabled: true,
csiMigrationEnabled: true, csiMigrationEnabled: true,
pluginFeatureComplete: features.CSIMigrationGCEComplete, inTreePluginUnregister: features.InTreePluginGCEUnregister,
pluginFeatureCompleteEnabled: true, inTreePluginUnregisterEnabled: true,
csiMigrationResult: true, csiMigrationResult: true,
csiMigrationCompleteResult: true, csiMigrationCompleteResult: true,
}, },
@ -265,8 +296,8 @@ func TestMigrationFeatureFlagStatus(t *testing.T) {
pluginFeature: features.CSIMigrationAWS, pluginFeature: features.CSIMigrationAWS,
pluginFeatureEnabled: false, pluginFeatureEnabled: false,
csiMigrationEnabled: false, csiMigrationEnabled: false,
pluginFeatureComplete: features.CSIMigrationAWSComplete, inTreePluginUnregister: features.InTreePluginAWSUnregister,
pluginFeatureCompleteEnabled: false, inTreePluginUnregisterEnabled: false,
csiMigrationResult: false, csiMigrationResult: false,
csiMigrationCompleteResult: false, csiMigrationCompleteResult: false,
}, },
@ -276,8 +307,8 @@ func TestMigrationFeatureFlagStatus(t *testing.T) {
pluginFeature: features.CSIMigrationAWS, pluginFeature: features.CSIMigrationAWS,
pluginFeatureEnabled: false, pluginFeatureEnabled: false,
csiMigrationEnabled: true, csiMigrationEnabled: true,
pluginFeatureComplete: features.CSIMigrationAWSComplete, inTreePluginUnregister: features.InTreePluginAWSUnregister,
pluginFeatureCompleteEnabled: false, inTreePluginUnregisterEnabled: false,
csiMigrationResult: false, csiMigrationResult: false,
csiMigrationCompleteResult: false, csiMigrationCompleteResult: false,
}, },
@ -287,8 +318,8 @@ func TestMigrationFeatureFlagStatus(t *testing.T) {
pluginFeature: features.CSIMigrationAWS, pluginFeature: features.CSIMigrationAWS,
pluginFeatureEnabled: true, pluginFeatureEnabled: true,
csiMigrationEnabled: true, csiMigrationEnabled: true,
pluginFeatureComplete: features.CSIMigrationAWSComplete, inTreePluginUnregister: features.InTreePluginAWSUnregister,
pluginFeatureCompleteEnabled: false, inTreePluginUnregisterEnabled: false,
csiMigrationResult: true, csiMigrationResult: true,
csiMigrationCompleteResult: false, csiMigrationCompleteResult: false,
}, },
@ -298,8 +329,8 @@ func TestMigrationFeatureFlagStatus(t *testing.T) {
pluginFeature: features.CSIMigrationAWS, pluginFeature: features.CSIMigrationAWS,
pluginFeatureEnabled: true, pluginFeatureEnabled: true,
csiMigrationEnabled: true, csiMigrationEnabled: true,
pluginFeatureComplete: features.CSIMigrationAWSComplete, inTreePluginUnregister: features.InTreePluginAWSUnregister,
pluginFeatureCompleteEnabled: true, inTreePluginUnregisterEnabled: true,
csiMigrationResult: true, csiMigrationResult: true,
csiMigrationCompleteResult: true, csiMigrationCompleteResult: true,
}, },
@ -310,7 +341,7 @@ func TestMigrationFeatureFlagStatus(t *testing.T) {
t.Run(fmt.Sprintf("Testing %v", test.name), func(t *testing.T) { t.Run(fmt.Sprintf("Testing %v", test.name), func(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIMigration, test.csiMigrationEnabled)() defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIMigration, test.csiMigrationEnabled)()
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, test.pluginFeature, test.pluginFeatureEnabled)() defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, test.pluginFeature, test.pluginFeatureEnabled)()
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, test.pluginFeatureComplete, test.pluginFeatureCompleteEnabled)() defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, test.inTreePluginUnregister, test.inTreePluginUnregisterEnabled)()
csiMigrationResult := pm.IsMigrationEnabledForPlugin(test.pluginName) csiMigrationResult := pm.IsMigrationEnabledForPlugin(test.pluginName)
if csiMigrationResult != test.csiMigrationResult { if csiMigrationResult != test.csiMigrationResult {