diff --git a/cmd/kube-controller-manager/app/BUILD b/cmd/kube-controller-manager/app/BUILD index 5b99320fd24..d362bd4fb0d 100644 --- a/cmd/kube-controller-manager/app/BUILD +++ b/cmd/kube-controller-manager/app/BUILD @@ -92,6 +92,7 @@ go_library( "//pkg/volume/azure_file:go_default_library", "//pkg/volume/cinder:go_default_library", "//pkg/volume/csi:go_default_library", + "//pkg/volume/csimigration:go_default_library", "//pkg/volume/fc:go_default_library", "//pkg/volume/flexvolume:go_default_library", "//pkg/volume/flocker:go_default_library", @@ -140,10 +141,12 @@ go_library( "//staging/src/k8s.io/cloud-provider:go_default_library", "//staging/src/k8s.io/component-base/cli/flag:go_default_library", "//staging/src/k8s.io/component-base/cli/globalflag:go_default_library", + "//staging/src/k8s.io/component-base/featuregate:go_default_library", "//staging/src/k8s.io/component-base/metrics/prometheus/ratelimiter:go_default_library", "//staging/src/k8s.io/component-base/version:go_default_library", "//staging/src/k8s.io/component-base/version/verflag:go_default_library", "//staging/src/k8s.io/csi-translation-lib:go_default_library", + "//staging/src/k8s.io/csi-translation-lib/plugins:go_default_library", "//staging/src/k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1:go_default_library", "//staging/src/k8s.io/metrics/pkg/client/custom_metrics:go_default_library", "//staging/src/k8s.io/metrics/pkg/client/external_metrics:go_default_library", diff --git a/cmd/kube-controller-manager/app/core.go b/cmd/kube-controller-manager/app/core.go index ff592155a72..b2afaffc56b 100644 --- a/cmd/kube-controller-manager/app/core.go +++ b/cmd/kube-controller-manager/app/core.go @@ -65,6 +65,7 @@ import ( kubefeatures "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/quota/v1/generic" quotainstall "k8s.io/kubernetes/pkg/quota/v1/install" + "k8s.io/kubernetes/pkg/volume/csimigration" netutils "k8s.io/utils/net" ) @@ -254,10 +255,14 @@ func startRouteController(ctx ControllerContext) (http.Handler, bool, error) { } func startPersistentVolumeBinderController(ctx ControllerContext) (http.Handler, bool, error) { + plugins, err := ProbeControllerVolumePlugins(ctx.Cloud, ctx.ComponentConfig.PersistentVolumeBinderController.VolumeConfiguration) + if err != nil { + return nil, true, fmt.Errorf("failed to probe volume plugins when starting persistentvolume controller: %v", err) + } params := persistentvolumecontroller.ControllerParameters{ KubeClient: ctx.ClientBuilder.ClientOrDie("persistent-volume-binder"), SyncPeriod: ctx.ComponentConfig.PersistentVolumeBinderController.PVClaimBinderSyncPeriod.Duration, - VolumePlugins: ProbeControllerVolumePlugins(ctx.Cloud, ctx.ComponentConfig.PersistentVolumeBinderController.VolumeConfiguration), + VolumePlugins: plugins, Cloud: ctx.Cloud, ClusterName: ctx.ComponentConfig.KubeCloudShared.ClusterName, VolumeInformer: ctx.InformerFactory.Core().V1().PersistentVolumes(), @@ -291,6 +296,11 @@ func startAttachDetachController(ctx ControllerContext) (http.Handler, bool, err csiDriverInformer = ctx.InformerFactory.Storage().V1beta1().CSIDrivers() } + plugins, err := ProbeAttachableVolumePlugins() + if err != nil { + return nil, true, fmt.Errorf("failed to probe volume plugins when starting attach/detach controller: %v", err) + } + attachDetachController, attachDetachControllerErr := attachdetach.NewAttachDetachController( ctx.ClientBuilder.ClientOrDie("attachdetach-controller"), @@ -301,7 +311,7 @@ func startAttachDetachController(ctx ControllerContext) (http.Handler, bool, err csiNodeInformer, csiDriverInformer, ctx.Cloud, - ProbeAttachableVolumePlugins(), + plugins, GetDynamicPluginProber(ctx.ComponentConfig.PersistentVolumeBinderController.VolumeConfiguration), ctx.ComponentConfig.AttachDetachController.DisableAttachDetachReconcilerSync, ctx.ComponentConfig.AttachDetachController.ReconcilerSyncLoopPeriod.Duration, @@ -316,17 +326,23 @@ func startAttachDetachController(ctx ControllerContext) (http.Handler, bool, err func startVolumeExpandController(ctx ControllerContext) (http.Handler, bool, error) { if utilfeature.DefaultFeatureGate.Enabled(features.ExpandPersistentVolumes) { + plugins, err := ProbeExpandableVolumePlugins(ctx.ComponentConfig.PersistentVolumeBinderController.VolumeConfiguration) + if err != nil { + return nil, true, fmt.Errorf("failed to probe volume plugins when starting volume expand controller: %v", err) + } + csiTranslator := csitrans.New() expandController, expandControllerErr := expand.NewExpandController( ctx.ClientBuilder.ClientOrDie("expand-controller"), ctx.InformerFactory.Core().V1().PersistentVolumeClaims(), ctx.InformerFactory.Core().V1().PersistentVolumes(), ctx.InformerFactory.Storage().V1().StorageClasses(), ctx.Cloud, - ProbeExpandableVolumePlugins(ctx.ComponentConfig.PersistentVolumeBinderController.VolumeConfiguration), - csitrans.New()) + plugins, + csiTranslator, + csimigration.NewPluginManager(csiTranslator)) if expandControllerErr != nil { - return nil, true, fmt.Errorf("failed to start volume expand controller : %v", expandControllerErr) + return nil, true, fmt.Errorf("failed to start volume expand controller: %v", expandControllerErr) } go expandController.Run(ctx.Stop) return nil, true, nil diff --git a/cmd/kube-controller-manager/app/plugins.go b/cmd/kube-controller-manager/app/plugins.go index 9e136df94d3..a9c61ac24ef 100644 --- a/cmd/kube-controller-manager/app/plugins.go +++ b/cmd/kube-controller-manager/app/plugins.go @@ -57,10 +57,13 @@ import ( // detach controller. // The list of plugins is manually compiled. This code and the plugin // initialization code for kubelet really, really need a through refactor. -func ProbeAttachableVolumePlugins() []volume.VolumePlugin { +func ProbeAttachableVolumePlugins() ([]volume.VolumePlugin, error) { + var err error allPlugins := []volume.VolumePlugin{} - - allPlugins = appendAttachableLegacyProviderVolumes(allPlugins) + allPlugins, err = appendAttachableLegacyProviderVolumes(allPlugins, utilfeature.DefaultFeatureGate) + if err != nil { + return allPlugins, err + } allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...) allPlugins = append(allPlugins, scaleio.ProbeVolumePlugins()...) allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...) @@ -68,7 +71,7 @@ func ProbeAttachableVolumePlugins() []volume.VolumePlugin { allPlugins = append(allPlugins, iscsi.ProbeVolumePlugins()...) allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...) allPlugins = append(allPlugins, csi.ProbeVolumePlugins()...) - return allPlugins + return allPlugins, nil } // GetDynamicPluginProber gets the probers of dynamically discoverable plugins @@ -79,23 +82,26 @@ func GetDynamicPluginProber(config persistentvolumeconfig.VolumeConfiguration) v } // ProbeExpandableVolumePlugins returns volume plugins which are expandable -func ProbeExpandableVolumePlugins(config persistentvolumeconfig.VolumeConfiguration) []volume.VolumePlugin { +func ProbeExpandableVolumePlugins(config persistentvolumeconfig.VolumeConfiguration) ([]volume.VolumePlugin, error) { + var err error allPlugins := []volume.VolumePlugin{} - - allPlugins = appendExpandableLegacyProviderVolumes(allPlugins) + allPlugins, err = appendExpandableLegacyProviderVolumes(allPlugins, utilfeature.DefaultFeatureGate) + if err != nil { + return allPlugins, err + } allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...) allPlugins = append(allPlugins, glusterfs.ProbeVolumePlugins()...) allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...) allPlugins = append(allPlugins, scaleio.ProbeVolumePlugins()...) allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...) allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...) - return allPlugins + return allPlugins, nil } // ProbeControllerVolumePlugins collects all persistent volume plugins into an // easy to use list. Only volume plugins that implement any of // provisioner/recycler/deleter interface should be returned. -func ProbeControllerVolumePlugins(cloud cloudprovider.Interface, config persistentvolumeconfig.VolumeConfiguration) []volume.VolumePlugin { +func ProbeControllerVolumePlugins(cloud cloudprovider.Interface, config persistentvolumeconfig.VolumeConfiguration) ([]volume.VolumePlugin, error) { allPlugins := []volume.VolumePlugin{} // The list of plugins to probe is decided by this binary, not @@ -131,7 +137,11 @@ func ProbeControllerVolumePlugins(cloud cloudprovider.Interface, config persiste // add rbd provisioner allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...) allPlugins = append(allPlugins, quobyte.ProbeVolumePlugins()...) - allPlugins = appendLegacyProviderVolumes(allPlugins) + var err error + allPlugins, err = appendExpandableLegacyProviderVolumes(allPlugins, utilfeature.DefaultFeatureGate) + if err != nil { + return allPlugins, err + } allPlugins = append(allPlugins, flocker.ProbeVolumePlugins()...) allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...) @@ -143,7 +153,7 @@ func ProbeControllerVolumePlugins(cloud cloudprovider.Interface, config persiste allPlugins = append(allPlugins, csi.ProbeVolumePlugins()...) } - return allPlugins + return allPlugins, nil } // AttemptToLoadRecycler tries decoding a pod from a filepath for use as a recycler for a volume. diff --git a/cmd/kube-controller-manager/app/plugins_providers.go b/cmd/kube-controller-manager/app/plugins_providers.go index 8c505642dac..0c1fd8d095f 100644 --- a/cmd/kube-controller-manager/app/plugins_providers.go +++ b/cmd/kube-controller-manager/app/plugins_providers.go @@ -19,34 +19,84 @@ limitations under the License. package app import ( + "k8s.io/component-base/featuregate" + "k8s.io/csi-translation-lib/plugins" + "k8s.io/klog" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/awsebs" "k8s.io/kubernetes/pkg/volume/azure_dd" "k8s.io/kubernetes/pkg/volume/azure_file" "k8s.io/kubernetes/pkg/volume/cinder" + "k8s.io/kubernetes/pkg/volume/csimigration" "k8s.io/kubernetes/pkg/volume/gcepd" "k8s.io/kubernetes/pkg/volume/vsphere_volume" ) -func appendAttachableLegacyProviderVolumes(allPlugins []volume.VolumePlugin) []volume.VolumePlugin { - allPlugins = append(allPlugins, awsebs.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, azure_dd.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, cinder.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, gcepd.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, vsphere_volume.ProbeVolumePlugins()...) - return allPlugins +type probeFn func() []volume.VolumePlugin + +func appendPluginBasedOnMigrationFeatureFlags(plugins []volume.VolumePlugin, inTreePluginName string, featureGate featuregate.FeatureGate, pluginMigration, pluginMigrationComplete featuregate.Feature, fn probeFn) ([]volume.VolumePlugin, error) { + // Skip appending the in-tree plugin to the list of plugins to be probed/initialized + // if the CSIMigration feature flag and plugin specific feature flag indicating + // CSI migration is complete + err := csimigration.CheckMigrationFeatureFlags(featureGate, pluginMigration, pluginMigrationComplete) + if err != nil { + klog.Warningf("Unexpected CSI Migration Feature Flags combination detected: %v. CSI Migration may not take effect", err) + // TODO: fail and return here once alpha only tests can set the feature flags for a plugin correctly + } + if featureGate.Enabled(features.CSIMigration) && featureGate.Enabled(pluginMigration) && featureGate.Enabled(pluginMigrationComplete) { + klog.Infof("Skip registration of plugin %s since feature flag %v is enabled", inTreePluginName, pluginMigrationComplete) + return plugins, nil + } + plugins = append(plugins, fn()...) + return plugins, nil } -func appendExpandableLegacyProviderVolumes(allPlugins []volume.VolumePlugin) []volume.VolumePlugin { - return appendLegacyProviderVolumes(allPlugins) +type pluginInfo struct { + pluginMigrationFeature featuregate.Feature + pluginMigrationCompleteFeature featuregate.Feature + pluginProbeFunction probeFn } -func appendLegacyProviderVolumes(allPlugins []volume.VolumePlugin) []volume.VolumePlugin { - allPlugins = append(allPlugins, awsebs.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, azure_dd.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, azure_file.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, cinder.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, gcepd.ProbeVolumePlugins()...) +func appendAttachableLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) { + pluginMigrationStatus := make(map[string]pluginInfo) + pluginMigrationStatus[plugins.AWSEBSInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAWS, pluginMigrationCompleteFeature: features.CSIMigrationAWSComplete, pluginProbeFunction: awsebs.ProbeVolumePlugins} + pluginMigrationStatus[plugins.GCEPDInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationGCE, pluginMigrationCompleteFeature: features.CSIMigrationGCEComplete, pluginProbeFunction: gcepd.ProbeVolumePlugins} + pluginMigrationStatus[plugins.CinderInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationOpenStack, pluginMigrationCompleteFeature: features.CSIMigrationOpenStackComplete, pluginProbeFunction: cinder.ProbeVolumePlugins} + pluginMigrationStatus[plugins.AzureDiskInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureDisk, pluginMigrationCompleteFeature: features.CSIMigrationAzureDiskComplete, pluginProbeFunction: azure_dd.ProbeVolumePlugins} + + var err error + for pluginName, pluginInfo := range pluginMigrationStatus { + allPlugins, err = appendPluginBasedOnMigrationFeatureFlags(allPlugins, pluginName, featureGate, pluginInfo.pluginMigrationFeature, pluginInfo.pluginMigrationCompleteFeature, pluginInfo.pluginProbeFunction) + if err != nil { + return allPlugins, err + } + } + allPlugins = append(allPlugins, vsphere_volume.ProbeVolumePlugins()...) - return allPlugins + return allPlugins, nil +} + +func appendExpandableLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) { + return appendLegacyProviderVolumes(allPlugins, featureGate) +} + +func appendLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) { + pluginMigrationStatus := make(map[string]pluginInfo) + pluginMigrationStatus[plugins.AWSEBSInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAWS, pluginMigrationCompleteFeature: features.CSIMigrationAWSComplete, pluginProbeFunction: awsebs.ProbeVolumePlugins} + pluginMigrationStatus[plugins.GCEPDInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationGCE, pluginMigrationCompleteFeature: features.CSIMigrationGCEComplete, pluginProbeFunction: gcepd.ProbeVolumePlugins} + pluginMigrationStatus[plugins.CinderInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationOpenStack, pluginMigrationCompleteFeature: features.CSIMigrationOpenStackComplete, pluginProbeFunction: cinder.ProbeVolumePlugins} + pluginMigrationStatus[plugins.AzureDiskInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureDisk, pluginMigrationCompleteFeature: features.CSIMigrationAzureDiskComplete, pluginProbeFunction: azure_dd.ProbeVolumePlugins} + pluginMigrationStatus[plugins.AzureFileInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureFile, pluginMigrationCompleteFeature: features.CSIMigrationAzureFileComplete, pluginProbeFunction: azure_file.ProbeVolumePlugins} + + var err error + for pluginName, pluginInfo := range pluginMigrationStatus { + allPlugins, err = appendPluginBasedOnMigrationFeatureFlags(allPlugins, pluginName, featureGate, pluginInfo.pluginMigrationFeature, pluginInfo.pluginMigrationCompleteFeature, pluginInfo.pluginProbeFunction) + if err != nil { + return allPlugins, err + } + } + + allPlugins = append(allPlugins, vsphere_volume.ProbeVolumePlugins()...) + return allPlugins, nil } diff --git a/cmd/kubelet/app/BUILD b/cmd/kubelet/app/BUILD index 69334690446..4b280bd129a 100644 --- a/cmd/kubelet/app/BUILD +++ b/cmd/kubelet/app/BUILD @@ -86,6 +86,7 @@ go_library( "//pkg/volume/cinder:go_default_library", "//pkg/volume/configmap:go_default_library", "//pkg/volume/csi:go_default_library", + "//pkg/volume/csimigration:go_default_library", "//pkg/volume/downwardapi:go_default_library", "//pkg/volume/emptydir:go_default_library", "//pkg/volume/fc:go_default_library", @@ -138,8 +139,10 @@ go_library( "//staging/src/k8s.io/client-go/util/keyutil:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", "//staging/src/k8s.io/component-base/cli/flag:go_default_library", + "//staging/src/k8s.io/component-base/featuregate:go_default_library", "//staging/src/k8s.io/component-base/version:go_default_library", "//staging/src/k8s.io/component-base/version/verflag:go_default_library", + "//staging/src/k8s.io/csi-translation-lib/plugins:go_default_library", "//staging/src/k8s.io/kubelet/config/v1beta1:go_default_library", "//vendor/github.com/coreos/go-systemd/daemon:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", diff --git a/cmd/kubelet/app/plugins.go b/cmd/kubelet/app/plugins.go index 4ddf1d65f51..af9f708dc52 100644 --- a/cmd/kubelet/app/plugins.go +++ b/cmd/kubelet/app/plugins.go @@ -22,6 +22,8 @@ import ( _ "k8s.io/kubernetes/pkg/credentialprovider/aws" _ "k8s.io/kubernetes/pkg/credentialprovider/azure" _ "k8s.io/kubernetes/pkg/credentialprovider/gcp" + + "k8s.io/component-base/featuregate" "k8s.io/utils/exec" // Volume plugins @@ -53,7 +55,7 @@ import ( ) // ProbeVolumePlugins collects all volume plugins into an easy to use list. -func ProbeVolumePlugins() []volume.VolumePlugin { +func ProbeVolumePlugins(featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) { allPlugins := []volume.VolumePlugin{} // The list of plugins to probe is decided by the kubelet binary, not @@ -62,7 +64,11 @@ func ProbeVolumePlugins() []volume.VolumePlugin { // // Kubelet does not currently need to configure volume plugins. // If/when it does, see kube-controller-manager/app/plugins.go for example of using volume.VolumeConfig - allPlugins = appendLegacyProviderVolumes(allPlugins) + var err error + allPlugins, err = appendLegacyProviderVolumes(allPlugins, featureGate) + if err != nil { + return allPlugins, err + } allPlugins = append(allPlugins, emptydir.ProbeVolumePlugins()...) allPlugins = append(allPlugins, git_repo.ProbeVolumePlugins()...) allPlugins = append(allPlugins, hostpath.ProbeVolumePlugins(volume.VolumeConfig{})...) @@ -83,7 +89,7 @@ func ProbeVolumePlugins() []volume.VolumePlugin { allPlugins = append(allPlugins, local.ProbeVolumePlugins()...) allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...) allPlugins = append(allPlugins, csi.ProbeVolumePlugins()...) - return allPlugins + return allPlugins, nil } // GetDynamicPluginProber gets the probers of dynamically discoverable plugins diff --git a/cmd/kubelet/app/plugins_providers.go b/cmd/kubelet/app/plugins_providers.go index e40b11a761e..d54706bf1bd 100644 --- a/cmd/kubelet/app/plugins_providers.go +++ b/cmd/kubelet/app/plugins_providers.go @@ -19,21 +19,61 @@ limitations under the License. package app import ( + "k8s.io/component-base/featuregate" + "k8s.io/csi-translation-lib/plugins" + "k8s.io/klog" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/awsebs" "k8s.io/kubernetes/pkg/volume/azure_dd" "k8s.io/kubernetes/pkg/volume/azure_file" "k8s.io/kubernetes/pkg/volume/cinder" + "k8s.io/kubernetes/pkg/volume/csimigration" "k8s.io/kubernetes/pkg/volume/gcepd" "k8s.io/kubernetes/pkg/volume/vsphere_volume" ) -func appendLegacyProviderVolumes(allPlugins []volume.VolumePlugin) []volume.VolumePlugin { - allPlugins = append(allPlugins, awsebs.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, azure_dd.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, azure_file.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, cinder.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, gcepd.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, vsphere_volume.ProbeVolumePlugins()...) - return allPlugins +type probeFn func() []volume.VolumePlugin + +func appendPluginBasedOnMigrationFeatureFlags(plugins []volume.VolumePlugin, inTreePluginName string, featureGate featuregate.FeatureGate, pluginMigration, pluginMigrationComplete featuregate.Feature, fn probeFn) ([]volume.VolumePlugin, error) { + // Skip appending the in-tree plugin to the list of plugins to be probed/initialized + // if the CSIMigration feature flag and plugin specific feature flag indicating + // CSI migration is complete + err := csimigration.CheckMigrationFeatureFlags(featureGate, pluginMigration, pluginMigrationComplete) + if err != nil { + klog.Warningf("Unexpected CSI Migration Feature Flags combination detected: %v. CSI Migration may not take effect", err) + // TODO: fail and return here once alpha only tests can set the feature flags for a plugin correctly + } + if featureGate.Enabled(features.CSIMigration) && featureGate.Enabled(pluginMigration) && featureGate.Enabled(pluginMigrationComplete) { + klog.Infof("Skip registration of plugin %s since feature flag %v is enabled", inTreePluginName, pluginMigrationComplete) + return plugins, nil + } + plugins = append(plugins, fn()...) + return plugins, nil +} + +type pluginInfo struct { + pluginMigrationFeature featuregate.Feature + pluginMigrationCompleteFeature featuregate.Feature + pluginProbeFunction probeFn +} + +func appendLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) { + pluginMigrationStatus := make(map[string]pluginInfo) + pluginMigrationStatus[plugins.AWSEBSInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAWS, pluginMigrationCompleteFeature: features.CSIMigrationAWSComplete, pluginProbeFunction: awsebs.ProbeVolumePlugins} + pluginMigrationStatus[plugins.GCEPDInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationGCE, pluginMigrationCompleteFeature: features.CSIMigrationGCEComplete, pluginProbeFunction: gcepd.ProbeVolumePlugins} + pluginMigrationStatus[plugins.CinderInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationOpenStack, pluginMigrationCompleteFeature: features.CSIMigrationOpenStackComplete, pluginProbeFunction: cinder.ProbeVolumePlugins} + pluginMigrationStatus[plugins.AzureDiskInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureDisk, pluginMigrationCompleteFeature: features.CSIMigrationAzureDiskComplete, pluginProbeFunction: azure_dd.ProbeVolumePlugins} + pluginMigrationStatus[plugins.AzureFileInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureFile, pluginMigrationCompleteFeature: features.CSIMigrationAzureFileComplete, pluginProbeFunction: azure_file.ProbeVolumePlugins} + + var err error + for pluginName, pluginInfo := range pluginMigrationStatus { + allPlugins, err = appendPluginBasedOnMigrationFeatureFlags(allPlugins, pluginName, featureGate, pluginInfo.pluginMigrationFeature, pluginInfo.pluginMigrationCompleteFeature, pluginInfo.pluginProbeFunction) + if err != nil { + return allPlugins, err + } + } + + allPlugins = append(allPlugins, vsphere_volume.ProbeVolumePlugins()...) + return allPlugins, nil } diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index bdc50194b70..fb11f762b68 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -60,6 +60,7 @@ import ( "k8s.io/client-go/util/keyutil" cloudprovider "k8s.io/cloud-provider" cliflag "k8s.io/component-base/cli/flag" + "k8s.io/component-base/featuregate" "k8s.io/component-base/version" "k8s.io/component-base/version/verflag" kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" @@ -247,7 +248,7 @@ HTTP server: The kubelet can also listen for HTTP and respond to a simple API } // use kubeletServer to construct the default KubeletDeps - kubeletDeps, err := UnsecuredDependencies(kubeletServer) + kubeletDeps, err := UnsecuredDependencies(kubeletServer, utilfeature.DefaultFeatureGate) if err != nil { klog.Fatal(err) } @@ -268,7 +269,7 @@ HTTP server: The kubelet can also listen for HTTP and respond to a simple API // run the kubelet klog.V(5).Infof("KubeletConfiguration: %#v", kubeletServer.KubeletConfiguration) - if err := Run(kubeletServer, kubeletDeps, stopCh); err != nil { + if err := Run(kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate, stopCh); err != nil { klog.Fatal(err) } }, @@ -362,7 +363,7 @@ func loadConfigFile(name string) (*kubeletconfiginternal.KubeletConfiguration, e // UnsecuredDependencies returns a Dependencies suitable for being run, or an error if the server setup // is not valid. It will not start any background processes, and does not include authentication/authorization -func UnsecuredDependencies(s *options.KubeletServer) (*kubelet.Dependencies, error) { +func UnsecuredDependencies(s *options.KubeletServer, featureGate featuregate.FeatureGate) (*kubelet.Dependencies, error) { // Initialize the TLS Options tlsOptions, err := InitializeTLS(&s.KubeletFlags, &s.KubeletConfiguration) if err != nil { @@ -383,6 +384,10 @@ func UnsecuredDependencies(s *options.KubeletServer) (*kubelet.Dependencies, err } } + plugins, err := ProbeVolumePlugins(featureGate) + if err != nil { + return nil, err + } return &kubelet.Dependencies{ Auth: nil, // default does not enforce auth[nz] CAdvisorInterface: nil, // cadvisor.New launches background processes (bg http.ListenAndServe, and some bg cleaners), not set here @@ -397,7 +402,7 @@ func UnsecuredDependencies(s *options.KubeletServer) (*kubelet.Dependencies, err Subpather: subpather, OOMAdjuster: oom.NewOOMAdjuster(), OSInterface: kubecontainer.RealOS{}, - VolumePlugins: ProbeVolumePlugins(), + VolumePlugins: plugins, DynamicPluginProber: GetDynamicPluginProber(s.VolumePluginDir, pluginRunner), TLSOptions: tlsOptions}, nil } @@ -406,13 +411,13 @@ func UnsecuredDependencies(s *options.KubeletServer) (*kubelet.Dependencies, err // The kubeDeps argument may be nil - if so, it is initialized from the settings on KubeletServer. // Otherwise, the caller is assumed to have set up the Dependencies object and a default one will // not be generated. -func Run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan struct{}) error { +func Run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, featureGate featuregate.FeatureGate, stopCh <-chan struct{}) error { // To help debugging, immediately log version klog.Infof("Version: %+v", version.Get()) if err := initForOS(s.KubeletFlags.WindowsService); err != nil { return fmt.Errorf("failed OS init: %v", err) } - if err := run(s, kubeDeps, stopCh); err != nil { + if err := run(s, kubeDeps, featureGate, stopCh); err != nil { return fmt.Errorf("failed to run Kubelet: %v", err) } return nil @@ -469,7 +474,7 @@ func makeEventRecorder(kubeDeps *kubelet.Dependencies, nodeName types.NodeName) } } -func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan struct{}) (err error) { +func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, featureGate featuregate.FeatureGate, stopCh <-chan struct{}) (err error) { // Set global feature gates based on the value on the initial KubeletServer err = utilfeature.DefaultMutableFeatureGate.SetFromMap(s.KubeletConfiguration.FeatureGates) if err != nil { @@ -511,7 +516,7 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan } if kubeDeps == nil { - kubeDeps, err = UnsecuredDependencies(s) + kubeDeps, err = UnsecuredDependencies(s, featureGate) if err != nil { return err } diff --git a/pkg/controller/volume/attachdetach/BUILD b/pkg/controller/volume/attachdetach/BUILD index f4254b70d2d..de845cc0863 100644 --- a/pkg/controller/volume/attachdetach/BUILD +++ b/pkg/controller/volume/attachdetach/BUILD @@ -20,6 +20,7 @@ go_library( "//pkg/features:go_default_library", "//pkg/util/mount:go_default_library", "//pkg/volume:go_default_library", + "//pkg/volume/csimigration:go_default_library", "//pkg/volume/util:go_default_library", "//pkg/volume/util/operationexecutor:go_default_library", "//pkg/volume/util/subpath:go_default_library", @@ -45,6 +46,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", + "//staging/src/k8s.io/csi-translation-lib:go_default_library", "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], diff --git a/pkg/controller/volume/attachdetach/attach_detach_controller.go b/pkg/controller/volume/attachdetach/attach_detach_controller.go index 80d60cb5e27..3f1212ae56f 100644 --- a/pkg/controller/volume/attachdetach/attach_detach_controller.go +++ b/pkg/controller/volume/attachdetach/attach_detach_controller.go @@ -44,6 +44,7 @@ import ( "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" cloudprovider "k8s.io/cloud-provider" + csitrans "k8s.io/csi-translation-lib" "k8s.io/klog" utilexec "k8s.io/utils/exec" @@ -56,6 +57,7 @@ import ( "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" + "k8s.io/kubernetes/pkg/volume/csimigration" volumeutil "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/operationexecutor" "k8s.io/kubernetes/pkg/volume/util/subpath" @@ -116,6 +118,7 @@ func NewAttachDetachController( disableReconciliationSync bool, reconcilerSyncDuration time.Duration, timerConfig TimerConfig) (AttachDetachController, error) { + adc := &attachDetachController{ kubeClient: kubeClient, pvcLister: pvcInformer.Lister(), @@ -176,6 +179,10 @@ func NewAttachDetachController( adc.nodeStatusUpdater, recorder) + csiTranslator := csitrans.New() + adc.intreeToCSITranslator = csiTranslator + adc.csiMigratedPluginManager = csimigration.NewPluginManager(csiTranslator) + adc.desiredStateOfWorldPopulator = populator.NewDesiredStateOfWorldPopulator( timerConfig.DesiredStateOfWorldPopulatorLoopSleepPeriod, timerConfig.DesiredStateOfWorldPopulatorListPodsRetryDuration, @@ -183,7 +190,9 @@ func NewAttachDetachController( adc.desiredStateOfWorld, &adc.volumePluginMgr, pvcInformer.Lister(), - pvInformer.Lister()) + pvInformer.Lister(), + adc.csiMigratedPluginManager, + adc.intreeToCSITranslator) podInformer.Informer().AddEventHandler(kcache.ResourceEventHandlerFuncs{ AddFunc: adc.podAdd, @@ -318,6 +327,12 @@ type attachDetachController struct { // pvcQueue is used to queue pvc objects pvcQueue workqueue.RateLimitingInterface + + // csiMigratedPluginManager detects in-tree plugins that have been migrated to CSI + csiMigratedPluginManager csimigration.PluginManager + + // intreeToCSITranslator translates from in-tree volume specs to CSI + intreeToCSITranslator csimigration.InTreeToCSITranslator } func (adc *attachDetachController) Run(stopCh <-chan struct{}) { @@ -355,7 +370,9 @@ func (adc *attachDetachController) Run(stopCh <-chan struct{}) { adc.podLister, adc.actualStateOfWorld, adc.desiredStateOfWorld, - &adc.volumePluginMgr) + &adc.volumePluginMgr, + adc.csiMigratedPluginManager, + adc.intreeToCSITranslator) <-stopCh } @@ -421,10 +438,11 @@ func (adc *attachDetachController) populateDesiredStateOfWorld() error { podToAdd := pod adc.podAdd(podToAdd) for _, podVolume := range podToAdd.Spec.Volumes { + nodeName := types.NodeName(podToAdd.Spec.NodeName) // The volume specs present in the ActualStateOfWorld are nil, let's replace those // with the correct ones found on pods. The present in the ASW with no corresponding // pod will be detached and the spec is irrelevant. - volumeSpec, err := util.CreateVolumeSpec(podVolume, podToAdd.Namespace, adc.pvcLister, adc.pvLister) + volumeSpec, err := util.CreateVolumeSpec(podVolume, podToAdd.Namespace, nodeName, &adc.volumePluginMgr, adc.pvcLister, adc.pvLister, adc.csiMigratedPluginManager, adc.intreeToCSITranslator) if err != nil { klog.Errorf( "Error creating spec for volume %q, pod %q/%q: %v", @@ -434,7 +452,6 @@ func (adc *attachDetachController) populateDesiredStateOfWorld() error { err) continue } - nodeName := types.NodeName(podToAdd.Spec.NodeName) plugin, err := adc.volumePluginMgr.FindAttachablePluginBySpec(volumeSpec) if err != nil || plugin == nil { klog.V(10).Infof( @@ -488,7 +505,7 @@ func (adc *attachDetachController) podAdd(obj interface{}) { true /* default volume action */) util.ProcessPodVolumes(pod, volumeActionFlag, /* addVolumes */ - adc.desiredStateOfWorld, &adc.volumePluginMgr, adc.pvcLister, adc.pvLister) + adc.desiredStateOfWorld, &adc.volumePluginMgr, adc.pvcLister, adc.pvLister, adc.csiMigratedPluginManager, adc.intreeToCSITranslator) } // GetDesiredStateOfWorld returns desired state of world associated with controller @@ -512,7 +529,7 @@ func (adc *attachDetachController) podUpdate(oldObj, newObj interface{}) { true /* default volume action */) util.ProcessPodVolumes(pod, volumeActionFlag, /* addVolumes */ - adc.desiredStateOfWorld, &adc.volumePluginMgr, adc.pvcLister, adc.pvLister) + adc.desiredStateOfWorld, &adc.volumePluginMgr, adc.pvcLister, adc.pvLister, adc.csiMigratedPluginManager, adc.intreeToCSITranslator) } func (adc *attachDetachController) podDelete(obj interface{}) { @@ -522,7 +539,7 @@ func (adc *attachDetachController) podDelete(obj interface{}) { } util.ProcessPodVolumes(pod, false, /* addVolumes */ - adc.desiredStateOfWorld, &adc.volumePluginMgr, adc.pvcLister, adc.pvLister) + adc.desiredStateOfWorld, &adc.volumePluginMgr, adc.pvcLister, adc.pvLister, adc.csiMigratedPluginManager, adc.intreeToCSITranslator) } func (adc *attachDetachController) nodeAdd(obj interface{}) { @@ -640,7 +657,7 @@ func (adc *attachDetachController) syncPVCByKey(key string) error { true /* default volume action */) util.ProcessPodVolumes(pod, volumeActionFlag, /* addVolumes */ - adc.desiredStateOfWorld, &adc.volumePluginMgr, adc.pvcLister, adc.pvLister) + adc.desiredStateOfWorld, &adc.volumePluginMgr, adc.pvcLister, adc.pvLister, adc.csiMigratedPluginManager, adc.intreeToCSITranslator) } return nil } diff --git a/pkg/controller/volume/attachdetach/attach_detach_controller_test.go b/pkg/controller/volume/attachdetach/attach_detach_controller_test.go index 67fd38acab8..5758916185c 100644 --- a/pkg/controller/volume/attachdetach/attach_detach_controller_test.go +++ b/pkg/controller/volume/attachdetach/attach_detach_controller_test.go @@ -52,7 +52,8 @@ func Test_NewAttachDetachController_Positive(t *testing.T) { nil, /* prober */ false, 5*time.Second, - DefaultTimerConfig) + DefaultTimerConfig, + ) // Assert if err != nil { diff --git a/pkg/controller/volume/attachdetach/metrics/BUILD b/pkg/controller/volume/attachdetach/metrics/BUILD index 806d8a721ef..08002eb4c0a 100644 --- a/pkg/controller/volume/attachdetach/metrics/BUILD +++ b/pkg/controller/volume/attachdetach/metrics/BUILD @@ -9,8 +9,10 @@ go_library( "//pkg/controller/volume/attachdetach/cache:go_default_library", "//pkg/controller/volume/attachdetach/util:go_default_library", "//pkg/volume:go_default_library", + "//pkg/volume/csimigration:go_default_library", "//pkg/volume/util:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", "//staging/src/k8s.io/component-base/metrics:go_default_library", "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library", @@ -26,6 +28,7 @@ go_test( "//pkg/controller:go_default_library", "//pkg/controller/volume/attachdetach/cache:go_default_library", "//pkg/controller/volume/attachdetach/testing:go_default_library", + "//pkg/volume/csimigration:go_default_library", "//pkg/volume/testing:go_default_library", "//pkg/volume/util/types:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", @@ -34,6 +37,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", + "//staging/src/k8s.io/csi-translation-lib:go_default_library", ], ) diff --git a/pkg/controller/volume/attachdetach/metrics/metrics.go b/pkg/controller/volume/attachdetach/metrics/metrics.go index f5dedc62836..d082b4914ff 100644 --- a/pkg/controller/volume/attachdetach/metrics/metrics.go +++ b/pkg/controller/volume/attachdetach/metrics/metrics.go @@ -20,6 +20,7 @@ import ( "sync" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" corelisters "k8s.io/client-go/listers/core/v1" "k8s.io/component-base/metrics" "k8s.io/component-base/metrics/legacyregistry" @@ -27,6 +28,7 @@ import ( "k8s.io/kubernetes/pkg/controller/volume/attachdetach/cache" "k8s.io/kubernetes/pkg/controller/volume/attachdetach/util" "k8s.io/kubernetes/pkg/volume" + "k8s.io/kubernetes/pkg/volume/csimigration" volumeutil "k8s.io/kubernetes/pkg/volume/util" ) @@ -60,14 +62,18 @@ func Register(pvcLister corelisters.PersistentVolumeClaimLister, podLister corelisters.PodLister, asw cache.ActualStateOfWorld, dsw cache.DesiredStateOfWorld, - pluginMgr *volume.VolumePluginMgr) { + pluginMgr *volume.VolumePluginMgr, + csiMigratedPluginManager csimigration.PluginManager, + intreeToCSITranslator csimigration.InTreeToCSITranslator) { registerMetrics.Do(func() { legacyregistry.CustomMustRegister(newAttachDetachStateCollector(pvcLister, podLister, pvLister, asw, dsw, - pluginMgr)) + pluginMgr, + csiMigratedPluginManager, + intreeToCSITranslator)) legacyregistry.MustRegister(forcedDetachMetricCounter) }) } @@ -75,12 +81,14 @@ func Register(pvcLister corelisters.PersistentVolumeClaimLister, type attachDetachStateCollector struct { metrics.BaseStableCollector - pvcLister corelisters.PersistentVolumeClaimLister - podLister corelisters.PodLister - pvLister corelisters.PersistentVolumeLister - asw cache.ActualStateOfWorld - dsw cache.DesiredStateOfWorld - volumePluginMgr *volume.VolumePluginMgr + pvcLister corelisters.PersistentVolumeClaimLister + podLister corelisters.PodLister + pvLister corelisters.PersistentVolumeLister + asw cache.ActualStateOfWorld + dsw cache.DesiredStateOfWorld + volumePluginMgr *volume.VolumePluginMgr + csiMigratedPluginManager csimigration.PluginManager + intreeToCSITranslator csimigration.InTreeToCSITranslator } // volumeCount is a map of maps used as a counter, e.g.: @@ -105,8 +113,10 @@ func newAttachDetachStateCollector( pvLister corelisters.PersistentVolumeLister, asw cache.ActualStateOfWorld, dsw cache.DesiredStateOfWorld, - pluginMgr *volume.VolumePluginMgr) *attachDetachStateCollector { - return &attachDetachStateCollector{pvcLister: pvcLister, podLister: podLister, pvLister: pvLister, asw: asw, dsw: dsw, volumePluginMgr: pluginMgr} + pluginMgr *volume.VolumePluginMgr, + csiMigratedPluginManager csimigration.PluginManager, + intreeToCSITranslator csimigration.InTreeToCSITranslator) *attachDetachStateCollector { + return &attachDetachStateCollector{pvcLister: pvcLister, podLister: podLister, pvLister: pvLister, asw: asw, dsw: dsw, volumePluginMgr: pluginMgr, csiMigratedPluginManager: csiMigratedPluginManager, intreeToCSITranslator: intreeToCSITranslator} } // Check if our collector implements necessary collector interface @@ -158,7 +168,7 @@ func (collector *attachDetachStateCollector) getVolumeInUseCount() volumeCount { continue } for _, podVolume := range pod.Spec.Volumes { - volumeSpec, err := util.CreateVolumeSpec(podVolume, pod.Namespace, collector.pvcLister, collector.pvLister) + volumeSpec, err := util.CreateVolumeSpec(podVolume, pod.Namespace, types.NodeName(pod.Spec.NodeName), collector.volumePluginMgr, collector.pvcLister, collector.pvLister, collector.csiMigratedPluginManager, collector.intreeToCSITranslator) if err != nil { continue } diff --git a/pkg/controller/volume/attachdetach/metrics/metrics_test.go b/pkg/controller/volume/attachdetach/metrics/metrics_test.go index 239e5cfc5e9..4f4e172e09c 100644 --- a/pkg/controller/volume/attachdetach/metrics/metrics_test.go +++ b/pkg/controller/volume/attachdetach/metrics/metrics_test.go @@ -25,9 +25,11 @@ import ( k8stypes "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" + csitrans "k8s.io/csi-translation-lib" "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/controller/volume/attachdetach/cache" controllervolumetesting "k8s.io/kubernetes/pkg/controller/volume/attachdetach/testing" + "k8s.io/kubernetes/pkg/volume/csimigration" volumetesting "k8s.io/kubernetes/pkg/volume/testing" "k8s.io/kubernetes/pkg/volume/util/types" ) @@ -107,13 +109,16 @@ func TestVolumesInUseMetricCollection(t *testing.T) { pvcLister := pvcInformer.Lister() pvLister := pvInformer.Lister() + csiTranslator := csitrans.New() metricCollector := newAttachDetachStateCollector( pvcLister, fakePodInformer.Lister(), pvLister, nil, nil, - fakeVolumePluginMgr) + fakeVolumePluginMgr, + csimigration.NewPluginManager(csiTranslator), + csiTranslator) nodeUseMap := metricCollector.getVolumeInUseCount() if len(nodeUseMap) < 1 { t.Errorf("Expected one volume in use got %d", len(nodeUseMap)) @@ -145,13 +150,16 @@ func TestTotalVolumesMetricCollection(t *testing.T) { } asw.AddVolumeNode(volumeName, volumeSpec, nodeName, "", true) + csiTranslator := csitrans.New() metricCollector := newAttachDetachStateCollector( nil, nil, nil, asw, dsw, - fakeVolumePluginMgr) + fakeVolumePluginMgr, + csimigration.NewPluginManager(csiTranslator), + csiTranslator) totalVolumesMap := metricCollector.getTotalVolumesCount() if len(totalVolumesMap) != 2 { diff --git a/pkg/controller/volume/attachdetach/populator/BUILD b/pkg/controller/volume/attachdetach/populator/BUILD index b216acf90f7..9b9a7bafe62 100644 --- a/pkg/controller/volume/attachdetach/populator/BUILD +++ b/pkg/controller/volume/attachdetach/populator/BUILD @@ -14,6 +14,7 @@ go_library( "//pkg/controller/volume/attachdetach/cache:go_default_library", "//pkg/controller/volume/attachdetach/util:go_default_library", "//pkg/volume:go_default_library", + "//pkg/volume/csimigration:go_default_library", "//pkg/volume/util:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", @@ -45,6 +46,7 @@ go_test( deps = [ "//pkg/controller:go_default_library", "//pkg/controller/volume/attachdetach/cache:go_default_library", + "//pkg/volume/csimigration:go_default_library", "//pkg/volume/testing:go_default_library", "//pkg/volume/util:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", @@ -52,5 +54,6 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", + "//staging/src/k8s.io/csi-translation-lib:go_default_library", ], ) diff --git a/pkg/controller/volume/attachdetach/populator/desired_state_of_world_populator.go b/pkg/controller/volume/attachdetach/populator/desired_state_of_world_populator.go index 2d9005b97a1..17f5737f25f 100644 --- a/pkg/controller/volume/attachdetach/populator/desired_state_of_world_populator.go +++ b/pkg/controller/volume/attachdetach/populator/desired_state_of_world_populator.go @@ -33,6 +33,7 @@ import ( "k8s.io/kubernetes/pkg/controller/volume/attachdetach/cache" "k8s.io/kubernetes/pkg/controller/volume/attachdetach/util" "k8s.io/kubernetes/pkg/volume" + "k8s.io/kubernetes/pkg/volume/csimigration" volutil "k8s.io/kubernetes/pkg/volume/util" ) @@ -58,27 +59,33 @@ func NewDesiredStateOfWorldPopulator( desiredStateOfWorld cache.DesiredStateOfWorld, volumePluginMgr *volume.VolumePluginMgr, pvcLister corelisters.PersistentVolumeClaimLister, - pvLister corelisters.PersistentVolumeLister) DesiredStateOfWorldPopulator { + pvLister corelisters.PersistentVolumeLister, + csiMigratedPluginManager csimigration.PluginManager, + intreeToCSITranslator csimigration.InTreeToCSITranslator) DesiredStateOfWorldPopulator { return &desiredStateOfWorldPopulator{ - loopSleepDuration: loopSleepDuration, - listPodsRetryDuration: listPodsRetryDuration, - podLister: podLister, - desiredStateOfWorld: desiredStateOfWorld, - volumePluginMgr: volumePluginMgr, - pvcLister: pvcLister, - pvLister: pvLister, + loopSleepDuration: loopSleepDuration, + listPodsRetryDuration: listPodsRetryDuration, + podLister: podLister, + desiredStateOfWorld: desiredStateOfWorld, + volumePluginMgr: volumePluginMgr, + pvcLister: pvcLister, + pvLister: pvLister, + csiMigratedPluginManager: csiMigratedPluginManager, + intreeToCSITranslator: intreeToCSITranslator, } } type desiredStateOfWorldPopulator struct { - loopSleepDuration time.Duration - podLister corelisters.PodLister - desiredStateOfWorld cache.DesiredStateOfWorld - volumePluginMgr *volume.VolumePluginMgr - pvcLister corelisters.PersistentVolumeClaimLister - pvLister corelisters.PersistentVolumeLister - listPodsRetryDuration time.Duration - timeOfLastListPods time.Time + loopSleepDuration time.Duration + podLister corelisters.PodLister + desiredStateOfWorld cache.DesiredStateOfWorld + volumePluginMgr *volume.VolumePluginMgr + pvcLister corelisters.PersistentVolumeClaimLister + pvLister corelisters.PersistentVolumeLister + listPodsRetryDuration time.Duration + timeOfLastListPods time.Time + csiMigratedPluginManager csimigration.PluginManager + intreeToCSITranslator csimigration.InTreeToCSITranslator } func (dswp *desiredStateOfWorldPopulator) Run(stopCh <-chan struct{}) { @@ -163,7 +170,7 @@ func (dswp *desiredStateOfWorldPopulator) findAndAddActivePods() { continue } util.ProcessPodVolumes(pod, true, - dswp.desiredStateOfWorld, dswp.volumePluginMgr, dswp.pvcLister, dswp.pvLister) + dswp.desiredStateOfWorld, dswp.volumePluginMgr, dswp.pvcLister, dswp.pvLister, dswp.csiMigratedPluginManager, dswp.intreeToCSITranslator) } diff --git a/pkg/controller/volume/attachdetach/populator/desired_state_of_world_populator_test.go b/pkg/controller/volume/attachdetach/populator/desired_state_of_world_populator_test.go index 275f6d29081..a16ce9a2cf2 100644 --- a/pkg/controller/volume/attachdetach/populator/desired_state_of_world_populator_test.go +++ b/pkg/controller/volume/attachdetach/populator/desired_state_of_world_populator_test.go @@ -25,8 +25,10 @@ import ( k8stypes "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" + csitrans "k8s.io/csi-translation-lib" "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/controller/volume/attachdetach/cache" + "k8s.io/kubernetes/pkg/volume/csimigration" volumetesting "k8s.io/kubernetes/pkg/volume/testing" "k8s.io/kubernetes/pkg/volume/util" ) @@ -73,14 +75,17 @@ func TestFindAndAddActivePods_FindAndRemoveDeletedPods(t *testing.T) { pvcLister := fakeInformerFactory.Core().V1().PersistentVolumeClaims().Lister() pvLister := fakeInformerFactory.Core().V1().PersistentVolumes().Lister() + csiTranslator := csitrans.New() dswp := &desiredStateOfWorldPopulator{ - loopSleepDuration: 100 * time.Millisecond, - listPodsRetryDuration: 3 * time.Second, - desiredStateOfWorld: fakesDSW, - volumePluginMgr: fakeVolumePluginMgr, - podLister: fakePodInformer.Lister(), - pvcLister: pvcLister, - pvLister: pvLister, + loopSleepDuration: 100 * time.Millisecond, + listPodsRetryDuration: 3 * time.Second, + desiredStateOfWorld: fakesDSW, + volumePluginMgr: fakeVolumePluginMgr, + podLister: fakePodInformer.Lister(), + pvcLister: pvcLister, + pvLister: pvLister, + csiMigratedPluginManager: csimigration.NewPluginManager(csiTranslator), + intreeToCSITranslator: csiTranslator, } //add the given node to the list of nodes managed by dsw diff --git a/pkg/controller/volume/attachdetach/testing/testvolumespec.go b/pkg/controller/volume/attachdetach/testing/testvolumespec.go index 3e23162f3e5..f402c2c6f2d 100644 --- a/pkg/controller/volume/attachdetach/testing/testvolumespec.go +++ b/pkg/controller/volume/attachdetach/testing/testvolumespec.go @@ -251,10 +251,6 @@ func (plugin *TestPlugin) CanSupport(spec *volume.Spec) bool { return true } -func (plugin *TestPlugin) IsMigratedToCSI() bool { - return false -} - func (plugin *TestPlugin) RequiresRemount() bool { return false } diff --git a/pkg/controller/volume/attachdetach/util/BUILD b/pkg/controller/volume/attachdetach/util/BUILD index 59c1db5809f..5e42c06070e 100644 --- a/pkg/controller/volume/attachdetach/util/BUILD +++ b/pkg/controller/volume/attachdetach/util/BUILD @@ -11,10 +11,14 @@ go_library( importpath = "k8s.io/kubernetes/pkg/controller/volume/attachdetach/util", deps = [ "//pkg/controller/volume/attachdetach/cache:go_default_library", + "//pkg/features:go_default_library", "//pkg/volume:go_default_library", + "//pkg/volume/csimigration:go_default_library", "//pkg/volume/util:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", "//vendor/k8s.io/klog:go_default_library", ], diff --git a/pkg/controller/volume/attachdetach/util/util.go b/pkg/controller/volume/attachdetach/util/util.go index 8affeb46f08..72c973d8e17 100644 --- a/pkg/controller/volume/attachdetach/util/util.go +++ b/pkg/controller/volume/attachdetach/util/util.go @@ -17,20 +17,29 @@ limitations under the License. package util import ( + "errors" "fmt" + "strings" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + utilfeature "k8s.io/apiserver/pkg/util/feature" corelisters "k8s.io/client-go/listers/core/v1" "k8s.io/klog" "k8s.io/kubernetes/pkg/controller/volume/attachdetach/cache" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/volume" + "k8s.io/kubernetes/pkg/volume/csimigration" "k8s.io/kubernetes/pkg/volume/util" ) // CreateVolumeSpec creates and returns a mutatable volume.Spec object for the // specified volume. It dereference any PVC to get PV objects, if needed. -func CreateVolumeSpec(podVolume v1.Volume, podNamespace string, pvcLister corelisters.PersistentVolumeClaimLister, pvLister corelisters.PersistentVolumeLister) (*volume.Spec, error) { +// A volume.Spec that refers to an in-tree plugin spec is translated to refer +// to a migrated CSI plugin spec if all conditions for CSI migration on a node +// for the in-tree plugin is satisfied. +func CreateVolumeSpec(podVolume v1.Volume, podNamespace string, nodeName types.NodeName, vpm *volume.VolumePluginMgr, pvcLister corelisters.PersistentVolumeClaimLister, pvLister corelisters.PersistentVolumeLister, csiMigratedPluginManager csimigration.PluginManager, csiTranslator csimigration.InTreeToCSITranslator) (*volume.Spec, error) { if pvcSource := podVolume.VolumeSource.PersistentVolumeClaim; pvcSource != nil { klog.V(10).Infof( "Found PVC, ClaimName: %q/%q", @@ -66,6 +75,15 @@ func CreateVolumeSpec(podVolume v1.Volume, podNamespace string, pvcLister coreli err) } + volumeSpec, err = translateInTreeSpecToCSIIfNeeded(volumeSpec, nodeName, vpm, csiMigratedPluginManager, csiTranslator) + if err != nil { + return nil, fmt.Errorf( + "error performing CSI migration checks and translation for PVC %q/%q: %v", + podNamespace, + pvcSource.ClaimName, + err) + } + klog.V(10).Infof( "Extracted volumeSpec (%v) from bound PV (pvName %q) and PVC (ClaimName %q/%q pvcUID %v)", volumeSpec.Name(), @@ -81,7 +99,15 @@ func CreateVolumeSpec(podVolume v1.Volume, podNamespace string, pvcLister coreli // informer it may be mutated by another consumer. clonedPodVolume := podVolume.DeepCopy() - return volume.NewSpecFromVolume(clonedPodVolume), nil + origspec := volume.NewSpecFromVolume(clonedPodVolume) + spec, err := translateInTreeSpecToCSIIfNeeded(origspec, nodeName, vpm, csiMigratedPluginManager, csiTranslator) + if err != nil { + return nil, fmt.Errorf( + "error performing CSI migration checks and translation for inline volume %q: %v", + podVolume.Name, + err) + } + return spec, nil } // getPVCFromCacheExtractPV fetches the PVC object with the given namespace and @@ -160,7 +186,7 @@ func DetermineVolumeAction(pod *v1.Pod, desiredStateOfWorld cache.DesiredStateOf // ProcessPodVolumes processes the volumes in the given pod and adds them to the // desired state of the world if addVolumes is true, otherwise it removes them. -func ProcessPodVolumes(pod *v1.Pod, addVolumes bool, desiredStateOfWorld cache.DesiredStateOfWorld, volumePluginMgr *volume.VolumePluginMgr, pvcLister corelisters.PersistentVolumeClaimLister, pvLister corelisters.PersistentVolumeLister) { +func ProcessPodVolumes(pod *v1.Pod, addVolumes bool, desiredStateOfWorld cache.DesiredStateOfWorld, volumePluginMgr *volume.VolumePluginMgr, pvcLister corelisters.PersistentVolumeClaimLister, pvLister corelisters.PersistentVolumeLister, csiMigratedPluginManager csimigration.PluginManager, csiTranslator csimigration.InTreeToCSITranslator) { if pod == nil { return } @@ -193,7 +219,7 @@ func ProcessPodVolumes(pod *v1.Pod, addVolumes bool, desiredStateOfWorld cache.D // Process volume spec for each volume defined in pod for _, podVolume := range pod.Spec.Volumes { - volumeSpec, err := CreateVolumeSpec(podVolume, pod.Namespace, pvcLister, pvLister) + volumeSpec, err := CreateVolumeSpec(podVolume, pod.Namespace, nodeName, volumePluginMgr, pvcLister, pvLister, csiMigratedPluginManager, csiTranslator) if err != nil { klog.V(10).Infof( "Error processing volume %q for pod %q/%q: %v", @@ -249,3 +275,110 @@ func ProcessPodVolumes(pod *v1.Pod, addVolumes bool, desiredStateOfWorld cache.D } return } + +func translateInTreeSpecToCSIIfNeeded(spec *volume.Spec, nodeName types.NodeName, vpm *volume.VolumePluginMgr, csiMigratedPluginManager csimigration.PluginManager, csiTranslator csimigration.InTreeToCSITranslator) (*volume.Spec, error) { + translatedSpec := spec + migratable, err := csiMigratedPluginManager.IsMigratable(spec) + if err != nil { + return nil, err + } + migrationSupportedOnNode, err := isCSIMigrationSupportedOnNode(nodeName, spec, vpm, csiMigratedPluginManager) + if err != nil { + return nil, err + } + if migratable && migrationSupportedOnNode { + translatedSpec, err = csimigration.TranslateInTreeSpecToCSI(spec, csiTranslator) + if err != nil { + return nil, err + } + } + return translatedSpec, nil +} + +func isCSIMigrationSupportedOnNode(nodeName types.NodeName, spec *volume.Spec, vpm *volume.VolumePluginMgr, csiMigratedPluginManager csimigration.PluginManager) (bool, error) { + if !utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) || + !utilfeature.DefaultFeatureGate.Enabled(features.CSINodeInfo) { + // If CSIMigration is disabled, CSI migration paths will not be taken for + // the node. If CSINodeInfo is disabled, checking of installation status + // of a migrated CSI plugin cannot be performed. Therefore stick to + // in-tree plugins. + return false, nil + } + + pluginName, err := csiMigratedPluginManager.GetInTreePluginNameFromSpec(spec.PersistentVolume, spec.Volume) + if err != nil { + return false, err + } + + if len(pluginName) == 0 { + // Could not find a plugin name from translation directory, assume not translated + return false, nil + } + + if csiMigratedPluginManager.IsMigrationCompleteForPlugin(pluginName) { + // All nodes are expected to have migrated CSI plugin installed and + // configured when CSI Migration Complete flag is enabled for a plugin. + // CSI migration is supported even if there is version skew between + // managers and node. + return true, nil + } + + if len(nodeName) == 0 { + return false, errors.New("nodeName is empty") + } + + kubeClient := vpm.Host.GetKubeClient() + if kubeClient == nil { + // Don't handle the controller/kubelet version skew check and fallback + // to just checking the feature gates. This can happen if + // we are in a standalone (headless) Kubelet + return true, nil + } + + adcHost, ok := vpm.Host.(volume.AttachDetachVolumeHost) + if !ok { + // Don't handle the controller/kubelet version skew check and fallback + // to just checking the feature gates. This can happen if + // "enableControllerAttachDetach" is set to true on kubelet + return true, nil + } + + if adcHost.CSINodeLister() == nil { + return false, errors.New("could not find CSINodeLister in attachDetachController") + } + + csiNode, err := adcHost.CSINodeLister().Get(string(nodeName)) + if err != nil { + return false, err + } + + ann := csiNode.GetAnnotations() + if ann == nil { + return false, nil + } + + mpa := ann[v1.MigratedPluginsAnnotationKey] + tok := strings.Split(mpa, ",") + mpaSet := sets.NewString(tok...) + + isMigratedOnNode := mpaSet.Has(pluginName) + + if isMigratedOnNode { + installed := false + driverName, err := csiMigratedPluginManager.GetCSINameFromInTreeName(pluginName) + if err != nil { + return isMigratedOnNode, err + } + for _, driver := range csiNode.Spec.Drivers { + if driver.Name == driverName { + installed = true + break + } + } + if !installed { + return true, fmt.Errorf("in-tree plugin %s is migrated on node %s but driver %s is not installed", pluginName, string(nodeName), driverName) + } + } + + return isMigratedOnNode, nil +} diff --git a/pkg/controller/volume/expand/BUILD b/pkg/controller/volume/expand/BUILD index d04ccfeae5b..91621e2d260 100644 --- a/pkg/controller/volume/expand/BUILD +++ b/pkg/controller/volume/expand/BUILD @@ -11,6 +11,7 @@ go_library( "//pkg/controller/volume/events:go_default_library", "//pkg/util/mount:go_default_library", "//pkg/volume:go_default_library", + "//pkg/volume/csimigration:go_default_library", "//pkg/volume/util:go_default_library", "//pkg/volume/util/operationexecutor:go_default_library", "//pkg/volume/util/subpath:go_default_library", @@ -63,6 +64,7 @@ go_test( "//pkg/features:go_default_library", "//pkg/volume:go_default_library", "//pkg/volume/awsebs:go_default_library", + "//pkg/volume/csimigration:go_default_library", "//pkg/volume/util/operationexecutor:go_default_library", "//pkg/volume/util/types:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", diff --git a/pkg/controller/volume/expand/expand_controller.go b/pkg/controller/volume/expand/expand_controller.go index 60319e1ef52..ec4133b58ac 100644 --- a/pkg/controller/volume/expand/expand_controller.go +++ b/pkg/controller/volume/expand/expand_controller.go @@ -47,6 +47,7 @@ import ( "k8s.io/kubernetes/pkg/controller/volume/events" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" + "k8s.io/kubernetes/pkg/volume/csimigration" "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/operationexecutor" "k8s.io/kubernetes/pkg/volume/util/subpath" @@ -100,6 +101,8 @@ type expandController struct { queue workqueue.RateLimitingInterface translator CSINameTranslator + + csiMigratedPluginManager csimigration.PluginManager } // NewExpandController expands the pvs @@ -110,19 +113,21 @@ func NewExpandController( scInformer storageclassinformer.StorageClassInformer, cloud cloudprovider.Interface, plugins []volume.VolumePlugin, - translator CSINameTranslator) (ExpandController, error) { + translator CSINameTranslator, + csiMigratedPluginManager csimigration.PluginManager) (ExpandController, error) { expc := &expandController{ - kubeClient: kubeClient, - cloud: cloud, - pvcLister: pvcInformer.Lister(), - pvcsSynced: pvcInformer.Informer().HasSynced, - pvLister: pvInformer.Lister(), - pvSynced: pvInformer.Informer().HasSynced, - classLister: scInformer.Lister(), - classListerSynced: scInformer.Informer().HasSynced, - queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "volume_expand"), - translator: translator, + kubeClient: kubeClient, + cloud: cloud, + pvcLister: pvcInformer.Lister(), + pvcsSynced: pvcInformer.Informer().HasSynced, + pvLister: pvInformer.Lister(), + pvSynced: pvInformer.Informer().HasSynced, + classLister: scInformer.Lister(), + classListerSynced: scInformer.Informer().HasSynced, + queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "volume_expand"), + translator: translator, + csiMigratedPluginManager: csiMigratedPluginManager, } if err := expc.volumePluginMgr.InitPlugins(plugins, nil, expc); err != nil { @@ -244,25 +249,15 @@ func (expc *expandController) syncHandler(key string) error { return nil } - volumeSpec := volume.NewSpecFromPersistentVolume(pv, false) - volumePlugin, err := expc.volumePluginMgr.FindExpandablePluginBySpec(volumeSpec) volumeResizerName := class.Provisioner - - if err != nil || volumePlugin == nil { - msg := fmt.Errorf("didn't find a plugin capable of expanding the volume; " + - "waiting for an external controller to process this PVC") - eventType := v1.EventTypeNormal - if err != nil { - eventType = v1.EventTypeWarning - } - expc.recorder.Event(pvc, eventType, events.ExternalExpanding, fmt.Sprintf("Ignoring the PVC: %v.", msg)) - klog.Infof("Ignoring the PVC %q (uid: %q) : %v.", util.GetPersistentVolumeClaimQualifiedName(pvc), pvc.UID, msg) - // If we are expecting that an external plugin will handle resizing this volume then - // is no point in requeuing this PVC. + volumeSpec := volume.NewSpecFromPersistentVolume(pv, false) + migratable, err := expc.csiMigratedPluginManager.IsMigratable(volumeSpec) + if err != nil { + klog.V(4).Infof("failed to check CSI migration status for PVC: %s with error: %v", util.ClaimToClaimKey(pvc), err) return nil } - - if volumePlugin.IsMigratedToCSI() { + // handle CSI migration scenarios before invoking FindExpandablePluginBySpec for in-tree + if migratable { msg := fmt.Sprintf("CSI migration enabled for %s; waiting for external resizer to expand the pvc", volumeResizerName) expc.recorder.Event(pvc, v1.EventTypeNormal, events.ExternalExpanding, msg) csiResizerName, err := expc.translator.GetCSINameFromInTreeName(class.Provisioner) @@ -281,6 +276,21 @@ func (expc *expandController) syncHandler(key string) error { return nil } + volumePlugin, err := expc.volumePluginMgr.FindExpandablePluginBySpec(volumeSpec) + if err != nil || volumePlugin == nil { + msg := fmt.Errorf("didn't find a plugin capable of expanding the volume; " + + "waiting for an external controller to process this PVC") + eventType := v1.EventTypeNormal + if err != nil { + eventType = v1.EventTypeWarning + } + expc.recorder.Event(pvc, eventType, events.ExternalExpanding, fmt.Sprintf("Ignoring the PVC: %v.", msg)) + klog.Infof("Ignoring the PVC %q (uid: %q) : %v.", util.GetPersistentVolumeClaimQualifiedName(pvc), pvc.UID, msg) + // If we are expecting that an external plugin will handle resizing this volume then + // is no point in requeuing this PVC. + return nil + } + return expc.expand(pvc, pv, volumeResizerName) } diff --git a/pkg/controller/volume/expand/expand_controller_test.go b/pkg/controller/volume/expand/expand_controller_test.go index 4eb44d753eb..1ddc70a1592 100644 --- a/pkg/controller/volume/expand/expand_controller_test.go +++ b/pkg/controller/volume/expand/expand_controller_test.go @@ -40,6 +40,7 @@ import ( "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/awsebs" + "k8s.io/kubernetes/pkg/volume/csimigration" "k8s.io/kubernetes/pkg/volume/util/operationexecutor" volumetypes "k8s.io/kubernetes/pkg/volume/util/types" ) @@ -124,7 +125,8 @@ func TestSyncHandler(t *testing.T) { if tc.storageClass != nil { informerFactory.Storage().V1().StorageClasses().Informer().GetIndexer().Add(tc.storageClass) } - expc, err := NewExpandController(fakeKubeClient, pvcInformer, pvInformer, storageClassInformer, nil, allPlugins, csitrans.New()) + translator := csitrans.New() + expc, err := NewExpandController(fakeKubeClient, pvcInformer, pvInformer, storageClassInformer, nil, allPlugins, translator, csimigration.NewPluginManager(translator)) if err != nil { t.Fatalf("error creating expand controller : %v", err) } diff --git a/pkg/controller/volume/persistentvolume/BUILD b/pkg/controller/volume/persistentvolume/BUILD index 7aa1337a541..8a34932e0f6 100644 --- a/pkg/controller/volume/persistentvolume/BUILD +++ b/pkg/controller/volume/persistentvolume/BUILD @@ -26,6 +26,7 @@ go_library( "//pkg/util/goroutinemap/exponentialbackoff:go_default_library", "//pkg/util/mount:go_default_library", "//pkg/volume:go_default_library", + "//pkg/volume/csimigration:go_default_library", "//pkg/volume/util:go_default_library", "//pkg/volume/util/recyclerclient:go_default_library", "//pkg/volume/util/subpath:go_default_library", diff --git a/pkg/controller/volume/persistentvolume/framework_test.go b/pkg/controller/volume/persistentvolume/framework_test.go index a09f40eaeeb..fd6c57591fe 100644 --- a/pkg/controller/volume/persistentvolume/framework_test.go +++ b/pkg/controller/volume/persistentvolume/framework_test.go @@ -527,16 +527,21 @@ func (t fakeCSINameTranslator) GetCSINameFromInTreeName(pluginName string) (stri return "vendor.com/MockCSIPlugin", nil } +type fakeCSIMigratedPluginManager struct{} + +func (t fakeCSIMigratedPluginManager) IsMigrationEnabledForPlugin(pluginName string) bool { + return true +} + // wrapTestWithCSIMigrationProvisionCalls returns a testCall that: // - configures controller with a volume plugin that emulates CSI migration // - calls given testCall func wrapTestWithCSIMigrationProvisionCalls(toWrap testCall) testCall { + plugin := &mockVolumePlugin{} return func(ctrl *PersistentVolumeController, reactor *pvtesting.VolumeReactor, test controllerTest) error { - plugin := &mockVolumePlugin{ - isMigratedToCSI: true, - } ctrl.volumePluginMgr.InitPlugins([]vol.VolumePlugin{plugin}, nil /* prober */, ctrl) ctrl.translator = fakeCSINameTranslator{} + ctrl.csiMigratedPluginManager = fakeCSIMigratedPluginManager{} return toWrap(ctrl, reactor, test) } } @@ -782,7 +787,6 @@ type mockVolumePlugin struct { deleteCallCounter int recycleCalls []error recycleCallCounter int - isMigratedToCSI bool provisionOptions vol.VolumeOptions } @@ -812,10 +816,6 @@ func (plugin *mockVolumePlugin) CanSupport(spec *vol.Spec) bool { return true } -func (plugin *mockVolumePlugin) IsMigratedToCSI() bool { - return plugin.isMigratedToCSI -} - func (plugin *mockVolumePlugin) RequiresRemount() bool { return false } diff --git a/pkg/controller/volume/persistentvolume/pv_controller.go b/pkg/controller/volume/persistentvolume/pv_controller.go index fec61c4f245..1a5cfc2284e 100644 --- a/pkg/controller/volume/persistentvolume/pv_controller.go +++ b/pkg/controller/volume/persistentvolume/pv_controller.go @@ -138,6 +138,11 @@ type CSINameTranslator interface { GetCSINameFromInTreeName(pluginName string) (string, error) } +// CSIMigratedPluginManager keeps track of CSI migration status of a plugin +type CSIMigratedPluginManager interface { + IsMigrationEnabledForPlugin(pluginName string) bool +} + // PersistentVolumeController is a controller that synchronizes // PersistentVolumeClaims and PersistentVolumes. It starts two // cache.Controllers that watch PersistentVolume and PersistentVolumeClaim @@ -226,7 +231,8 @@ type PersistentVolumeController struct { // abort: N.A. operationTimestamps metrics.OperationStartTimeCache - translator CSINameTranslator + translator CSINameTranslator + csiMigratedPluginManager CSIMigratedPluginManager } // syncClaim is the main controller method to decide what to do with a claim. @@ -1324,6 +1330,7 @@ func (ctrl *PersistentVolumeController) provisionClaim(claim *v1.PersistentVolum klog.V(4).Infof("provisionClaim[%s]: started", claimToClaimKey(claim)) opName := fmt.Sprintf("provision-%s[%s]", claimToClaimKey(claim), string(claim.UID)) plugin, storageClass, err := ctrl.findProvisionablePlugin(claim) + // findProvisionablePlugin does not return err for external provisioners if err != nil { ctrl.eventRecorder.Event(claim, v1.EventTypeWarning, events.ProvisioningFailed, err.Error()) klog.Errorf("error finding provisioning plugin for claim %s: %v", claimToClaimKey(claim), err) @@ -1338,8 +1345,8 @@ func (ctrl *PersistentVolumeController) provisionClaim(claim *v1.PersistentVolum claimKey := claimToClaimKey(claim) ctrl.operationTimestamps.AddIfNotExist(claimKey, ctrl.getProvisionerName(plugin, storageClass), "provision") var err error - if plugin == nil || plugin.IsMigratedToCSI() { - _, err = ctrl.provisionClaimOperationExternal(claim, plugin, storageClass) + if plugin == nil { + _, err = ctrl.provisionClaimOperationExternal(claim, storageClass) } else { _, err = ctrl.provisionClaimOperation(claim, plugin, storageClass) } @@ -1362,8 +1369,7 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation( claimClass := v1helper.GetPersistentVolumeClaimClass(claim) klog.V(4).Infof("provisionClaimOperation [%s] started, class: %q", claimToClaimKey(claim), claimClass) - // called from provisionClaim(), in this case, plugin MUST NOT be nil and - // plugin.IsMigratedToCSI() MUST return FALSE + // called from provisionClaim(), in this case, plugin MUST NOT be nil // NOTE: checks on plugin/storageClass has been saved pluginName := plugin.GetPluginName() provisionerName := storageClass.Provisioner @@ -1553,15 +1559,14 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation( // This method will be running in a standalone go-routine scheduled in "provisionClaim" func (ctrl *PersistentVolumeController) provisionClaimOperationExternal( claim *v1.PersistentVolumeClaim, - plugin vol.ProvisionableVolumePlugin, storageClass *storage.StorageClass) (string, error) { claimClass := v1helper.GetPersistentVolumeClaimClass(claim) klog.V(4).Infof("provisionClaimOperationExternal [%s] started, class: %q", claimToClaimKey(claim), claimClass) // Set provisionerName to external provisioner name by setClaimProvisioner var err error provisionerName := storageClass.Provisioner - if plugin != nil { - // update the provisioner name to use the CSI in-tree name + if ctrl.csiMigratedPluginManager.IsMigrationEnabledForPlugin(storageClass.Provisioner) { + // update the provisioner name to use the migrated CSI plugin name provisionerName, err = ctrl.translator.GetCSINameFromInTreeName(storageClass.Provisioner) if err != nil { strerr := fmt.Sprintf("error getting CSI name for In tree plugin %s: %v", storageClass.Provisioner, err) @@ -1660,6 +1665,10 @@ func (ctrl *PersistentVolumeController) findProvisionablePlugin(claim *v1.Persis } // Find a plugin for the class + if ctrl.csiMigratedPluginManager.IsMigrationEnabledForPlugin(class.Provisioner) { + // CSI migration scenario - do not depend on in-tree plugin + return nil, class, nil + } plugin, err := ctrl.volumePluginMgr.FindProvisionablePluginByName(class.Provisioner) if err != nil { if !strings.HasPrefix(class.Provisioner, "kubernetes.io/") { @@ -1708,7 +1717,7 @@ func (ctrl *PersistentVolumeController) getProvisionerNameFromVolume(volume *v1. if err != nil { return "N/A" } - if plugin != nil && !plugin.IsMigratedToCSI() { + if plugin != nil { return plugin.GetPluginName() } // If reached here, Either an external provisioner was used for provisioning @@ -1722,22 +1731,25 @@ func (ctrl *PersistentVolumeController) getProvisionerNameFromVolume(volume *v1. if err != nil { return "N/A" } - if plugin != nil { + if ctrl.csiMigratedPluginManager.IsMigrationEnabledForPlugin(class.Provisioner) { provisionerName, err := ctrl.translator.GetCSINameFromInTreeName(class.Provisioner) - if err == nil { - return provisionerName + if err != nil { + return "N/A" } + return provisionerName } return class.Provisioner } -// obtain plugin/external provisioner name from plugin and storage class +// obtain plugin/external provisioner name from plugin and storage class for timestamp logging purposes func (ctrl *PersistentVolumeController) getProvisionerName(plugin vol.ProvisionableVolumePlugin, storageClass *storage.StorageClass) string { - // intree plugin, returns the plugin's name - if plugin != nil && !plugin.IsMigratedToCSI() { + // non CSI-migrated in-tree plugin, returns the plugin's name + if plugin != nil { return plugin.GetPluginName() - } else if plugin != nil { - // get the CSI in-tree name from storage class provisioner name + } + if ctrl.csiMigratedPluginManager.IsMigrationEnabledForPlugin(storageClass.Provisioner) { + // get the name of the CSI plugin that the in-tree storage class + // provisioner has migrated to provisionerName, err := ctrl.translator.GetCSINameFromInTreeName(storageClass.Provisioner) if err != nil { return "N/A" diff --git a/pkg/controller/volume/persistentvolume/pv_controller_base.go b/pkg/controller/volume/persistentvolume/pv_controller_base.go index 449efd7a9e9..1faad0da1f5 100644 --- a/pkg/controller/volume/persistentvolume/pv_controller_base.go +++ b/pkg/controller/volume/persistentvolume/pv_controller_base.go @@ -44,6 +44,7 @@ import ( pvutil "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/util" "k8s.io/kubernetes/pkg/util/goroutinemap" vol "k8s.io/kubernetes/pkg/volume" + "k8s.io/kubernetes/pkg/volume/csimigration" "k8s.io/klog" ) @@ -94,7 +95,6 @@ func NewController(p ControllerParameters) (*PersistentVolumeController, error) volumeQueue: workqueue.NewNamed("volumes"), resyncPeriod: p.SyncPeriod, operationTimestamps: metrics.NewOperationStartTimeCache(), - translator: csitrans.New(), } // Prober is nil because PV is not aware of Flexvolume. @@ -128,6 +128,11 @@ func NewController(p ControllerParameters) (*PersistentVolumeController, error) controller.podListerSynced = p.PodInformer.Informer().HasSynced controller.NodeLister = p.NodeInformer.Lister() controller.NodeListerSynced = p.NodeInformer.Informer().HasSynced + + csiTranslator := csitrans.New() + controller.translator = csiTranslator + controller.csiMigratedPluginManager = csimigration.NewPluginManager(csiTranslator) + return controller, nil } diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index fd1359e87d9..42dc87927a7 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -389,24 +389,52 @@ const ( // Enables the GCE PD in-tree driver to GCE CSI Driver migration feature. CSIMigrationGCE featuregate.Feature = "CSIMigrationGCE" + // owner: @davidz627 + // alpha: v1.17 + // + // Disables the GCE PD in-tree driver. + // Expects GCE PD CSI Driver to be installed and configured on all nodes. + CSIMigrationGCEComplete featuregate.Feature = "CSIMigrationGCEComplete" + // owner: @leakingtapan // alpha: v1.14 // // Enables the AWS EBS in-tree driver to AWS EBS CSI Driver migration feature. CSIMigrationAWS featuregate.Feature = "CSIMigrationAWS" + // owner: @leakingtapan + // alpha: v1.17 + // + // Disables the AWS EBS in-tree driver. + // Expects AWS EBS CSI Driver to be installed and configured on all nodes. + CSIMigrationAWSComplete featuregate.Feature = "CSIMigrationAWSComplete" + // owner: @andyzhangx // alpha: v1.15 // // Enables the Azure Disk in-tree driver to Azure Disk Driver migration feature. CSIMigrationAzureDisk featuregate.Feature = "CSIMigrationAzureDisk" + // owner: @andyzhangx + // alpha: v1.17 + // + // Disables the Azure Disk in-tree driver. + // Expects Azure Disk CSI Driver to be installed and configured on all nodes. + CSIMigrationAzureDiskComplete featuregate.Feature = "CSIMigrationAzureDiskComplete" + // owner: @andyzhangx // alpha: v1.15 // // Enables the Azure File in-tree driver to Azure File Driver migration feature. CSIMigrationAzureFile featuregate.Feature = "CSIMigrationAzureFile" + // owner: @andyzhangx + // alpha: v1.17 + // + // Disables the Azure File in-tree driver. + // Expects Azure File CSI Driver to be installed and configured on all nodes. + CSIMigrationAzureFileComplete featuregate.Feature = "CSIMigrationAzureFileComplete" + // owner: @RobertKrawitz // beta: v1.15 // @@ -433,6 +461,13 @@ const ( // Enables the OpenStack Cinder in-tree driver to OpenStack Cinder CSI Driver migration feature. CSIMigrationOpenStack featuregate.Feature = "CSIMigrationOpenStack" + // owner: @adisky + // alpha: v1.17 + // + // Disables the OpenStack Cinder in-tree driver. + // Expects the OpenStack Cinder CSI Driver to be installed and configured on all nodes. + CSIMigrationOpenStackComplete featuregate.Feature = "CSIMigrationOpenStackComplete" + // owner: @MrHohn // alpha: v1.15 // beta: v1.16 @@ -552,11 +587,16 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS CRIContainerLogRotation: {Default: true, PreRelease: featuregate.Beta}, CSIMigration: {Default: false, PreRelease: featuregate.Alpha}, CSIMigrationGCE: {Default: false, PreRelease: featuregate.Alpha}, + CSIMigrationGCEComplete: {Default: false, PreRelease: featuregate.Alpha}, CSIMigrationAWS: {Default: false, PreRelease: featuregate.Alpha}, + CSIMigrationAWSComplete: {Default: false, PreRelease: featuregate.Alpha}, CSIMigrationAzureDisk: {Default: false, PreRelease: featuregate.Alpha}, + CSIMigrationAzureDiskComplete: {Default: false, PreRelease: featuregate.Alpha}, CSIMigrationAzureFile: {Default: false, PreRelease: featuregate.Alpha}, + CSIMigrationAzureFileComplete: {Default: false, PreRelease: featuregate.Alpha}, RunAsGroup: {Default: true, PreRelease: featuregate.Beta}, CSIMigrationOpenStack: {Default: false, PreRelease: featuregate.Alpha}, + CSIMigrationOpenStackComplete: {Default: false, PreRelease: featuregate.Alpha}, VolumeSubpath: {Default: true, PreRelease: featuregate.GA}, BalanceAttachedNodeVolumes: {Default: false, PreRelease: featuregate.Alpha}, VolumeSubpathEnvExpansion: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.19, diff --git a/pkg/kubelet/volumemanager/BUILD b/pkg/kubelet/volumemanager/BUILD index 14a84296c55..9603d1daef4 100644 --- a/pkg/kubelet/volumemanager/BUILD +++ b/pkg/kubelet/volumemanager/BUILD @@ -25,6 +25,7 @@ go_library( "//pkg/kubelet/volumemanager/reconciler:go_default_library", "//pkg/util/mount:go_default_library", "//pkg/volume:go_default_library", + "//pkg/volume/csimigration:go_default_library", "//pkg/volume/util:go_default_library", "//pkg/volume/util/hostutil:go_default_library", "//pkg/volume/util/operationexecutor:go_default_library", @@ -37,6 +38,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", + "//staging/src/k8s.io/csi-translation-lib:go_default_library", "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/volumemanager/populator/BUILD b/pkg/kubelet/volumemanager/populator/BUILD index 68a003ac6fe..09896dcadd0 100644 --- a/pkg/kubelet/volumemanager/populator/BUILD +++ b/pkg/kubelet/volumemanager/populator/BUILD @@ -19,6 +19,7 @@ go_library( "//pkg/kubelet/util/format:go_default_library", "//pkg/kubelet/volumemanager/cache:go_default_library", "//pkg/volume:go_default_library", + "//pkg/volume/csimigration:go_default_library", "//pkg/volume/util:go_default_library", "//pkg/volume/util/types:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", @@ -59,6 +60,7 @@ go_test( "//pkg/kubelet/status:go_default_library", "//pkg/kubelet/status/testing:go_default_library", "//pkg/kubelet/volumemanager/cache:go_default_library", + "//pkg/volume/csimigration:go_default_library", "//pkg/volume/testing:go_default_library", "//pkg/volume/util:go_default_library", "//pkg/volume/util/types:go_default_library", @@ -70,5 +72,6 @@ go_test( "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/client-go/testing:go_default_library", "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library", + "//staging/src/k8s.io/csi-translation-lib:go_default_library", ], ) diff --git a/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator.go b/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator.go index ac83f3c0168..fce0d839911 100644 --- a/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator.go +++ b/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator.go @@ -43,6 +43,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/util/format" "k8s.io/kubernetes/pkg/kubelet/volumemanager/cache" "k8s.io/kubernetes/pkg/volume" + "k8s.io/kubernetes/pkg/volume/csimigration" "k8s.io/kubernetes/pkg/volume/util" volumetypes "k8s.io/kubernetes/pkg/volume/util/types" ) @@ -87,7 +88,9 @@ func NewDesiredStateOfWorldPopulator( desiredStateOfWorld cache.DesiredStateOfWorld, actualStateOfWorld cache.ActualStateOfWorld, kubeContainerRuntime kubecontainer.Runtime, - keepTerminatedPodVolumes bool) DesiredStateOfWorldPopulator { + keepTerminatedPodVolumes bool, + csiMigratedPluginManager csimigration.PluginManager, + intreeToCSITranslator csimigration.InTreeToCSITranslator) DesiredStateOfWorldPopulator { return &desiredStateOfWorldPopulator{ kubeClient: kubeClient, loopSleepDuration: loopSleepDuration, @@ -102,6 +105,8 @@ func NewDesiredStateOfWorldPopulator( keepTerminatedPodVolumes: keepTerminatedPodVolumes, hasAddedPods: false, hasAddedPodsLock: sync.RWMutex{}, + csiMigratedPluginManager: csiMigratedPluginManager, + intreeToCSITranslator: intreeToCSITranslator, } } @@ -119,6 +124,8 @@ type desiredStateOfWorldPopulator struct { keepTerminatedPodVolumes bool hasAddedPods bool hasAddedPodsLock sync.RWMutex + csiMigratedPluginManager csimigration.PluginManager + intreeToCSITranslator csimigration.InTreeToCSITranslator } type processedPods struct { @@ -505,6 +512,17 @@ func (dswp *desiredStateOfWorldPopulator) createVolumeSpec( pvcSource.ClaimName, pvcUID) + migratable, err := dswp.csiMigratedPluginManager.IsMigratable(volumeSpec) + if err != nil { + return nil, nil, "", err + } + if migratable { + volumeSpec, err = csimigration.TranslateInTreeSpecToCSI(volumeSpec, dswp.intreeToCSITranslator) + if err != nil { + return nil, nil, "", err + } + } + // TODO: replace this with util.GetVolumeMode() when features.BlockVolume is removed. // The function will return the right value then. volumeMode := v1.PersistentVolumeFilesystem @@ -537,7 +555,18 @@ func (dswp *desiredStateOfWorldPopulator) createVolumeSpec( // Do not return the original volume object, since the source could mutate it clonedPodVolume := podVolume.DeepCopy() - return nil, volume.NewSpecFromVolume(clonedPodVolume), "", nil + spec := volume.NewSpecFromVolume(clonedPodVolume) + migratable, err := dswp.csiMigratedPluginManager.IsMigratable(spec) + if err != nil { + return nil, nil, "", err + } + if migratable { + spec, err = csimigration.TranslateInTreeSpecToCSI(spec, dswp.intreeToCSITranslator) + if err != nil { + return nil, nil, "", err + } + } + return nil, spec, "", nil } // getPVCExtractPV fetches the PVC object with the given namespace and name from diff --git a/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator_test.go b/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator_test.go index c339545c932..7d872deb8e8 100644 --- a/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator_test.go +++ b/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator_test.go @@ -30,6 +30,7 @@ import ( "k8s.io/client-go/kubernetes/fake" core "k8s.io/client-go/testing" featuregatetesting "k8s.io/component-base/featuregate/testing" + csitrans "k8s.io/csi-translation-lib" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/configmap" containertest "k8s.io/kubernetes/pkg/kubelet/container/testing" @@ -39,6 +40,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/status" statustest "k8s.io/kubernetes/pkg/kubelet/status/testing" "k8s.io/kubernetes/pkg/kubelet/volumemanager/cache" + "k8s.io/kubernetes/pkg/volume/csimigration" volumetesting "k8s.io/kubernetes/pkg/volume/testing" "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/types" @@ -962,6 +964,7 @@ func createDswpWithVolume(t *testing.T, pv *v1.PersistentVolume, pvc *v1.Persist fakeStatusManager := status.NewManager(fakeClient, fakePodManager, &statustest.FakePodDeletionSafetyProvider{}) + csiTranslator := csitrans.New() dswp := &desiredStateOfWorldPopulator{ kubeClient: fakeClient, loopSleepDuration: 100 * time.Millisecond, @@ -974,6 +977,8 @@ func createDswpWithVolume(t *testing.T, pv *v1.PersistentVolume, pvc *v1.Persist processedPods: make(map[types.UniquePodName]bool)}, kubeContainerRuntime: fakeRuntime, keepTerminatedPodVolumes: false, + csiMigratedPluginManager: csimigration.NewPluginManager(csiTranslator), + intreeToCSITranslator: csiTranslator, } return dswp, fakePodManager, fakesDSW } diff --git a/pkg/kubelet/volumemanager/volume_manager.go b/pkg/kubelet/volumemanager/volume_manager.go index 0aa368761e5..5b349ffefe9 100644 --- a/pkg/kubelet/volumemanager/volume_manager.go +++ b/pkg/kubelet/volumemanager/volume_manager.go @@ -31,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/record" + csitrans "k8s.io/csi-translation-lib" "k8s.io/klog" "k8s.io/kubernetes/pkg/kubelet/config" "k8s.io/kubernetes/pkg/kubelet/container" @@ -43,6 +44,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/volumemanager/reconciler" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" + "k8s.io/kubernetes/pkg/volume/csimigration" "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/hostutil" "k8s.io/kubernetes/pkg/volume/util/operationexecutor" @@ -174,6 +176,11 @@ func NewVolumeManager( blockVolumePathHandler)), } + intreeToCSITranslator := csitrans.New() + csiMigratedPluginManager := csimigration.NewPluginManager(intreeToCSITranslator) + + vm.intreeToCSITranslator = intreeToCSITranslator + vm.csiMigratedPluginManager = csiMigratedPluginManager vm.desiredStateOfWorldPopulator = populator.NewDesiredStateOfWorldPopulator( kubeClient, desiredStateOfWorldPopulatorLoopSleepPeriod, @@ -183,7 +190,9 @@ func NewVolumeManager( vm.desiredStateOfWorld, vm.actualStateOfWorld, kubeContainerRuntime, - keepTerminatedPodVolumes) + keepTerminatedPodVolumes, + csiMigratedPluginManager, + intreeToCSITranslator) vm.reconciler = reconciler.NewReconciler( kubeClient, controllerAttachDetachEnabled, @@ -238,6 +247,12 @@ type volumeManager struct { // desiredStateOfWorldPopulator runs an asynchronous periodic loop to // populate the desiredStateOfWorld using the kubelet PodManager. desiredStateOfWorldPopulator populator.DesiredStateOfWorldPopulator + + // csiMigratedPluginManager keeps track of CSI migration status of plugins + csiMigratedPluginManager csimigration.PluginManager + + // intreeToCSITranslator translates in-tree volume specs to CSI + intreeToCSITranslator csimigration.InTreeToCSITranslator } func (vm *volumeManager) Run(sourcesReady config.SourcesReady, stopCh <-chan struct{}) { diff --git a/pkg/volume/BUILD b/pkg/volume/BUILD index 46956c0de1e..bba57a77212 100644 --- a/pkg/volume/BUILD +++ b/pkg/volume/BUILD @@ -88,6 +88,7 @@ filegroup( "//pkg/volume/cinder:all-srcs", "//pkg/volume/configmap:all-srcs", "//pkg/volume/csi:all-srcs", + "//pkg/volume/csimigration:all-srcs", "//pkg/volume/downwardapi:all-srcs", "//pkg/volume/emptydir:all-srcs", "//pkg/volume/fc:all-srcs", diff --git a/pkg/volume/awsebs/aws_ebs.go b/pkg/volume/awsebs/aws_ebs.go index 4885d991cef..7666a1b0cf5 100644 --- a/pkg/volume/awsebs/aws_ebs.go +++ b/pkg/volume/awsebs/aws_ebs.go @@ -89,11 +89,6 @@ func (plugin *awsElasticBlockStorePlugin) CanSupport(spec *volume.Spec) bool { (spec.Volume != nil && spec.Volume.AWSElasticBlockStore != nil) } -func (plugin *awsElasticBlockStorePlugin) IsMigratedToCSI() bool { - return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) && - utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationAWS) -} - func (plugin *awsElasticBlockStorePlugin) RequiresRemount() bool { return false } diff --git a/pkg/volume/azure_dd/azure_dd.go b/pkg/volume/azure_dd/azure_dd.go index 14d997808c4..02de3a1473e 100644 --- a/pkg/volume/azure_dd/azure_dd.go +++ b/pkg/volume/azure_dd/azure_dd.go @@ -31,8 +31,6 @@ import ( "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" - utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" "k8s.io/legacy-cloud-providers/azure" @@ -122,11 +120,6 @@ func (plugin *azureDataDiskPlugin) CanSupport(spec *volume.Spec) bool { (spec.Volume != nil && spec.Volume.AzureDisk != nil) } -func (plugin *azureDataDiskPlugin) IsMigratedToCSI() bool { - return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) && - utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationAzureDisk) -} - func (plugin *azureDataDiskPlugin) RequiresRemount() bool { return false } diff --git a/pkg/volume/azure_file/BUILD b/pkg/volume/azure_file/BUILD index aa27444aeef..656bc65e0cf 100644 --- a/pkg/volume/azure_file/BUILD +++ b/pkg/volume/azure_file/BUILD @@ -16,7 +16,6 @@ go_library( ], importpath = "k8s.io/kubernetes/pkg/volume/azure_file", deps = [ - "//pkg/features:go_default_library", "//pkg/util/mount:go_default_library", "//pkg/volume:go_default_library", "//pkg/volume/util:go_default_library", @@ -25,7 +24,6 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", "//staging/src/k8s.io/cloud-provider/volume/helpers:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/azure:go_default_library", diff --git a/pkg/volume/azure_file/azure_file.go b/pkg/volume/azure_file/azure_file.go index d6c25bc0c79..dfbb5ebd49f 100644 --- a/pkg/volume/azure_file/azure_file.go +++ b/pkg/volume/azure_file/azure_file.go @@ -28,11 +28,9 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - utilfeature "k8s.io/apiserver/pkg/util/feature" cloudprovider "k8s.io/cloud-provider" volumehelpers "k8s.io/cloud-provider/volume/helpers" "k8s.io/klog" - "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" volutil "k8s.io/kubernetes/pkg/volume/util" @@ -85,11 +83,6 @@ func (plugin *azureFilePlugin) CanSupport(spec *volume.Spec) bool { (spec.Volume != nil && spec.Volume.AzureFile != nil) } -func (plugin *azureFilePlugin) IsMigratedToCSI() bool { - return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) && - utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationAzureFile) -} - func (plugin *azureFilePlugin) RequiresRemount() bool { return false } diff --git a/pkg/volume/cephfs/cephfs.go b/pkg/volume/cephfs/cephfs.go index f8a6280c7c0..91142b5ee01 100644 --- a/pkg/volume/cephfs/cephfs.go +++ b/pkg/volume/cephfs/cephfs.go @@ -71,10 +71,6 @@ func (plugin *cephfsPlugin) CanSupport(spec *volume.Spec) bool { return (spec.Volume != nil && spec.Volume.CephFS != nil) || (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.CephFS != nil) } -func (plugin *cephfsPlugin) IsMigratedToCSI() bool { - return false -} - func (plugin *cephfsPlugin) RequiresRemount() bool { return false } diff --git a/pkg/volume/cinder/cinder.go b/pkg/volume/cinder/cinder.go index b95929a82c2..8aa3f336d87 100644 --- a/pkg/volume/cinder/cinder.go +++ b/pkg/volume/cinder/cinder.go @@ -111,11 +111,6 @@ func (plugin *cinderPlugin) CanSupport(spec *volume.Spec) bool { return (spec.Volume != nil && spec.Volume.Cinder != nil) || (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.Cinder != nil) } -func (plugin *cinderPlugin) IsMigratedToCSI() bool { - return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) && - utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationOpenStack) -} - func (plugin *cinderPlugin) RequiresRemount() bool { return false } diff --git a/pkg/volume/configmap/configmap.go b/pkg/volume/configmap/configmap.go index cdaa54ac014..450fe1f7f85 100644 --- a/pkg/volume/configmap/configmap.go +++ b/pkg/volume/configmap/configmap.go @@ -77,10 +77,6 @@ func (plugin *configMapPlugin) CanSupport(spec *volume.Spec) bool { return spec.Volume != nil && spec.Volume.ConfigMap != nil } -func (plugin *configMapPlugin) IsMigratedToCSI() bool { - return false -} - func (plugin *configMapPlugin) RequiresRemount() bool { return true } diff --git a/pkg/volume/csi/csi_plugin.go b/pkg/volume/csi/csi_plugin.go index 9b9deb063fa..d0e8f831bd4 100644 --- a/pkg/volume/csi/csi_plugin.go +++ b/pkg/volume/csi/csi_plugin.go @@ -314,10 +314,6 @@ func (p *csiPlugin) CanSupport(spec *volume.Spec) bool { return spec.PersistentVolume != nil && spec.PersistentVolume.Spec.CSI != nil } -func (p *csiPlugin) IsMigratedToCSI() bool { - return false -} - func (p *csiPlugin) RequiresRemount() bool { return false } diff --git a/pkg/volume/csimigration/BUILD b/pkg/volume/csimigration/BUILD new file mode 100644 index 00000000000..4b70f22351a --- /dev/null +++ b/pkg/volume/csimigration/BUILD @@ -0,0 +1,45 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["plugin_manager.go"], + importpath = "k8s.io/kubernetes/pkg/volume/csimigration", + visibility = ["//visibility:public"], + deps = [ + "//pkg/features:go_default_library", + "//pkg/volume:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/component-base/featuregate:go_default_library", + "//staging/src/k8s.io/csi-translation-lib/plugins:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) + +go_test( + name = "go_default_test", + srcs = ["plugin_manager_test.go"], + embed = [":go_default_library"], + deps = [ + "//pkg/features:go_default_library", + "//pkg/volume:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/component-base/featuregate:go_default_library", + "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library", + "//staging/src/k8s.io/csi-translation-lib:go_default_library", + ], +) diff --git a/pkg/volume/csimigration/plugin_manager.go b/pkg/volume/csimigration/plugin_manager.go new file mode 100644 index 00000000000..cab93506823 --- /dev/null +++ b/pkg/volume/csimigration/plugin_manager.go @@ -0,0 +1,156 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csimigration + +import ( + "errors" + "fmt" + + "k8s.io/api/core/v1" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/component-base/featuregate" + csilibplugins "k8s.io/csi-translation-lib/plugins" + "k8s.io/kubernetes/pkg/features" + "k8s.io/kubernetes/pkg/volume" +) + +// PluginNameMapper contains utility methods to retrieve names of plugins +// that support a spec, map intree <=> migrated CSI plugin names, etc +type PluginNameMapper interface { + GetInTreePluginNameFromSpec(pv *v1.PersistentVolume, vol *v1.Volume) (string, error) + GetCSINameFromInTreeName(pluginName string) (string, error) +} + +// PluginManager keeps track of migrated state of in-tree plugins +type PluginManager struct { + PluginNameMapper +} + +// NewPluginManager returns a new PluginManager instance +func NewPluginManager(m PluginNameMapper) PluginManager { + return PluginManager{ + PluginNameMapper: m, + } +} + +// IsMigrationCompleteForPlugin indicates whether CSI migration has been completed +// for a particular storage plugin +func (pm PluginManager) IsMigrationCompleteForPlugin(pluginName string) bool { + // CSIMigration feature and plugin specific migration feature flags should + // be enabled for plugin specific migration completion feature flags to be + // take effect + if !pm.IsMigrationEnabledForPlugin(pluginName) { + return false + } + + switch pluginName { + case csilibplugins.AWSEBSInTreePluginName: + return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationAWSComplete) + case csilibplugins.GCEPDInTreePluginName: + return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationGCEComplete) + case csilibplugins.AzureFileInTreePluginName: + return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationAzureFileComplete) + case csilibplugins.AzureDiskInTreePluginName: + return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationAzureDiskComplete) + case csilibplugins.CinderInTreePluginName: + return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationOpenStackComplete) + default: + return false + } +} + +// IsMigrationEnabledForPlugin indicates whether CSI migration has been enabled +// for a particular storage plugin +func (pm PluginManager) IsMigrationEnabledForPlugin(pluginName string) bool { + // CSIMigration feature should be enabled along with the plugin-specific one + if !utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) { + return false + } + + switch pluginName { + case csilibplugins.AWSEBSInTreePluginName: + return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationAWS) + case csilibplugins.GCEPDInTreePluginName: + return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationGCE) + case csilibplugins.AzureFileInTreePluginName: + return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationAzureFile) + case csilibplugins.AzureDiskInTreePluginName: + return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationAzureDisk) + case csilibplugins.CinderInTreePluginName: + return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationOpenStack) + default: + return false + } +} + +// IsMigratable indicates whether CSI migration has been enabled for a volume +// plugin that the spec refers to +func (pm PluginManager) IsMigratable(spec *volume.Spec) (bool, error) { + if spec == nil { + return false, fmt.Errorf("could not find if plugin is migratable because volume spec is nil") + } + + pluginName, _ := pm.GetInTreePluginNameFromSpec(spec.PersistentVolume, spec.Volume) + if pluginName == "" { + return false, nil + } + // found an in-tree plugin that supports the spec + return pm.IsMigrationEnabledForPlugin(pluginName), nil +} + +// InTreeToCSITranslator performs translation of Volume sources for PV and Volume objects +// from references to in-tree plugins to migrated CSI plugins +type InTreeToCSITranslator interface { + TranslateInTreePVToCSI(pv *v1.PersistentVolume) (*v1.PersistentVolume, error) + TranslateInTreeInlineVolumeToCSI(volume *v1.Volume) (*v1.PersistentVolume, error) +} + +// TranslateInTreeSpecToCSI translates a volume spec (either PV or inline volume) +// supported by an in-tree plugin to CSI +func TranslateInTreeSpecToCSI(spec *volume.Spec, translator InTreeToCSITranslator) (*volume.Spec, error) { + var csiPV *v1.PersistentVolume + var err error + inlineVolume := false + if spec.PersistentVolume != nil { + csiPV, err = translator.TranslateInTreePVToCSI(spec.PersistentVolume) + } else if spec.Volume != nil { + csiPV, err = translator.TranslateInTreeInlineVolumeToCSI(spec.Volume) + inlineVolume = true + } else { + err = errors.New("not a valid volume spec") + } + if err != nil { + return nil, fmt.Errorf("failed to translate in-tree pv to CSI: %v", err) + } + return &volume.Spec{ + PersistentVolume: csiPV, + ReadOnly: spec.ReadOnly, + InlineVolumeSpecForCSIMigration: inlineVolume, + }, nil +} + +// CheckMigrationFeatureFlags checks the configuration of feature flags related +// to CSI Migration is valid +func CheckMigrationFeatureFlags(f featuregate.FeatureGate, pluginMigration, pluginMigrationComplete featuregate.Feature) error { + if f.Enabled(pluginMigration) && !f.Enabled(features.CSIMigration) { + return fmt.Errorf("enabling %q requires CSIMigration to be enabled", pluginMigration) + } + if f.Enabled(pluginMigrationComplete) && !f.Enabled(pluginMigration) { + return fmt.Errorf("enabling %q requires %q to be enabled", pluginMigrationComplete, pluginMigration) + } + return nil +} diff --git a/pkg/volume/csimigration/plugin_manager_test.go b/pkg/volume/csimigration/plugin_manager_test.go new file mode 100644 index 00000000000..51621db9896 --- /dev/null +++ b/pkg/volume/csimigration/plugin_manager_test.go @@ -0,0 +1,325 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csimigration + +import ( + "fmt" + "testing" + + "k8s.io/api/core/v1" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/component-base/featuregate" + featuregatetesting "k8s.io/component-base/featuregate/testing" + csitrans "k8s.io/csi-translation-lib" + "k8s.io/kubernetes/pkg/features" + "k8s.io/kubernetes/pkg/volume" +) + +func TestIsMigratable(t *testing.T) { + testCases := []struct { + name string + pluginFeature featuregate.Feature + pluginFeatureEnabled bool + csiMigrationEnabled bool + isMigratable bool + spec *volume.Spec + }{ + { + name: "GCE PD PV source with CSIMigrationGCE enabled", + pluginFeature: features.CSIMigrationGCE, + pluginFeatureEnabled: true, + isMigratable: true, + csiMigrationEnabled: true, + spec: &volume.Spec{ + PersistentVolume: &v1.PersistentVolume{ + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{ + GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ + PDName: "test-disk", + FSType: "ext4", + Partition: 0, + ReadOnly: false, + }, + }, + }, + }, + }, + }, + { + name: "GCE PD PV Source with CSIMigrationGCE disabled", + pluginFeature: features.CSIMigrationGCE, + pluginFeatureEnabled: false, + isMigratable: false, + csiMigrationEnabled: true, + spec: &volume.Spec{ + PersistentVolume: &v1.PersistentVolume{ + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{ + GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ + PDName: "test-disk", + FSType: "ext4", + Partition: 0, + ReadOnly: false, + }, + }, + }, + }, + }, + }, + { + name: "AWS EBS PV with CSIMigrationAWS enabled", + pluginFeature: features.CSIMigrationAWS, + pluginFeatureEnabled: true, + isMigratable: true, + csiMigrationEnabled: true, + spec: &volume.Spec{ + PersistentVolume: &v1.PersistentVolume{ + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{ + AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{ + VolumeID: "vol01", + FSType: "ext3", + Partition: 1, + ReadOnly: true, + }, + }, + }, + }, + }, + }, + { + name: "AWS EBS PV with CSIMigration and CSIMigrationAWS disabled", + pluginFeature: features.CSIMigrationAWS, + pluginFeatureEnabled: false, + isMigratable: false, + csiMigrationEnabled: false, + spec: &volume.Spec{ + PersistentVolume: &v1.PersistentVolume{ + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{ + AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{ + VolumeID: "vol01", + FSType: "ext3", + Partition: 1, + ReadOnly: true, + }, + }, + }, + }, + }, + }, + } + csiTranslator := csitrans.New() + for _, test := range testCases { + pm := NewPluginManager(csiTranslator) + t.Run(fmt.Sprintf("Testing %v", test.name), func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIMigration, test.csiMigrationEnabled)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, test.pluginFeature, test.pluginFeatureEnabled)() + migratable, err := pm.IsMigratable(test.spec) + if migratable != test.isMigratable { + t.Errorf("Expected migratability of spec: %v does not match obtained migratability: %v", test.isMigratable, migratable) + } + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + }) + } +} + +func TestCheckMigrationFeatureFlags(t *testing.T) { + testCases := []struct { + name string + pluginFeature featuregate.Feature + pluginFeatureEnabled bool + csiMigrationEnabled bool + pluginFeatureComplete featuregate.Feature + pluginFeatureCompleteEnabled bool + result bool + }{ + { + name: "plugin specific feature flag enabled with migration flag disabled", + pluginFeature: features.CSIMigrationGCE, + pluginFeatureEnabled: true, + csiMigrationEnabled: false, + pluginFeatureComplete: features.CSIMigrationGCEComplete, + pluginFeatureCompleteEnabled: false, + result: false, + }, + { + name: "plugin specific complete flag enabled but plugin specific feature flag disabled", + pluginFeature: features.CSIMigrationAWS, + pluginFeatureEnabled: false, + csiMigrationEnabled: true, + pluginFeatureComplete: features.CSIMigrationAWSComplete, + pluginFeatureCompleteEnabled: true, + result: false, + }, + { + name: "plugin specific complete feature disabled but plugin specific migration feature and CSI migration flag enabled", + pluginFeature: features.CSIMigrationGCE, + pluginFeatureEnabled: true, + csiMigrationEnabled: true, + pluginFeatureComplete: features.CSIMigrationGCEComplete, + pluginFeatureCompleteEnabled: false, + result: true, + }, + { + name: "all features enabled", + pluginFeature: features.CSIMigrationAWS, + pluginFeatureEnabled: true, + csiMigrationEnabled: true, + pluginFeatureComplete: features.CSIMigrationAWSComplete, + pluginFeatureCompleteEnabled: true, + result: true, + }, + } + for _, test := range testCases { + t.Run(fmt.Sprintf("Testing %v", test.name), func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIMigration, test.csiMigrationEnabled)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, test.pluginFeature, test.pluginFeatureEnabled)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, test.pluginFeatureComplete, test.pluginFeatureCompleteEnabled)() + err := CheckMigrationFeatureFlags(utilfeature.DefaultFeatureGate, test.pluginFeature, test.pluginFeatureComplete) + if err != nil && test.result == true { + t.Errorf("Unexpected error: %v", err) + } + if err == nil && test.result == false { + t.Errorf("Unexpected validation pass") + } + }) + } +} + +func TestMigrationFeatureFlagStatus(t *testing.T) { + testCases := []struct { + name string + pluginName string + csiMigrationEnabled bool + pluginFeature featuregate.Feature + pluginFeatureEnabled bool + pluginFeatureComplete featuregate.Feature + pluginFeatureCompleteEnabled bool + csiMigrationResult bool + csiMigrationCompleteResult bool + }{ + { + name: "gce-pd migration flag disabled and migration-complete flag disabled with CSI migration flag disabled", + pluginName: "kubernetes.io/gce-pd", + pluginFeature: features.CSIMigrationGCE, + pluginFeatureEnabled: false, + csiMigrationEnabled: false, + pluginFeatureComplete: features.CSIMigrationGCEComplete, + pluginFeatureCompleteEnabled: false, + csiMigrationResult: false, + csiMigrationCompleteResult: false, + }, + { + name: "gce-pd migration flag disabled and migration-complete flag disabled with CSI migration flag enabled", + pluginName: "kubernetes.io/gce-pd", + pluginFeature: features.CSIMigrationGCE, + pluginFeatureEnabled: false, + csiMigrationEnabled: true, + pluginFeatureComplete: features.CSIMigrationGCEComplete, + pluginFeatureCompleteEnabled: false, + csiMigrationResult: false, + csiMigrationCompleteResult: false, + }, + { + name: "gce-pd migration flag enabled and migration-complete flag disabled with CSI migration flag enabled", + pluginName: "kubernetes.io/gce-pd", + pluginFeature: features.CSIMigrationGCE, + pluginFeatureEnabled: true, + csiMigrationEnabled: true, + pluginFeatureComplete: features.CSIMigrationGCEComplete, + pluginFeatureCompleteEnabled: false, + csiMigrationResult: true, + csiMigrationCompleteResult: false, + }, + { + name: "gce-pd migration flag enabled and migration-complete flag enabled with CSI migration flag enabled", + pluginName: "kubernetes.io/gce-pd", + pluginFeature: features.CSIMigrationGCE, + pluginFeatureEnabled: true, + csiMigrationEnabled: true, + pluginFeatureComplete: features.CSIMigrationGCEComplete, + pluginFeatureCompleteEnabled: true, + csiMigrationResult: true, + csiMigrationCompleteResult: true, + }, + { + name: "aws-ebs migration flag disabled and migration-complete flag disabled with CSI migration flag disabled", + pluginName: "kubernetes.io/aws-ebs", + pluginFeature: features.CSIMigrationAWS, + pluginFeatureEnabled: false, + csiMigrationEnabled: false, + pluginFeatureComplete: features.CSIMigrationAWSComplete, + pluginFeatureCompleteEnabled: false, + csiMigrationResult: false, + csiMigrationCompleteResult: false, + }, + { + name: "aws-ebs migration flag disabled and migration-complete flag disabled with CSI migration flag enabled", + pluginName: "kubernetes.io/aws-ebs", + pluginFeature: features.CSIMigrationAWS, + pluginFeatureEnabled: false, + csiMigrationEnabled: true, + pluginFeatureComplete: features.CSIMigrationAWSComplete, + pluginFeatureCompleteEnabled: false, + csiMigrationResult: false, + csiMigrationCompleteResult: false, + }, + { + name: "aws-ebs migration flag enabled and migration-complete flag disabled with CSI migration flag enabled", + pluginName: "kubernetes.io/aws-ebs", + pluginFeature: features.CSIMigrationAWS, + pluginFeatureEnabled: true, + csiMigrationEnabled: true, + pluginFeatureComplete: features.CSIMigrationAWSComplete, + pluginFeatureCompleteEnabled: false, + csiMigrationResult: true, + csiMigrationCompleteResult: false, + }, + { + name: "aws-ebs migration flag enabled and migration-complete flag enabled with CSI migration flag enabled", + pluginName: "kubernetes.io/aws-ebs", + pluginFeature: features.CSIMigrationAWS, + pluginFeatureEnabled: true, + csiMigrationEnabled: true, + pluginFeatureComplete: features.CSIMigrationAWSComplete, + pluginFeatureCompleteEnabled: true, + csiMigrationResult: true, + csiMigrationCompleteResult: true, + }, + } + csiTranslator := csitrans.New() + for _, test := range testCases { + pm := NewPluginManager(csiTranslator) + t.Run(fmt.Sprintf("Testing %v", test.name), func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIMigration, test.csiMigrationEnabled)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, test.pluginFeature, test.pluginFeatureEnabled)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, test.pluginFeatureComplete, test.pluginFeatureCompleteEnabled)() + + csiMigrationResult := pm.IsMigrationEnabledForPlugin(test.pluginName) + if csiMigrationResult != test.csiMigrationResult { + t.Errorf("Expected migratability of plugin %v: %v does not match obtained migratability: %v", test.pluginName, test.csiMigrationResult, csiMigrationResult) + } + csiMigrationCompleteResult := pm.IsMigrationCompleteForPlugin(test.pluginName) + if csiMigrationCompleteResult != test.csiMigrationCompleteResult { + t.Errorf("Expected migration complete status of plugin: %v: %v does not match obtained migratability: %v", test.pluginName, test.csiMigrationCompleteResult, csiMigrationResult) + } + }) + } +} diff --git a/pkg/volume/downwardapi/downwardapi.go b/pkg/volume/downwardapi/downwardapi.go index 8d2aabb019a..12746696888 100644 --- a/pkg/volume/downwardapi/downwardapi.go +++ b/pkg/volume/downwardapi/downwardapi.go @@ -80,10 +80,6 @@ func (plugin *downwardAPIPlugin) CanSupport(spec *volume.Spec) bool { return spec.Volume != nil && spec.Volume.DownwardAPI != nil } -func (plugin *downwardAPIPlugin) IsMigratedToCSI() bool { - return false -} - func (plugin *downwardAPIPlugin) RequiresRemount() bool { return true } diff --git a/pkg/volume/emptydir/empty_dir.go b/pkg/volume/emptydir/empty_dir.go index e179a8565da..4ac5f2a18df 100644 --- a/pkg/volume/emptydir/empty_dir.go +++ b/pkg/volume/emptydir/empty_dir.go @@ -90,10 +90,6 @@ func (plugin *emptyDirPlugin) CanSupport(spec *volume.Spec) bool { return false } -func (plugin *emptyDirPlugin) IsMigratedToCSI() bool { - return false -} - func (plugin *emptyDirPlugin) RequiresRemount() bool { return false } diff --git a/pkg/volume/fc/fc.go b/pkg/volume/fc/fc.go index 5ad7bfc427c..cf4a1d0508d 100644 --- a/pkg/volume/fc/fc.go +++ b/pkg/volume/fc/fc.go @@ -88,10 +88,6 @@ func (plugin *fcPlugin) CanSupport(spec *volume.Spec) bool { return (spec.Volume != nil && spec.Volume.FC != nil) || (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.FC != nil) } -func (plugin *fcPlugin) IsMigratedToCSI() bool { - return false -} - func (plugin *fcPlugin) RequiresRemount() bool { return false } diff --git a/pkg/volume/flexvolume/plugin.go b/pkg/volume/flexvolume/plugin.go index 10fccbf17e6..a1204098e19 100644 --- a/pkg/volume/flexvolume/plugin.go +++ b/pkg/volume/flexvolume/plugin.go @@ -148,10 +148,6 @@ func (plugin *flexVolumePlugin) CanSupport(spec *volume.Spec) bool { return sourceDriver == plugin.driverName } -func (plugin *flexVolumePlugin) IsMigratedToCSI() bool { - return false -} - // RequiresRemount is part of the volume.VolumePlugin interface. func (plugin *flexVolumePlugin) RequiresRemount() bool { return false diff --git a/pkg/volume/flocker/flocker.go b/pkg/volume/flocker/flocker.go index c69a96776d0..353bf4c0148 100644 --- a/pkg/volume/flocker/flocker.go +++ b/pkg/volume/flocker/flocker.go @@ -107,10 +107,6 @@ func (p *flockerPlugin) CanSupport(spec *volume.Spec) bool { (spec.Volume != nil && spec.Volume.Flocker != nil) } -func (p *flockerPlugin) IsMigratedToCSI() bool { - return false -} - func (p *flockerPlugin) RequiresRemount() bool { return false } diff --git a/pkg/volume/gcepd/gce_pd.go b/pkg/volume/gcepd/gce_pd.go index 9f770cdf69d..cbc093669c1 100644 --- a/pkg/volume/gcepd/gce_pd.go +++ b/pkg/volume/gcepd/gce_pd.go @@ -100,11 +100,6 @@ func (plugin *gcePersistentDiskPlugin) CanSupport(spec *volume.Spec) bool { (spec.Volume != nil && spec.Volume.GCEPersistentDisk != nil) } -func (plugin *gcePersistentDiskPlugin) IsMigratedToCSI() bool { - return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) && - utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationGCE) -} - func (plugin *gcePersistentDiskPlugin) RequiresRemount() bool { return false } diff --git a/pkg/volume/git_repo/git_repo.go b/pkg/volume/git_repo/git_repo.go index 932296c2845..aee38f5bbf9 100644 --- a/pkg/volume/git_repo/git_repo.go +++ b/pkg/volume/git_repo/git_repo.go @@ -77,10 +77,6 @@ func (plugin *gitRepoPlugin) CanSupport(spec *volume.Spec) bool { return spec.Volume != nil && spec.Volume.GitRepo != nil } -func (plugin *gitRepoPlugin) IsMigratedToCSI() bool { - return false -} - func (plugin *gitRepoPlugin) RequiresRemount() bool { return false } diff --git a/pkg/volume/glusterfs/glusterfs.go b/pkg/volume/glusterfs/glusterfs.go index 0169935df74..72e4fb68ae2 100644 --- a/pkg/volume/glusterfs/glusterfs.go +++ b/pkg/volume/glusterfs/glusterfs.go @@ -118,10 +118,6 @@ func (plugin *glusterfsPlugin) CanSupport(spec *volume.Spec) bool { (spec.Volume != nil && spec.Volume.Glusterfs != nil) } -func (plugin *glusterfsPlugin) IsMigratedToCSI() bool { - return false -} - func (plugin *glusterfsPlugin) RequiresRemount() bool { return false } diff --git a/pkg/volume/hostpath/host_path.go b/pkg/volume/hostpath/host_path.go index ecc66675257..54bc282fb65 100644 --- a/pkg/volume/hostpath/host_path.go +++ b/pkg/volume/hostpath/host_path.go @@ -84,10 +84,6 @@ func (plugin *hostPathPlugin) CanSupport(spec *volume.Spec) bool { (spec.Volume != nil && spec.Volume.HostPath != nil) } -func (plugin *hostPathPlugin) IsMigratedToCSI() bool { - return false -} - func (plugin *hostPathPlugin) RequiresRemount() bool { return false } diff --git a/pkg/volume/iscsi/iscsi.go b/pkg/volume/iscsi/iscsi.go index 12ed5f0a7be..f190359a5ca 100644 --- a/pkg/volume/iscsi/iscsi.go +++ b/pkg/volume/iscsi/iscsi.go @@ -78,10 +78,6 @@ func (plugin *iscsiPlugin) CanSupport(spec *volume.Spec) bool { return (spec.Volume != nil && spec.Volume.ISCSI != nil) || (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.ISCSI != nil) } -func (plugin *iscsiPlugin) IsMigratedToCSI() bool { - return false -} - func (plugin *iscsiPlugin) RequiresRemount() bool { return false } diff --git a/pkg/volume/local/local.go b/pkg/volume/local/local.go index 058d927c5ac..0f8beb0641b 100644 --- a/pkg/volume/local/local.go +++ b/pkg/volume/local/local.go @@ -83,10 +83,6 @@ func (plugin *localVolumePlugin) CanSupport(spec *volume.Spec) bool { return (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.Local != nil) } -func (plugin *localVolumePlugin) IsMigratedToCSI() bool { - return false -} - func (plugin *localVolumePlugin) RequiresRemount() bool { return false } diff --git a/pkg/volume/nfs/nfs.go b/pkg/volume/nfs/nfs.go index 9a1bb45cb3d..b6d407333fd 100644 --- a/pkg/volume/nfs/nfs.go +++ b/pkg/volume/nfs/nfs.go @@ -89,10 +89,6 @@ func (plugin *nfsPlugin) CanSupport(spec *volume.Spec) bool { (spec.Volume != nil && spec.Volume.NFS != nil) } -func (plugin *nfsPlugin) IsMigratedToCSI() bool { - return false -} - func (plugin *nfsPlugin) RequiresRemount() bool { return false } diff --git a/pkg/volume/noop_expandable_plugin.go b/pkg/volume/noop_expandable_plugin.go index e4052d31387..3d3d5e1dfd7 100644 --- a/pkg/volume/noop_expandable_plugin.go +++ b/pkg/volume/noop_expandable_plugin.go @@ -48,10 +48,6 @@ func (n *noopExpandableVolumePluginInstance) CanSupport(spec *Spec) bool { return true } -func (n *noopExpandableVolumePluginInstance) IsMigratedToCSI() bool { - return false -} - func (n *noopExpandableVolumePluginInstance) RequiresRemount() bool { return false } diff --git a/pkg/volume/plugins.go b/pkg/volume/plugins.go index f122a994ef6..08de0e9af0d 100644 --- a/pkg/volume/plugins.go +++ b/pkg/volume/plugins.go @@ -159,10 +159,6 @@ type VolumePlugin interface { // const. CanSupport(spec *Spec) bool - // IsMigratedToCSI tests whether a CSIDriver implements this plugin's - // functionality - IsMigratedToCSI() bool - // RequiresRemount returns true if this plugin requires mount calls to be // reexecuted. Atomically updating volumes, like Downward API, depend on // this to update the contents of the volume. @@ -693,39 +689,6 @@ func (pm *VolumePluginMgr) FindPluginBySpec(spec *Spec) (VolumePlugin, error) { return matches[0], nil } -// IsPluginMigratableBySpec looks for a plugin that can support a given volume -// specification and whether that plugin is Migratable. If no plugins can -// support or more than one plugin can support it, return error. -func (pm *VolumePluginMgr) IsPluginMigratableBySpec(spec *Spec) (bool, error) { - pm.mutex.Lock() - defer pm.mutex.Unlock() - - if spec == nil { - return false, fmt.Errorf("could not find if plugin is migratable because volume spec is nil") - } - - matches := []VolumePlugin{} - for _, v := range pm.plugins { - if v.CanSupport(spec) { - matches = append(matches, v) - } - } - - if len(matches) == 0 { - // Not a known plugin (flex) in which case it is not migratable - return false, nil - } - if len(matches) > 1 { - matchedPluginNames := []string{} - for _, plugin := range matches { - matchedPluginNames = append(matchedPluginNames, plugin.GetPluginName()) - } - return false, fmt.Errorf("multiple volume plugins matched: %s", strings.Join(matchedPluginNames, ",")) - } - - return matches[0].IsMigratedToCSI(), nil -} - // FindPluginByName fetches a plugin by name or by legacy name. If no plugin // is found, returns error. func (pm *VolumePluginMgr) FindPluginByName(name string) (VolumePlugin, error) { diff --git a/pkg/volume/plugins_test.go b/pkg/volume/plugins_test.go index dd0a7397d84..167a47e6fa3 100644 --- a/pkg/volume/plugins_test.go +++ b/pkg/volume/plugins_test.go @@ -75,10 +75,6 @@ func (plugin *testPlugins) CanSupport(spec *Spec) bool { return true } -func (plugin *testPlugins) IsMigratedToCSI() bool { - return false -} - func (plugin *testPlugins) RequiresRemount() bool { return false } diff --git a/pkg/volume/portworx/portworx.go b/pkg/volume/portworx/portworx.go index c9c30f4bd49..8c592b869de 100644 --- a/pkg/volume/portworx/portworx.go +++ b/pkg/volume/portworx/portworx.go @@ -95,10 +95,6 @@ func (plugin *portworxVolumePlugin) CanSupport(spec *volume.Spec) bool { (spec.Volume != nil && spec.Volume.PortworxVolume != nil) } -func (plugin *portworxVolumePlugin) IsMigratedToCSI() bool { - return false -} - func (plugin *portworxVolumePlugin) RequiresRemount() bool { return false } diff --git a/pkg/volume/projected/projected.go b/pkg/volume/projected/projected.go index 7aaee7138ac..65e1ac5e2f1 100644 --- a/pkg/volume/projected/projected.go +++ b/pkg/volume/projected/projected.go @@ -95,10 +95,6 @@ func (plugin *projectedPlugin) CanSupport(spec *volume.Spec) bool { return spec.Volume != nil && spec.Volume.Projected != nil } -func (plugin *projectedPlugin) IsMigratedToCSI() bool { - return false -} - func (plugin *projectedPlugin) RequiresRemount() bool { return true } diff --git a/pkg/volume/quobyte/quobyte.go b/pkg/volume/quobyte/quobyte.go index 5397e8e0be1..25ffb5b429c 100644 --- a/pkg/volume/quobyte/quobyte.go +++ b/pkg/volume/quobyte/quobyte.go @@ -110,10 +110,6 @@ func (plugin *quobytePlugin) CanSupport(spec *volume.Spec) bool { return false } -func (plugin *quobytePlugin) IsMigratedToCSI() bool { - return false -} - func (plugin *quobytePlugin) RequiresRemount() bool { return false } diff --git a/pkg/volume/rbd/rbd.go b/pkg/volume/rbd/rbd.go index 380f1711bf0..eb6a808023e 100644 --- a/pkg/volume/rbd/rbd.go +++ b/pkg/volume/rbd/rbd.go @@ -108,10 +108,6 @@ func (plugin *rbdPlugin) CanSupport(spec *volume.Spec) bool { return (spec.Volume != nil && spec.Volume.RBD != nil) || (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.RBD != nil) } -func (plugin *rbdPlugin) IsMigratedToCSI() bool { - return false -} - func (plugin *rbdPlugin) RequiresRemount() bool { return false } diff --git a/pkg/volume/scaleio/sio_plugin.go b/pkg/volume/scaleio/sio_plugin.go index 7b045376dd5..b1dd18776d6 100644 --- a/pkg/volume/scaleio/sio_plugin.go +++ b/pkg/volume/scaleio/sio_plugin.go @@ -72,10 +72,6 @@ func (p *sioPlugin) CanSupport(spec *volume.Spec) bool { (spec.Volume != nil && spec.Volume.ScaleIO != nil) } -func (p *sioPlugin) IsMigratedToCSI() bool { - return false -} - func (p *sioPlugin) RequiresRemount() bool { return false } diff --git a/pkg/volume/secret/secret.go b/pkg/volume/secret/secret.go index 90345d3f96a..5679af20e56 100644 --- a/pkg/volume/secret/secret.go +++ b/pkg/volume/secret/secret.go @@ -80,10 +80,6 @@ func (plugin *secretPlugin) CanSupport(spec *volume.Spec) bool { return spec.Volume != nil && spec.Volume.Secret != nil } -func (plugin *secretPlugin) IsMigratedToCSI() bool { - return false -} - func (plugin *secretPlugin) RequiresRemount() bool { return true } diff --git a/pkg/volume/storageos/storageos.go b/pkg/volume/storageos/storageos.go index 8350b44b786..990d275a42a 100644 --- a/pkg/volume/storageos/storageos.go +++ b/pkg/volume/storageos/storageos.go @@ -91,10 +91,6 @@ func (plugin *storageosPlugin) CanSupport(spec *volume.Spec) bool { (spec.Volume != nil && spec.Volume.StorageOS != nil) } -func (plugin *storageosPlugin) IsMigratedToCSI() bool { - return false -} - func (plugin *storageosPlugin) RequiresRemount() bool { return false } diff --git a/pkg/volume/testing/testing.go b/pkg/volume/testing/testing.go index 95bf7ebc1d2..8e0abbf6c21 100644 --- a/pkg/volume/testing/testing.go +++ b/pkg/volume/testing/testing.go @@ -411,10 +411,6 @@ func (plugin *FakeVolumePlugin) CanSupport(spec *Spec) bool { return true } -func (plugin *FakeVolumePlugin) IsMigratedToCSI() bool { - return false -} - func (plugin *FakeVolumePlugin) RequiresRemount() bool { return false } @@ -653,10 +649,6 @@ func (f *FakeBasicVolumePlugin) CanSupport(spec *Spec) bool { return strings.HasPrefix(spec.Name(), f.GetPluginName()) } -func (plugin *FakeBasicVolumePlugin) IsMigratedToCSI() bool { - return false -} - func (f *FakeBasicVolumePlugin) ConstructVolumeSpec(ame, mountPath string) (*Spec, error) { return f.Plugin.ConstructVolumeSpec(ame, mountPath) } @@ -750,10 +742,6 @@ func (plugin *FakeFileVolumePlugin) CanSupport(spec *Spec) bool { return true } -func (plugin *FakeFileVolumePlugin) IsMigratedToCSI() bool { - return false -} - func (plugin *FakeFileVolumePlugin) RequiresRemount() bool { return false } diff --git a/pkg/volume/util/operationexecutor/BUILD b/pkg/volume/util/operationexecutor/BUILD index b0ccffe22f7..c600f89ed53 100644 --- a/pkg/volume/util/operationexecutor/BUILD +++ b/pkg/volume/util/operationexecutor/BUILD @@ -19,7 +19,6 @@ go_library( "//pkg/kubelet/events:go_default_library", "//pkg/util/mount:go_default_library", "//pkg/volume:go_default_library", - "//pkg/volume/csi:go_default_library", "//pkg/volume/util:go_default_library", "//pkg/volume/util/hostutil:go_default_library", "//pkg/volume/util/nestedpendingoperations:go_default_library", @@ -30,7 +29,6 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", @@ -48,10 +46,8 @@ go_test( ], embed = [":go_default_library"], deps = [ - "//pkg/features:go_default_library", "//pkg/volume:go_default_library", "//pkg/volume/awsebs:go_default_library", - "//pkg/volume/csi:go_default_library", "//pkg/volume/csi/testing:go_default_library", "//pkg/volume/gcepd:go_default_library", "//pkg/volume/testing:go_default_library", @@ -61,11 +57,8 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", - "//staging/src/k8s.io/component-base/featuregate:go_default_library", - "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library", "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library", "//staging/src/k8s.io/csi-translation-lib:go_default_library", "//staging/src/k8s.io/csi-translation-lib/plugins:go_default_library", diff --git a/pkg/volume/util/operationexecutor/operation_executor.go b/pkg/volume/util/operationexecutor/operation_executor.go index fbad284378b..18dd1f5655e 100644 --- a/pkg/volume/util/operationexecutor/operation_executor.go +++ b/pkg/volume/util/operationexecutor/operation_executor.go @@ -31,7 +31,6 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" - "k8s.io/kubernetes/pkg/volume/csi" "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/hostutil" "k8s.io/kubernetes/pkg/volume/util/nestedpendingoperations" @@ -640,45 +639,25 @@ func (oe *operationExecutor) VerifyVolumesAreAttached( continue } - // Migration: Must also check the Node since Attach would have been done with in-tree if node is not using Migration - nu, err := nodeUsingCSIPlugin(oe.operationGenerator.GetCSITranslator(), oe.operationGenerator.GetVolumePluginMgr(), volumeAttached.VolumeSpec, node) + volumePlugin, err := + oe.operationGenerator.GetVolumePluginMgr().FindPluginBySpec(volumeAttached.VolumeSpec) if err != nil { - klog.Errorf(volumeAttached.GenerateErrorDetailed("VolumesAreAttached.NodeUsingCSIPlugin failed", err).Error()) + klog.Errorf( + "VolumesAreAttached.FindPluginBySpec failed for volume %q (spec.Name: %q) on node %q with error: %v", + volumeAttached.VolumeName, + volumeAttached.VolumeSpec.Name(), + volumeAttached.NodeName, + err) continue } - - var volumePlugin volume.VolumePlugin - if useCSIPlugin(oe.operationGenerator.GetCSITranslator(), oe.operationGenerator.GetVolumePluginMgr(), volumeAttached.VolumeSpec) && nu { - // The volume represented by this spec is CSI and thus should be migrated - volumePlugin, err = oe.operationGenerator.GetVolumePluginMgr().FindPluginByName(csi.CSIPluginName) - if err != nil || volumePlugin == nil { - klog.Errorf( - "VolumesAreAttached.Name failed for volume %q (spec.Name: %q) on node %q with error: %v", - volumeAttached.VolumeName, - volumeAttached.VolumeSpec.Name(), - volumeAttached.NodeName, - err) - continue - } - - csiSpec, err := translateSpec(oe.operationGenerator.GetCSITranslator(), volumeAttached.VolumeSpec) - if err != nil { - klog.Errorf(volumeAttached.GenerateErrorDetailed("VolumesAreAttached.TranslateSpec failed", err).Error()) - continue - } - volumeAttached.VolumeSpec = csiSpec - } else { - volumePlugin, err = - oe.operationGenerator.GetVolumePluginMgr().FindPluginBySpec(volumeAttached.VolumeSpec) - if err != nil || volumePlugin == nil { - klog.Errorf( - "VolumesAreAttached.FindPluginBySpec failed for volume %q (spec.Name: %q) on node %q with error: %v", - volumeAttached.VolumeName, - volumeAttached.VolumeSpec.Name(), - volumeAttached.NodeName, - err) - continue - } + if volumePlugin == nil { + // should never happen since FindPluginBySpec always returns error if volumePlugin = nil + klog.Errorf( + "Failed to find volume plugin for volume %q (spec.Name: %q) on node %q", + volumeAttached.VolumeName, + volumeAttached.VolumeSpec.Name(), + volumeAttached.NodeName) + continue } pluginName := volumePlugin.GetPluginName() diff --git a/pkg/volume/util/operationexecutor/operation_generator.go b/pkg/volume/util/operationexecutor/operation_generator.go index 0d91d2ba2a0..91b86de990d 100644 --- a/pkg/volume/util/operationexecutor/operation_generator.go +++ b/pkg/volume/util/operationexecutor/operation_generator.go @@ -27,7 +27,6 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/sets" utilfeature "k8s.io/apiserver/pkg/util/feature" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/record" @@ -37,7 +36,6 @@ import ( "k8s.io/kubernetes/pkg/features" kevents "k8s.io/kubernetes/pkg/kubelet/events" "k8s.io/kubernetes/pkg/volume" - "k8s.io/kubernetes/pkg/volume/csi" "k8s.io/kubernetes/pkg/volume/util" ioutil "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/hostutil" @@ -170,38 +168,12 @@ func (og *operationGenerator) GenerateVolumesAreAttachedFunc( klog.Errorf("VerifyVolumesAreAttached.GenerateVolumesAreAttachedFunc: nil spec for volume %s", volumeAttached.VolumeName) continue } - - // Migration: Must also check the Node since Attach would have been done with in-tree if node is not using Migration - nu, err := nodeUsingCSIPlugin(og.translator, og.volumePluginMgr, volumeAttached.VolumeSpec, nodeName) - if err != nil { - klog.Errorf(volumeAttached.GenerateErrorDetailed("VolumesAreAttached.NodeUsingCSIPlugin failed", err).Error()) + volumePlugin, err := + og.volumePluginMgr.FindPluginBySpec(volumeAttached.VolumeSpec) + if err != nil || volumePlugin == nil { + klog.Errorf(volumeAttached.GenerateErrorDetailed("VolumesAreAttached.FindPluginBySpec failed", err).Error()) continue } - - var volumePlugin volume.VolumePlugin - if useCSIPlugin(og.translator, og.volumePluginMgr, volumeAttached.VolumeSpec) && nu { - // The volume represented by this spec is CSI and thus should be migrated - volumePlugin, err = og.volumePluginMgr.FindPluginByName(csi.CSIPluginName) - if err != nil || volumePlugin == nil { - klog.Errorf(volumeAttached.GenerateErrorDetailed("VolumesAreAttached.FindPluginByName failed", err).Error()) - continue - } - - csiSpec, err := translateSpec(og.translator, volumeAttached.VolumeSpec) - if err != nil { - klog.Errorf(volumeAttached.GenerateErrorDetailed("VolumesAreAttached.TranslateSpec failed", err).Error()) - continue - } - volumeAttached.VolumeSpec = csiSpec - } else { - volumePlugin, err = - og.volumePluginMgr.FindPluginBySpec(volumeAttached.VolumeSpec) - if err != nil || volumePlugin == nil { - klog.Errorf(volumeAttached.GenerateErrorDetailed("VolumesAreAttached.FindPluginBySpec failed", err).Error()) - continue - } - } - volumeSpecList, pluginExists := volumesPerPlugin[volumePlugin.GetPluginName()] if !pluginExists { volumeSpecList = []*volume.Spec{} @@ -345,34 +317,12 @@ func (og *operationGenerator) GenerateBulkVolumeVerifyFunc( func (og *operationGenerator) GenerateAttachVolumeFunc( volumeToAttach VolumeToAttach, actualStateOfWorld ActualStateOfWorldAttacherUpdater) volumetypes.GeneratedOperations { - originalSpec := volumeToAttach.VolumeSpec + attachVolumeFunc := func() (error, error) { - var attachableVolumePlugin volume.AttachableVolumePlugin - - nu, err := nodeUsingCSIPlugin(og.translator, og.volumePluginMgr, volumeToAttach.VolumeSpec, volumeToAttach.NodeName) - if err != nil { - return volumeToAttach.GenerateError("AttachVolume.NodeUsingCSIPlugin failed", err) - } - - // useCSIPlugin will check both CSIMigration and the plugin specific feature gates - if useCSIPlugin(og.translator, og.volumePluginMgr, volumeToAttach.VolumeSpec) && nu { - // The volume represented by this spec is CSI and thus should be migrated - attachableVolumePlugin, err = og.volumePluginMgr.FindAttachablePluginByName(csi.CSIPluginName) - if err != nil || attachableVolumePlugin == nil { - return volumeToAttach.GenerateError("AttachVolume.FindAttachablePluginByName failed", err) - } - - csiSpec, err := translateSpec(og.translator, volumeToAttach.VolumeSpec) - if err != nil { - return volumeToAttach.GenerateError("AttachVolume.TranslateSpec failed", err) - } - volumeToAttach.VolumeSpec = csiSpec - } else { - attachableVolumePlugin, err = - og.volumePluginMgr.FindAttachablePluginBySpec(volumeToAttach.VolumeSpec) - if err != nil || attachableVolumePlugin == nil { - return volumeToAttach.GenerateError("AttachVolume.FindAttachablePluginBySpec failed", err) - } + attachableVolumePlugin, err := + og.volumePluginMgr.FindAttachablePluginBySpec(volumeToAttach.VolumeSpec) + if err != nil || attachableVolumePlugin == nil { + return volumeToAttach.GenerateError("AttachVolume.FindAttachablePluginBySpec failed", err) } volumeAttacher, newAttacherErr := attachableVolumePlugin.NewAttacher() @@ -391,7 +341,7 @@ func (og *operationGenerator) GenerateAttachVolumeFunc( } addErr := actualStateOfWorld.MarkVolumeAsUncertain( v1.UniqueVolumeName(""), - originalSpec, + volumeToAttach.VolumeSpec, uncertainNode) if addErr != nil { klog.Errorf("AttachVolume.MarkVolumeAsUncertain fail to add the volume %q to actual state with %s", volumeToAttach.VolumeName, addErr) @@ -410,7 +360,7 @@ func (og *operationGenerator) GenerateAttachVolumeFunc( // Update actual state of world addVolumeNodeErr := actualStateOfWorld.MarkVolumeAsAttached( - v1.UniqueVolumeName(""), originalSpec, volumeToAttach.NodeName, devicePath) + v1.UniqueVolumeName(""), volumeToAttach.VolumeSpec, volumeToAttach.NodeName, devicePath) if addVolumeNodeErr != nil { // On failure, return error. Caller will log and retry. return volumeToAttach.GenerateError("AttachVolume.MarkVolumeAsAttached failed", addVolumeNodeErr) @@ -428,30 +378,7 @@ func (og *operationGenerator) GenerateAttachVolumeFunc( } attachableVolumePluginName := unknownAttachableVolumePlugin - // TODO(dyzz) Ignoring this error means that if the plugin is migrated and - // any transient error is encountered (API unavailable, driver not installed) - // the operation will have it's metric registered with the in-tree plugin instead - // of the CSI Driver we migrated to. Fixing this requires a larger refactor that - // involves determining the plugin_name for the metric generating "CompleteFunc" - // during the actual "OperationFunc" and not during this generation function - nu, err := nodeUsingCSIPlugin(og.translator, og.volumePluginMgr, volumeToAttach.VolumeSpec, volumeToAttach.NodeName) - if err != nil { - klog.Errorf("GenerateAttachVolumeFunc failed to check if node is using CSI Plugin, metric for this operation may be inaccurate: %v", err) - } - - // Need to translate the spec here if the plugin is migrated so that the metrics - // emitted show the correct (migrated) plugin - if useCSIPlugin(og.translator, og.volumePluginMgr, volumeToAttach.VolumeSpec) && nu { - csiSpec, err := translateSpec(og.translator, volumeToAttach.VolumeSpec) - if err == nil { - volumeToAttach.VolumeSpec = csiSpec - } - // If we have an error here we ignore it, the metric emitted will then be for the - // in-tree plugin. This error case(skipped one) will also trigger an error - // while the generated function is executed. And those errors will be handled during the execution of the generated - // function with a back off policy. - } // Get attacher plugin attachableVolumePlugin, err := og.volumePluginMgr.FindAttachablePluginBySpec(volumeToAttach.VolumeSpec) @@ -489,32 +416,10 @@ func (og *operationGenerator) GenerateDetachVolumeFunc( var err error if volumeToDetach.VolumeSpec != nil { - // Get attacher plugin - nu, err := nodeUsingCSIPlugin(og.translator, og.volumePluginMgr, volumeToDetach.VolumeSpec, volumeToDetach.NodeName) - if err != nil { - return volumetypes.GeneratedOperations{}, volumeToDetach.GenerateErrorDetailed("DetachVolume.NodeUsingCSIPlugin failed", err) - } - - // useCSIPlugin will check both CSIMigration and the plugin specific feature gate - if useCSIPlugin(og.translator, og.volumePluginMgr, volumeToDetach.VolumeSpec) && nu { - // The volume represented by this spec is CSI and thus should be migrated - attachableVolumePlugin, err = og.volumePluginMgr.FindAttachablePluginByName(csi.CSIPluginName) - if err != nil || attachableVolumePlugin == nil { - return volumetypes.GeneratedOperations{}, volumeToDetach.GenerateErrorDetailed("DetachVolume.FindAttachablePluginBySpec failed", err) - } - - csiSpec, err := translateSpec(og.translator, volumeToDetach.VolumeSpec) - if err != nil { - return volumetypes.GeneratedOperations{}, volumeToDetach.GenerateErrorDetailed("DetachVolume.TranslateSpec failed", err) - } - - volumeToDetach.VolumeSpec = csiSpec - } else { - attachableVolumePlugin, err = - og.volumePluginMgr.FindAttachablePluginBySpec(volumeToDetach.VolumeSpec) - if err != nil || attachableVolumePlugin == nil { - return volumetypes.GeneratedOperations{}, volumeToDetach.GenerateErrorDetailed("DetachVolume.FindAttachablePluginBySpec failed", err) - } + attachableVolumePlugin, err = + og.volumePluginMgr.FindAttachablePluginBySpec(volumeToDetach.VolumeSpec) + if err != nil || attachableVolumePlugin == nil { + return volumetypes.GeneratedOperations{}, volumeToDetach.GenerateErrorDetailed("DetachVolume.FindAttachablePluginBySpec failed", err) } volumeName, err = @@ -531,25 +436,9 @@ func (og *operationGenerator) GenerateDetachVolumeFunc( return volumetypes.GeneratedOperations{}, volumeToDetach.GenerateErrorDetailed("DetachVolume.SplitUniqueName failed", err) } - // TODO(dyzz): This case can't distinguish between PV and In-line which is necessary because - // if it was PV it may have been migrated, but the same plugin with in-line may not have been. - // Suggestions welcome... - if og.translator.IsMigratableIntreePluginByName(pluginName) && utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) { - // The volume represented by this spec is CSI and thus should be migrated - attachableVolumePlugin, err = og.volumePluginMgr.FindAttachablePluginByName(csi.CSIPluginName) - if err != nil || attachableVolumePlugin == nil { - return volumetypes.GeneratedOperations{}, volumeToDetach.GenerateErrorDetailed("AttachVolume.FindAttachablePluginBySpec failed", err) - } - // volumeToDetach.VolumeName here is always the in-tree volume name - // therefore a workaround is required. volumeToDetach.DevicePath - // is the attachID which happens to be what volumeName is needed for in Detach. - // Therefore we set volumeName to the attachID. And CSI Detach can detect and use that. - volumeName = volumeToDetach.DevicePath - } else { - attachableVolumePlugin, err = og.volumePluginMgr.FindAttachablePluginByName(pluginName) - if err != nil || attachableVolumePlugin == nil { - return volumetypes.GeneratedOperations{}, volumeToDetach.GenerateErrorDetailed("DetachVolume.FindAttachablePluginByName failed", err) - } + attachableVolumePlugin, err = og.volumePluginMgr.FindAttachablePluginByName(pluginName) + if err != nil || attachableVolumePlugin == nil { + return volumetypes.GeneratedOperations{}, volumeToDetach.GenerateErrorDetailed("DetachVolume.FindAttachablePluginByName failed", err) } } @@ -600,21 +489,8 @@ func (og *operationGenerator) GenerateMountVolumeFunc( volumeToMount VolumeToMount, actualStateOfWorld ActualStateOfWorldMounterUpdater, isRemount bool) volumetypes.GeneratedOperations { - // Get mounter plugin - originalSpec := volumeToMount.VolumeSpec + volumePluginName := unknownVolumePlugin - // Need to translate the spec here if the plugin is migrated so that the metrics - // emitted show the correct (migrated) plugin - if useCSIPlugin(og.translator, og.volumePluginMgr, volumeToMount.VolumeSpec) { - csiSpec, err := translateSpec(og.translator, volumeToMount.VolumeSpec) - if err == nil { - volumeToMount.VolumeSpec = csiSpec - } - // If we have an error here we ignore it, the metric emitted will then be for the - // in-tree plugin. This error case(skipped one) will also trigger an error - // while the generated function is executed. And those errors will be handled during the execution of the generated - // function with a back off policy. - } volumePlugin, err := og.volumePluginMgr.FindPluginBySpec(volumeToMount.VolumeSpec) if err == nil && volumePlugin != nil { @@ -622,16 +498,7 @@ func (og *operationGenerator) GenerateMountVolumeFunc( } mountVolumeFunc := func() (error, error) { - // Get mounter plugin - if useCSIPlugin(og.translator, og.volumePluginMgr, volumeToMount.VolumeSpec) { - csiSpec, err := translateSpec(og.translator, volumeToMount.VolumeSpec) - if err != nil { - return volumeToMount.GenerateError("MountVolume.TranslateSpec failed", err) - } - volumeToMount.VolumeSpec = csiSpec - } - volumePlugin, err := og.volumePluginMgr.FindPluginBySpec(volumeToMount.VolumeSpec) if err != nil || volumePlugin == nil { return volumeToMount.GenerateError("MountVolume.FindPluginBySpec failed", err) @@ -789,7 +656,7 @@ func (og *operationGenerator) GenerateMountVolumeFunc( nil, volumeToMount.OuterVolumeSpecName, volumeToMount.VolumeGidValue, - originalSpec) + volumeToMount.VolumeSpec) if markVolMountedErr != nil { // On failure, return error. Caller will log and retry. return volumeToMount.GenerateError("MountVolume.MarkVolumeAsMounted failed", markVolMountedErr) @@ -816,16 +683,8 @@ func (og *operationGenerator) GenerateUnmountVolumeFunc( volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, podsDir string) (volumetypes.GeneratedOperations, error) { - - var pluginName string - if volumeToUnmount.VolumeSpec != nil && useCSIPlugin(og.translator, og.volumePluginMgr, volumeToUnmount.VolumeSpec) { - pluginName = csi.CSIPluginName - } else { - pluginName = volumeToUnmount.PluginName - } - // Get mountable plugin - volumePlugin, err := og.volumePluginMgr.FindPluginByName(pluginName) + volumePlugin, err := og.volumePluginMgr.FindPluginByName(volumeToUnmount.PluginName) if err != nil || volumePlugin == nil { return volumetypes.GeneratedOperations{}, volumeToUnmount.GenerateErrorDetailed("UnmountVolume.FindPluginByName failed", err) } @@ -884,22 +743,9 @@ func (og *operationGenerator) GenerateUnmountDeviceFunc( deviceToDetach AttachedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, hostutil hostutil.HostUtils) (volumetypes.GeneratedOperations, error) { - - var pluginName string - if useCSIPlugin(og.translator, og.volumePluginMgr, deviceToDetach.VolumeSpec) { - pluginName = csi.CSIPluginName - csiSpec, err := translateSpec(og.translator, deviceToDetach.VolumeSpec) - if err != nil { - return volumetypes.GeneratedOperations{}, deviceToDetach.GenerateErrorDetailed("UnmountDevice.TranslateSpec failed", err) - } - deviceToDetach.VolumeSpec = csiSpec - } else { - pluginName = deviceToDetach.PluginName - } - // Get DeviceMounter plugin deviceMountableVolumePlugin, err := - og.volumePluginMgr.FindDeviceMountablePluginByName(pluginName) + og.volumePluginMgr.FindDeviceMountablePluginByName(deviceToDetach.PluginName) if err != nil || deviceMountableVolumePlugin == nil { return volumetypes.GeneratedOperations{}, deviceToDetach.GenerateErrorDetailed("UnmountDevice.FindDeviceMountablePluginByName failed", err) } @@ -986,17 +832,6 @@ func (og *operationGenerator) GenerateMapVolumeFunc( volumeToMount VolumeToMount, actualStateOfWorld ActualStateOfWorldMounterUpdater) (volumetypes.GeneratedOperations, error) { - originalSpec := volumeToMount.VolumeSpec - // Translate to CSI spec if migration enabled - if useCSIPlugin(og.translator, og.volumePluginMgr, originalSpec) { - csiSpec, err := translateSpec(og.translator, originalSpec) - if err != nil { - return volumetypes.GeneratedOperations{}, volumeToMount.GenerateErrorDetailed("MapVolume.TranslateSpec failed", err) - } - - volumeToMount.VolumeSpec = csiSpec - } - // Get block volume mapper plugin blockVolumePlugin, err := og.volumePluginMgr.FindMapperPluginBySpec(volumeToMount.VolumeSpec) @@ -1156,7 +991,7 @@ func (og *operationGenerator) GenerateMapVolumeFunc( blockVolumeMapper, volumeToMount.OuterVolumeSpecName, volumeToMount.VolumeGidValue, - originalSpec) + volumeToMount.VolumeSpec) if markVolMountedErr != nil { // On failure, return error. Caller will log and retry. return volumeToMount.GenerateError("MapVolume.MarkVolumeAsMounted failed", markVolMountedErr) @@ -1187,32 +1022,12 @@ func (og *operationGenerator) GenerateUnmapVolumeFunc( volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater) (volumetypes.GeneratedOperations, error) { - var blockVolumePlugin volume.BlockVolumePlugin - var err error - // Translate to CSI spec if migration enabled - // And get block volume unmapper plugin - if volumeToUnmount.VolumeSpec != nil && useCSIPlugin(og.translator, og.volumePluginMgr, volumeToUnmount.VolumeSpec) { - csiSpec, err := translateSpec(og.translator, volumeToUnmount.VolumeSpec) - if err != nil { - return volumetypes.GeneratedOperations{}, volumeToUnmount.GenerateErrorDetailed("UnmapVolume.TranslateSpec failed", err) - } - - volumeToUnmount.VolumeSpec = csiSpec - - blockVolumePlugin, err = - og.volumePluginMgr.FindMapperPluginByName(csi.CSIPluginName) - if err != nil { - return volumetypes.GeneratedOperations{}, volumeToUnmount.GenerateErrorDetailed("UnmapVolume.FindMapperPluginByName failed", err) - } - } else { - blockVolumePlugin, err = - og.volumePluginMgr.FindMapperPluginByName(volumeToUnmount.PluginName) - if err != nil { - return volumetypes.GeneratedOperations{}, volumeToUnmount.GenerateErrorDetailed("UnmapVolume.FindMapperPluginByName failed", err) - } + // Get block volume unmapper plugin + blockVolumePlugin, err := + og.volumePluginMgr.FindMapperPluginByName(volumeToUnmount.PluginName) + if err != nil { + return volumetypes.GeneratedOperations{}, volumeToUnmount.GenerateErrorDetailed("UnmapVolume.FindMapperPluginByName failed", err) } - - var blockVolumeUnmapper volume.BlockVolumeUnmapper if blockVolumePlugin == nil { return volumetypes.GeneratedOperations{}, volumeToUnmount.GenerateErrorDetailed("UnmapVolume.FindMapperPluginByName failed to find BlockVolumeMapper plugin. Volume plugin is nil.", nil) } @@ -1289,27 +1104,10 @@ func (og *operationGenerator) GenerateUnmapDeviceFunc( actualStateOfWorld ActualStateOfWorldMounterUpdater, hostutil hostutil.HostUtils) (volumetypes.GeneratedOperations, error) { - var blockVolumePlugin volume.BlockVolumePlugin - var err error - // Translate to CSI spec if migration enabled - if useCSIPlugin(og.translator, og.volumePluginMgr, deviceToDetach.VolumeSpec) { - csiSpec, err := translateSpec(og.translator, deviceToDetach.VolumeSpec) - if err != nil { - return volumetypes.GeneratedOperations{}, deviceToDetach.GenerateErrorDetailed("UnmapDevice.TranslateSpec failed", err) - } - - deviceToDetach.VolumeSpec = csiSpec - blockVolumePlugin, err = - og.volumePluginMgr.FindMapperPluginByName(csi.CSIPluginName) - if err != nil { - return volumetypes.GeneratedOperations{}, deviceToDetach.GenerateErrorDetailed("UnmapDevice.FindMapperPluginByName failed", err) - } - } else { - blockVolumePlugin, err = - og.volumePluginMgr.FindMapperPluginByName(deviceToDetach.PluginName) - if err != nil { - return volumetypes.GeneratedOperations{}, deviceToDetach.GenerateErrorDetailed("UnmapDevice.FindMapperPluginByName failed", err) - } + blockVolumePlugin, err := + og.volumePluginMgr.FindMapperPluginByName(deviceToDetach.PluginName) + if err != nil { + return volumetypes.GeneratedOperations{}, deviceToDetach.GenerateErrorDetailed("UnmapDevice.FindMapperPluginByName failed", err) } if blockVolumePlugin == nil { @@ -1586,22 +1384,13 @@ func (og *operationGenerator) GenerateExpandInUseVolumeFunc( volumeToMount VolumeToMount, actualStateOfWorld ActualStateOfWorldMounterUpdater) (volumetypes.GeneratedOperations, error) { - fsResizeFunc := func() (error, error) { - // Need to translate the spec here if the plugin is migrated so that the metrics - // emitted show the correct (migrated) plugin - if useCSIPlugin(og.translator, og.volumePluginMgr, volumeToMount.VolumeSpec) { - csiSpec, err := translateSpec(og.translator, volumeToMount.VolumeSpec) - if err != nil { - return volumeToMount.GenerateError("NodeExpandVolume.translateSpec failed", err) - } - volumeToMount.VolumeSpec = csiSpec - } - volumePlugin, err := - og.volumePluginMgr.FindPluginBySpec(volumeToMount.VolumeSpec) - if err != nil || volumePlugin == nil { - return volumeToMount.GenerateError("NodeExpandVolume.FindPluginBySpec failed", err) - } + volumePlugin, err := + og.volumePluginMgr.FindPluginBySpec(volumeToMount.VolumeSpec) + if err != nil || volumePlugin == nil { + return volumetypes.GeneratedOperations{}, volumeToMount.GenerateErrorDetailed("NodeExpandVolume.FindPluginBySpec failed", err) + } + fsResizeFunc := func() (error, error) { var resizeDone bool var simpleErr, detailedErr error resizeOptions := volume.NodeResizeOptions{ @@ -1659,24 +1448,6 @@ func (og *operationGenerator) GenerateExpandInUseVolumeFunc( } } - // Need to translate the spec here if the plugin is migrated so that the metrics - // emitted show the correct (migrated) plugin - if useCSIPlugin(og.translator, og.volumePluginMgr, volumeToMount.VolumeSpec) { - csiSpec, err := translateSpec(og.translator, volumeToMount.VolumeSpec) - if err == nil { - volumeToMount.VolumeSpec = csiSpec - } - // If we have an error here we ignore it, the metric emitted will then be for the - // in-tree plugin. This error case(skipped one) will also trigger an error - // while the generated function is executed. And those errors will be handled during the execution of the generated - // function with a back off policy. - } - volumePlugin, err := - og.volumePluginMgr.FindPluginBySpec(volumeToMount.VolumeSpec) - if err != nil || volumePlugin == nil { - return volumetypes.GeneratedOperations{}, volumeToMount.GenerateErrorDetailed("NodeExpandVolume.FindPluginBySpec failed", err) - } - return volumetypes.GeneratedOperations{ OperationName: "volume_fs_resize", OperationFunc: fsResizeFunc, @@ -1822,134 +1593,3 @@ func isDeviceOpened(deviceToDetach AttachedVolume, hostUtil hostutil.HostUtils) } return deviceOpened, nil } - -func useCSIPlugin(tr InTreeToCSITranslator, vpm *volume.VolumePluginMgr, spec *volume.Spec) bool { - // TODO(#75146) Check whether the driver is installed as well so that - // we can throw a better error when the driver is not installed. - // The error should be of the approximate form: - // fmt.Errorf("in-tree plugin %s is migrated on node %s but driver %s is not installed", pluginName, string(nodeName), driverName) - if !utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) { - return false - } - if tr.IsPVMigratable(spec.PersistentVolume) || tr.IsInlineMigratable(spec.Volume) { - migratable, err := vpm.IsPluginMigratableBySpec(spec) - if err == nil && migratable { - return true - } - } - return false -} - -func nodeUsingCSIPlugin(tr InTreeToCSITranslator, vpm *volume.VolumePluginMgr, spec *volume.Spec, nodeName types.NodeName) (bool, error) { - migratable, err := vpm.IsPluginMigratableBySpec(spec) - if err != nil { - return false, err - } - if !utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) || - !utilfeature.DefaultFeatureGate.Enabled(features.CSINodeInfo) || - !migratable { - return false, nil - } - - if len(nodeName) == 0 { - return false, goerrors.New("nodeName is empty") - } - - kubeClient := vpm.Host.GetKubeClient() - if kubeClient == nil { - // Don't handle the controller/kubelet version skew check and fallback - // to just checking the feature gates. This can happen if - // we are in a standalone (headless) Kubelet - return true, nil - } - - adcHost, ok := vpm.Host.(volume.AttachDetachVolumeHost) - if !ok { - // Don't handle the controller/kubelet version skew check and fallback - // to just checking the feature gates. This can happen if - // "enableControllerAttachDetach" is set to true on kubelet - return true, nil - } - - if adcHost.CSINodeLister() == nil { - return false, goerrors.New("could not find CSINodeLister in attachDetachController") - } - - csiNode, err := adcHost.CSINodeLister().Get(string(nodeName)) - if err != nil { - return false, err - } - - ann := csiNode.GetAnnotations() - if ann == nil { - return false, nil - } - - var mpaSet sets.String - mpa := ann[v1.MigratedPluginsAnnotationKey] - tok := strings.Split(mpa, ",") - if len(mpa) == 0 { - mpaSet = sets.NewString() - } else { - mpaSet = sets.NewString(tok...) - } - - pluginName, err := tr.GetInTreePluginNameFromSpec(spec.PersistentVolume, spec.Volume) - if err != nil { - return false, err - } - - if len(pluginName) == 0 { - // Could not find a plugin name from translation directory, assume not translated - return false, nil - } - - isMigratedOnNode := mpaSet.Has(pluginName) - - if isMigratedOnNode { - installed := false - driverName, err := tr.GetCSINameFromInTreeName(pluginName) - if err != nil { - return isMigratedOnNode, err - } - for _, driver := range csiNode.Spec.Drivers { - if driver.Name == driverName { - installed = true - break - } - } - if !installed { - return true, fmt.Errorf("in-tree plugin %s is migrated on node %s but driver %s is not installed", pluginName, string(nodeName), driverName) - } - } - - return isMigratedOnNode, nil - -} - -func translateSpec(tr InTreeToCSITranslator, spec *volume.Spec) (*volume.Spec, error) { - var csiPV *v1.PersistentVolume - var err error - inlineVolume := false - if spec.PersistentVolume != nil { - // TranslateInTreePVToCSI will create a new PV - csiPV, err = tr.TranslateInTreePVToCSI(spec.PersistentVolume) - if err != nil { - return nil, fmt.Errorf("failed to translate in tree pv to CSI: %v", err) - } - } else if spec.Volume != nil { - // TranslateInTreeInlineVolumeToCSI will create a new PV - csiPV, err = tr.TranslateInTreeInlineVolumeToCSI(spec.Volume) - if err != nil { - return nil, fmt.Errorf("failed to translate in tree inline volume to CSI: %v", err) - } - inlineVolume = true - } else { - return &volume.Spec{}, goerrors.New("not a valid volume spec") - } - return &volume.Spec{ - PersistentVolume: csiPV, - ReadOnly: spec.ReadOnly, - InlineVolumeSpecForCSIMigration: inlineVolume, - }, nil -} diff --git a/pkg/volume/util/operationexecutor/operation_generator_test.go b/pkg/volume/util/operationexecutor/operation_generator_test.go index 8e128d1fb3c..b0d5e62758f 100644 --- a/pkg/volume/util/operationexecutor/operation_generator_test.go +++ b/pkg/volume/util/operationexecutor/operation_generator_test.go @@ -17,24 +17,18 @@ limitations under the License. package operationexecutor import ( - "fmt" "github.com/prometheus/client_model/go" "github.com/stretchr/testify/assert" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/uuid" - utilfeature "k8s.io/apiserver/pkg/util/feature" fakeclient "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/tools/record" - "k8s.io/component-base/featuregate" - featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/component-base/metrics/legacyregistry" "k8s.io/csi-translation-lib/plugins" - "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/awsebs" - "k8s.io/kubernetes/pkg/volume/csi" csitesting "k8s.io/kubernetes/pkg/volume/csi/testing" "k8s.io/kubernetes/pkg/volume/gcepd" volumetesting "k8s.io/kubernetes/pkg/volume/testing" @@ -46,21 +40,16 @@ import ( // generated func so there is no need to test the plugin name that's used inside generated function func TestOperationGenerator_GenerateUnmapVolumeFunc_PluginName(t *testing.T) { type testcase struct { - name string - isCsiMigrationEnabled bool - pluginName string - csiDriverName string - csiMigrationFeature featuregate.Feature - pvSpec v1.PersistentVolumeSpec - probVolumePlugins []volume.VolumePlugin + name string + pluginName string + pvSpec v1.PersistentVolumeSpec + probVolumePlugins []volume.VolumePlugin } testcases := []testcase{ { - name: "gce pd plugin: csi migration disabled", - isCsiMigrationEnabled: false, - pluginName: plugins.GCEPDInTreePluginName, - csiMigrationFeature: features.CSIMigrationGCE, + name: "gce pd plugin: csi migration disabled", + pluginName: plugins.GCEPDInTreePluginName, pvSpec: v1.PersistentVolumeSpec{ PersistentVolumeSource: v1.PersistentVolumeSource{ GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{}, @@ -68,34 +57,8 @@ func TestOperationGenerator_GenerateUnmapVolumeFunc_PluginName(t *testing.T) { probVolumePlugins: gcepd.ProbeVolumePlugins(), }, { - name: "gce pd plugin: csi migration enabled", - isCsiMigrationEnabled: true, - pluginName: plugins.GCEPDInTreePluginName, - csiDriverName: plugins.GCEPDDriverName, - csiMigrationFeature: features.CSIMigrationGCE, - pvSpec: v1.PersistentVolumeSpec{ - PersistentVolumeSource: v1.PersistentVolumeSource{ - GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{}, - }}, - probVolumePlugins: gcepd.ProbeVolumePlugins(), - }, - { - name: "aws ebs plugin: csi migration disabled", - isCsiMigrationEnabled: false, - pluginName: plugins.AWSEBSInTreePluginName, - csiMigrationFeature: features.CSIMigrationAWS, - pvSpec: v1.PersistentVolumeSpec{ - PersistentVolumeSource: v1.PersistentVolumeSource{ - AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{}, - }}, - probVolumePlugins: awsebs.ProbeVolumePlugins(), - }, - { - name: "aws ebs plugin: csi migration enabled", - isCsiMigrationEnabled: true, - pluginName: plugins.AWSEBSInTreePluginName, - csiDriverName: plugins.AWSEBSDriverName, - csiMigrationFeature: features.CSIMigrationAWS, + name: "aws ebs plugin: csi migration disabled", + pluginName: plugins.AWSEBSInTreePluginName, pvSpec: v1.PersistentVolumeSpec{ PersistentVolumeSource: v1.PersistentVolumeSource{ AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{}, @@ -106,13 +69,7 @@ func TestOperationGenerator_GenerateUnmapVolumeFunc_PluginName(t *testing.T) { for _, tc := range testcases { expectedPluginName := tc.pluginName - if tc.isCsiMigrationEnabled { - expectedPluginName = fmt.Sprintf("%s:%s", csi.CSIPluginName, tc.csiDriverName) - } - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIMigration, tc.isCsiMigrationEnabled)() - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, tc.csiMigrationFeature, tc.isCsiMigrationEnabled)() - - volumePluginMgr, plugin, tmpDir := initTestPlugins(t, tc.probVolumePlugins, tc.pluginName) + volumePluginMgr, tmpDir := initTestPlugins(t, tc.probVolumePlugins, tc.pluginName) defer os.RemoveAll(tmpDir) operationGenerator := getTestOperationGenerator(volumePluginMgr) @@ -120,21 +77,6 @@ func TestOperationGenerator_GenerateUnmapVolumeFunc_PluginName(t *testing.T) { pod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: types.UID(string(uuid.NewUUID()))}} volumeToUnmount := getTestVolumeToUnmount(pod, tc.pvSpec, tc.pluginName) - if tc.isCsiMigrationEnabled { - // GenerateUnmapVolumeFunc call blockVolumePlugin.NewBlockVolumeUnmapper and when the plugin is csi, - // csi plugin looks a file that contains some information about the volume, - // and GenerateUnmapVolumeFuncfails if csi plugin can't find that file. - // So the reason for calling plugin.NewBlockVolumeMapper for csi enabled case is creating that file. - csiSpec, err := translateSpec(operationGenerator.GetCSITranslator(), volumeToUnmount.VolumeSpec) - if err != nil { - t.Fatalf("Can't translate volume to CSI") - } - _, mapperError := (*plugin).(volume.BlockVolumePlugin).NewBlockVolumeMapper(csiSpec, pod, volume.VolumeOptions{}) - if mapperError != nil { - t.Fatalf("mapper error: %v\n", mapperError) - } - } - unmapVolumeFunc, e := operationGenerator.GenerateUnmapVolumeFunc(volumeToUnmount, nil) if e != nil { t.Fatalf("Error occurred while generating unmapVolumeFunc: %v", e) @@ -238,9 +180,9 @@ func getMetricFamily(metricFamilyName string) *io_prometheus_client.MetricFamily return nil } -func initTestPlugins(t *testing.T, plugs []volume.VolumePlugin, pluginName string) (*volume.VolumePluginMgr, *volume.VolumePlugin, string) { +func initTestPlugins(t *testing.T, plugs []volume.VolumePlugin, pluginName string) (*volume.VolumePluginMgr, string) { client := fakeclient.NewSimpleClientset() - pluginMgr, csiPlugin, tmpDir := csitesting.NewTestPlugin(t, client) + pluginMgr, _, tmpDir := csitesting.NewTestPlugin(t, client) err := pluginMgr.InitPlugins(plugs, nil, pluginMgr.Host) if err != nil { @@ -252,5 +194,5 @@ func initTestPlugins(t *testing.T, plugs []volume.VolumePlugin, pluginName strin t.Fatalf("Can't find the plugin by name: %s", pluginName) } - return pluginMgr, csiPlugin, tmpDir + return pluginMgr, tmpDir } diff --git a/pkg/volume/vsphere_volume/vsphere_volume.go b/pkg/volume/vsphere_volume/vsphere_volume.go index b6de7fba54a..b78a1886ff0 100644 --- a/pkg/volume/vsphere_volume/vsphere_volume.go +++ b/pkg/volume/vsphere_volume/vsphere_volume.go @@ -85,10 +85,6 @@ func (plugin *vsphereVolumePlugin) CanSupport(spec *volume.Spec) bool { (spec.Volume != nil && spec.Volume.VsphereVolume != nil) } -func (plugin *vsphereVolumePlugin) IsMigratedToCSI() bool { - return false -} - func (plugin *vsphereVolumePlugin) RequiresRemount() bool { return false } diff --git a/staging/src/k8s.io/csi-translation-lib/plugins/gce_pd.go b/staging/src/k8s.io/csi-translation-lib/plugins/gce_pd.go index db93112f7ca..7728ff47b58 100644 --- a/staging/src/k8s.io/csi-translation-lib/plugins/gce_pd.go +++ b/staging/src/k8s.io/csi-translation-lib/plugins/gce_pd.go @@ -209,6 +209,7 @@ func (g *gcePersistentDiskCSITranslator) TranslateInTreeInlineVolumeToCSI(volume am = v1.ReadWriteOnce } + fsMode := v1.PersistentVolumeFilesystem return &v1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ // A.K.A InnerVolumeSpecName required to match for Unmount @@ -227,6 +228,7 @@ func (g *gcePersistentDiskCSITranslator) TranslateInTreeInlineVolumeToCSI(volume }, }, AccessModes: []v1.PersistentVolumeAccessMode{am}, + VolumeMode: &fsMode, }, }, nil } diff --git a/staging/src/k8s.io/csi-translation-lib/translate.go b/staging/src/k8s.io/csi-translation-lib/translate.go index 3b3e0c632ec..668bc872af6 100644 --- a/staging/src/k8s.io/csi-translation-lib/translate.go +++ b/staging/src/k8s.io/csi-translation-lib/translate.go @@ -67,7 +67,20 @@ func (CSITranslator) TranslateInTreeInlineVolumeToCSI(volume *v1.Volume) (*v1.Pe } for _, curPlugin := range inTreePlugins { if curPlugin.CanSupportInline(volume) { - return curPlugin.TranslateInTreeInlineVolumeToCSI(volume) + pv, err := curPlugin.TranslateInTreeInlineVolumeToCSI(volume) + if err != nil { + return nil, err + } + // Inline volumes only support PersistentVolumeFilesystem (and not block). + // If VolumeMode has not been set explicitly by plugin-specific + // translator, set it to Filesystem here. + // This is only necessary for inline volumes as the default PV + // initialization that populates VolumeMode does not apply to inline volumes. + if pv.Spec.VolumeMode == nil { + volumeMode := v1.PersistentVolumeFilesystem + pv.Spec.VolumeMode = &volumeMode + } + return pv, nil } } return nil, fmt.Errorf("could not find in-tree plugin translation logic for %#v", volume.Name)