diff --git a/pkg/scheduler/framework/plugins/feature/feature.go b/pkg/scheduler/framework/plugins/feature/feature.go index d4c5cc62d60..4f201bf9da0 100644 --- a/pkg/scheduler/framework/plugins/feature/feature.go +++ b/pkg/scheduler/framework/plugins/feature/feature.go @@ -23,6 +23,8 @@ type Features struct { EnableDRAAdminAccess bool EnableDynamicResourceAllocation bool EnableVolumeCapacityPriority bool + EnableVolumeAttributesClass bool + EnableCSIMigrationPortworx bool EnableNodeInclusionPolicyInPodTopologySpread bool EnableMatchLabelKeysInPodTopologySpread bool EnableInPlacePodVerticalScaling bool diff --git a/pkg/scheduler/framework/plugins/noderesources/balanced_allocation.go b/pkg/scheduler/framework/plugins/noderesources/balanced_allocation.go index e11569102ed..a0634848400 100644 --- a/pkg/scheduler/framework/plugins/noderesources/balanced_allocation.go +++ b/pkg/scheduler/framework/plugins/noderesources/balanced_allocation.go @@ -127,10 +127,12 @@ func NewBalancedAllocation(_ context.Context, baArgs runtime.Object, h framework return &BalancedAllocation{ handle: h, resourceAllocationScorer: resourceAllocationScorer{ - Name: BalancedAllocationName, - scorer: balancedResourceScorer, - useRequested: true, - resources: args.Resources, + Name: BalancedAllocationName, + enableInPlacePodVerticalScaling: fts.EnableInPlacePodVerticalScaling, + enablePodLevelResources: fts.EnablePodLevelResources, + scorer: balancedResourceScorer, + useRequested: true, + resources: args.Resources, }, }, nil } diff --git a/pkg/scheduler/framework/plugins/noderesources/resource_allocation.go b/pkg/scheduler/framework/plugins/noderesources/resource_allocation.go index e8526629485..ad4970f2aaa 100644 --- a/pkg/scheduler/framework/plugins/noderesources/resource_allocation.go +++ b/pkg/scheduler/framework/plugins/noderesources/resource_allocation.go @@ -21,11 +21,9 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" - utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/klog/v2" resourcehelper "k8s.io/component-helpers/resource" - "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/framework" schedutil "k8s.io/kubernetes/pkg/scheduler/util" @@ -36,7 +34,9 @@ type scorer func(args *config.NodeResourcesFitArgs) *resourceAllocationScorer // resourceAllocationScorer contains information to calculate resource allocation score. type resourceAllocationScorer struct { - Name string + Name string + enableInPlacePodVerticalScaling bool + enablePodLevelResources bool // used to decide whether to use Requested or NonZeroRequested for // cpu and memory. useRequested bool @@ -118,9 +118,9 @@ func (r *resourceAllocationScorer) calculateResourceAllocatableRequest(logger kl func (r *resourceAllocationScorer) calculatePodResourceRequest(pod *v1.Pod, resourceName v1.ResourceName) int64 { opts := resourcehelper.PodResourcesOptions{ - UseStatusResources: utilfeature.DefaultFeatureGate.Enabled(features.InPlacePodVerticalScaling), + UseStatusResources: r.enableInPlacePodVerticalScaling, // SkipPodLevelResources is set to false when PodLevelResources feature is enabled. - SkipPodLevelResources: !utilfeature.DefaultFeatureGate.Enabled(features.PodLevelResources), + SkipPodLevelResources: !r.enablePodLevelResources, } if !r.useRequested { diff --git a/pkg/scheduler/framework/plugins/nodevolumelimits/csi.go b/pkg/scheduler/framework/plugins/nodevolumelimits/csi.go index 1fe5821ca87..b0c35b4c660 100644 --- a/pkg/scheduler/framework/plugins/nodevolumelimits/csi.go +++ b/pkg/scheduler/framework/plugins/nodevolumelimits/csi.go @@ -63,7 +63,8 @@ type CSILimits struct { scLister storagelisters.StorageClassLister vaLister storagelisters.VolumeAttachmentLister - randomVolumeIDPrefix string + enableCSIMigrationPortworx bool + randomVolumeIDPrefix string translator InTreeToCSITranslator } @@ -377,7 +378,7 @@ func (pl *CSILimits) checkAttachableInlineVolume(logger klog.Logger, vol *v1.Vol if err != nil { return fmt.Errorf("looking up provisioner name for volume %s: %w", vol.Name, err) } - if !isCSIMigrationOn(csiNode, inTreeProvisionerName) { + if !isCSIMigrationOn(csiNode, inTreeProvisionerName, pl.enableCSIMigrationPortworx) { csiNodeName := "" if csiNode != nil { csiNodeName = csiNode.Name @@ -438,7 +439,7 @@ func (pl *CSILimits) getCSIDriverInfo(logger klog.Logger, csiNode *storagev1.CSI return "", "" } - if !isCSIMigrationOn(csiNode, pluginName) { + if !isCSIMigrationOn(csiNode, pluginName, pl.enableCSIMigrationPortworx) { logger.V(5).Info("CSI Migration of plugin is not enabled", "plugin", pluginName) return "", "" } @@ -486,7 +487,7 @@ func (pl *CSILimits) getCSIDriverInfoFromSC(logger klog.Logger, csiNode *storage provisioner := storageClass.Provisioner if pl.translator.IsMigratableIntreePluginByName(provisioner) { - if !isCSIMigrationOn(csiNode, provisioner) { + if !isCSIMigrationOn(csiNode, provisioner, pl.enableCSIMigrationPortworx) { logger.V(5).Info("CSI Migration of provisioner is not enabled", "provisioner", provisioner) return "", "" } @@ -513,13 +514,14 @@ func NewCSI(_ context.Context, _ runtime.Object, handle framework.Handle, fts fe csiTranslator := csitrans.New() return &CSILimits{ - csiNodeLister: csiNodesLister, - pvLister: pvLister, - pvcLister: pvcLister, - scLister: scLister, - vaLister: vaLister, - randomVolumeIDPrefix: rand.String(32), - translator: csiTranslator, + csiNodeLister: csiNodesLister, + pvLister: pvLister, + pvcLister: pvcLister, + scLister: scLister, + vaLister: vaLister, + enableCSIMigrationPortworx: fts.EnableCSIMigrationPortworx, + randomVolumeIDPrefix: rand.String(32), + translator: csiTranslator, }, nil } diff --git a/pkg/scheduler/framework/plugins/nodevolumelimits/utils.go b/pkg/scheduler/framework/plugins/nodevolumelimits/utils.go index d70b18f7255..7cb761379d3 100644 --- a/pkg/scheduler/framework/plugins/nodevolumelimits/utils.go +++ b/pkg/scheduler/framework/plugins/nodevolumelimits/utils.go @@ -22,14 +22,12 @@ import ( v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/util/sets" - utilfeature "k8s.io/apiserver/pkg/util/feature" csilibplugins "k8s.io/csi-translation-lib/plugins" - "k8s.io/kubernetes/pkg/features" ) // isCSIMigrationOn returns a boolean value indicating whether // the CSI migration has been enabled for a particular storage plugin. -func isCSIMigrationOn(csiNode *storagev1.CSINode, pluginName string) bool { +func isCSIMigrationOn(csiNode *storagev1.CSINode, pluginName string, enableCSIMigrationPortworx bool) bool { if csiNode == nil || len(pluginName) == 0 { return false } @@ -40,7 +38,7 @@ func isCSIMigrationOn(csiNode *storagev1.CSINode, pluginName string) bool { case csilibplugins.AWSEBSInTreePluginName: return true case csilibplugins.PortworxVolumePluginName: - if !utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationPortworx) { + if !enableCSIMigrationPortworx { return false } case csilibplugins.GCEPDInTreePluginName: diff --git a/pkg/scheduler/framework/plugins/registry.go b/pkg/scheduler/framework/plugins/registry.go index ae50798a3d6..af06e5952e3 100644 --- a/pkg/scheduler/framework/plugins/registry.go +++ b/pkg/scheduler/framework/plugins/registry.go @@ -49,6 +49,8 @@ func NewInTreeRegistry() runtime.Registry { EnableDRAAdminAccess: feature.DefaultFeatureGate.Enabled(features.DRAAdminAccess), EnableDynamicResourceAllocation: feature.DefaultFeatureGate.Enabled(features.DynamicResourceAllocation), EnableVolumeCapacityPriority: feature.DefaultFeatureGate.Enabled(features.VolumeCapacityPriority), + EnableVolumeAttributesClass: feature.DefaultFeatureGate.Enabled(features.VolumeAttributesClass), + EnableCSIMigrationPortworx: feature.DefaultFeatureGate.Enabled(features.CSIMigrationPortworx), EnableNodeInclusionPolicyInPodTopologySpread: feature.DefaultFeatureGate.Enabled(features.NodeInclusionPolicyInPodTopologySpread), EnableMatchLabelKeysInPodTopologySpread: feature.DefaultFeatureGate.Enabled(features.MatchLabelKeysInPodTopologySpread), EnableInPlacePodVerticalScaling: feature.DefaultFeatureGate.Enabled(features.InPlacePodVerticalScaling), diff --git a/pkg/scheduler/framework/plugins/volumebinding/binder.go b/pkg/scheduler/framework/plugins/volumebinding/binder.go index d20ca931822..269b1942d44 100644 --- a/pkg/scheduler/framework/plugins/volumebinding/binder.go +++ b/pkg/scheduler/framework/plugins/volumebinding/binder.go @@ -33,7 +33,6 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/storage" - utilfeature "k8s.io/apiserver/pkg/util/feature" coreinformers "k8s.io/client-go/informers/core/v1" storageinformers "k8s.io/client-go/informers/storage/v1" clientset "k8s.io/client-go/kubernetes" @@ -45,7 +44,7 @@ import ( csiplugins "k8s.io/csi-translation-lib/plugins" "k8s.io/klog/v2" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" - "k8s.io/kubernetes/pkg/features" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumebinding/metrics" "k8s.io/kubernetes/pkg/scheduler/util/assumecache" ) @@ -203,7 +202,9 @@ type PodVolumeClaims struct { } type volumeBinder struct { - kubeClient clientset.Interface + kubeClient clientset.Interface + enableVolumeAttributesClass bool + enableCSIMigrationPortworx bool classLister storagelisters.StorageClassLister podLister corelisters.PodLister @@ -238,6 +239,7 @@ type CapacityCheck struct { func NewVolumeBinder( logger klog.Logger, kubeClient clientset.Interface, + fts feature.Features, podInformer coreinformers.PodInformer, nodeInformer coreinformers.NodeInformer, csiNodeInformer storageinformers.CSINodeInformer, @@ -247,15 +249,17 @@ func NewVolumeBinder( capacityCheck CapacityCheck, bindTimeout time.Duration) SchedulerVolumeBinder { b := &volumeBinder{ - kubeClient: kubeClient, - podLister: podInformer.Lister(), - classLister: storageClassInformer.Lister(), - nodeLister: nodeInformer.Lister(), - csiNodeLister: csiNodeInformer.Lister(), - pvcCache: NewPVCAssumeCache(logger, pvcInformer.Informer()), - pvCache: NewPVAssumeCache(logger, pvInformer.Informer()), - bindTimeout: bindTimeout, - translator: csitrans.New(), + kubeClient: kubeClient, + enableVolumeAttributesClass: fts.EnableVolumeAttributesClass, + enableCSIMigrationPortworx: fts.EnableCSIMigrationPortworx, + podLister: podInformer.Lister(), + classLister: storageClassInformer.Lister(), + nodeLister: nodeInformer.Lister(), + csiNodeLister: csiNodeInformer.Lister(), + pvcCache: NewPVCAssumeCache(logger, pvcInformer.Informer()), + pvCache: NewPVAssumeCache(logger, pvInformer.Informer()), + bindTimeout: bindTimeout, + translator: csitrans.New(), } b.csiDriverLister = capacityCheck.CSIDriverInformer.Lister() @@ -855,7 +859,7 @@ func (b *volumeBinder) findMatchingVolumes(logger klog.Logger, pod *v1.Pod, clai pvs := unboundVolumesDelayBinding[storageClassName] // Find a matching PV - pv, err := volume.FindMatchingVolume(pvc, pvs, node, chosenPVs, true, utilfeature.DefaultFeatureGate.Enabled(features.VolumeAttributesClass)) + pv, err := volume.FindMatchingVolume(pvc, pvs, node, chosenPVs, true, b.enableVolumeAttributesClass) if err != nil { return false, nil, nil, err } @@ -1033,7 +1037,7 @@ func (a byPVCSize) Less(i, j int) bool { } // isCSIMigrationOnForPlugin checks if CSI migration is enabled for a given plugin. -func isCSIMigrationOnForPlugin(pluginName string) bool { +func isCSIMigrationOnForPlugin(pluginName string, enableCSIMigrationPortworx bool) bool { switch pluginName { case csiplugins.AWSEBSInTreePluginName: return true @@ -1044,7 +1048,7 @@ func isCSIMigrationOnForPlugin(pluginName string) bool { case csiplugins.CinderInTreePluginName: return true case csiplugins.PortworxVolumePluginName: - return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationPortworx) + return enableCSIMigrationPortworx } return false } @@ -1083,7 +1087,7 @@ func (b *volumeBinder) tryTranslatePVToCSI(logger klog.Logger, pv *v1.Persistent return nil, fmt.Errorf("could not get plugin name from pv: %v", err) } - if !isCSIMigrationOnForPlugin(pluginName) { + if !isCSIMigrationOnForPlugin(pluginName, b.enableCSIMigrationPortworx) { return pv, nil } diff --git a/pkg/scheduler/framework/plugins/volumebinding/binder_test.go b/pkg/scheduler/framework/plugins/volumebinding/binder_test.go index b2ae07836bf..9787b374d4d 100644 --- a/pkg/scheduler/framework/plugins/volumebinding/binder_test.go +++ b/pkg/scheduler/framework/plugins/volumebinding/binder_test.go @@ -45,6 +45,7 @@ import ( _ "k8s.io/klog/v2/ktesting/init" "k8s.io/kubernetes/pkg/controller" pvtesting "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/testing" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature" "k8s.io/kubernetes/pkg/scheduler/util/assumecache" ) @@ -167,6 +168,7 @@ func newTestBinder(t *testing.T, ctx context.Context) *testEnv { binder := NewVolumeBinder( logger, client, + feature.Features{}, podInformer, nodeInformer, csiNodeInformer, diff --git a/pkg/scheduler/framework/plugins/volumebinding/volume_binding.go b/pkg/scheduler/framework/plugins/volumebinding/volume_binding.go index 3eff2df8b0e..bdf096c226f 100644 --- a/pkg/scheduler/framework/plugins/volumebinding/volume_binding.go +++ b/pkg/scheduler/framework/plugins/volumebinding/volume_binding.go @@ -579,7 +579,7 @@ func New(ctx context.Context, plArgs runtime.Object, fh framework.Handle, fts fe CSIDriverInformer: fh.SharedInformerFactory().Storage().V1().CSIDrivers(), CSIStorageCapacityInformer: fh.SharedInformerFactory().Storage().V1().CSIStorageCapacities(), } - binder := NewVolumeBinder(klog.FromContext(ctx), fh.ClientSet(), podInformer, nodeInformer, csiNodeInformer, pvcInformer, pvInformer, storageClassInformer, capacityCheck, time.Duration(args.BindTimeoutSeconds)*time.Second) + binder := NewVolumeBinder(klog.FromContext(ctx), fh.ClientSet(), fts, podInformer, nodeInformer, csiNodeInformer, pvcInformer, pvInformer, storageClassInformer, capacityCheck, time.Duration(args.BindTimeoutSeconds)*time.Second) // build score function var scorer volumeCapacityScorer