diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 25596055348..d95141f8282 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -984,13 +984,6 @@ const ( // Proxies client to an apiserver capable of serving the request in the event of version skew. UnknownVersionInteroperabilityProxy featuregate.Feature = "UnknownVersionInteroperabilityProxy" - // owner: @gnufied - // kep: https://kep.k8s.io/5030 - // - // Enables volume limit scaling for CSI drivers. This allows scheduler to - // co-ordinate better with cluster-autoscaler for storage limits. - VolumeLimitScaling featuregate.Feature = "VolumeLimitScaling" - // owner: @saschagrunert // // Enables user namespace support for Pod Security Standards. Enabling this @@ -1014,6 +1007,13 @@ const ( // Enables user specified volume attributes for persistent volumes, like iops and throughput. VolumeAttributesClass featuregate.Feature = "VolumeAttributesClass" + // owner: @gnufied + // kep: https://kep.k8s.io/5030 + // + // Enables volume limit scaling for CSI drivers. This allows scheduler to + // co-ordinate better with cluster-autoscaler for storage limits. + VolumeLimitScaling featuregate.Feature = "VolumeLimitScaling" + // owner: @ksubrmnn // // Allows kube-proxy to create DSR loadbalancers for Windows @@ -1774,7 +1774,7 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate }, VolumeLimitScaling: { - {Version: version.MustParse("1.3"), Default: false, PreRelease: featuregate.Alpha}, + {Version: version.MustParse("1.35"), Default: false, PreRelease: featuregate.Alpha}, }, WinDSR: { diff --git a/pkg/scheduler/framework/plugins/feature/feature.go b/pkg/scheduler/framework/plugins/feature/feature.go index 89bd2c77fab..cf3e7802613 100644 --- a/pkg/scheduler/framework/plugins/feature/feature.go +++ b/pkg/scheduler/framework/plugins/feature/feature.go @@ -37,6 +37,7 @@ type Features struct { EnableDynamicResourceAllocation bool EnableVolumeAttributesClass bool EnableCSIMigrationPortworx bool + EnableVolumeLimitScaling bool EnableNodeInclusionPolicyInPodTopologySpread bool EnableMatchLabelKeysInPodTopologySpread bool EnableInPlacePodVerticalScaling bool @@ -61,6 +62,7 @@ func NewSchedulerFeaturesFromGates(featureGate featuregate.FeatureGate) Features EnableDynamicResourceAllocation: featureGate.Enabled(features.DynamicResourceAllocation), EnableVolumeAttributesClass: featureGate.Enabled(features.VolumeAttributesClass), EnableCSIMigrationPortworx: featureGate.Enabled(features.CSIMigrationPortworx), + EnableVolumeLimitScaling: featureGate.Enabled(features.VolumeLimitScaling), EnableNodeInclusionPolicyInPodTopologySpread: featureGate.Enabled(features.NodeInclusionPolicyInPodTopologySpread), EnableMatchLabelKeysInPodTopologySpread: featureGate.Enabled(features.MatchLabelKeysInPodTopologySpread), EnableInPlacePodVerticalScaling: featureGate.Enabled(features.InPlacePodVerticalScaling), diff --git a/pkg/scheduler/framework/plugins/nodevolumelimits/csi.go b/pkg/scheduler/framework/plugins/nodevolumelimits/csi.go index 41ccd8094ba..466e95d8ee5 100644 --- a/pkg/scheduler/framework/plugins/nodevolumelimits/csi.go +++ b/pkg/scheduler/framework/plugins/nodevolumelimits/csi.go @@ -65,8 +65,8 @@ type CSILimits struct { enableCSIMigrationPortworx bool randomVolumeIDPrefix string - - translator InTreeToCSITranslator + enableVolumeLimitScaling bool + translator InTreeToCSITranslator } var _ fwk.PreFilterPlugin = &CSILimits{} @@ -282,6 +282,15 @@ func (pl *CSILimits) Filter(ctx context.Context, _ fwk.CycleState, pod *v1.Pod, return nil } + if pl.enableVolumeLimitScaling { + for _, driverName := range newVolumes { + if !pl.checkCSIDriverOnNode(driverName, csiNode) { + driverNotInstalledMsg := fmt.Sprintf("%s CSI driver is not installed on the node", driverName) + return fwk.NewStatus(fwk.Unschedulable, driverNotInstalledMsg) + } + } + } + // If the node doesn't have volume limits, the predicate will always be true nodeVolumeLimits := getVolumeLimits(csiNode) if len(nodeVolumeLimits) == 0 { @@ -338,6 +347,19 @@ func (pl *CSILimits) Filter(ctx context.Context, _ fwk.CycleState, pod *v1.Pod, return nil } +func (pl *CSILimits) checkCSIDriverOnNode(pluginName string, csiNode *storagev1.CSINode) bool { + if csiNode == nil { + return false + } + + for _, driver := range csiNode.Spec.Drivers { + if driver.Name == pluginName { + return true + } + } + return false +} + // filterAttachableVolumes filters the attachable volumes from the pod and adds them to the result map. // The result map is a map of volumeUniqueName to driver name. The volumeUniqueName is a unique name for // the volume in the format of "driverName/volumeHandle". And driver name is the CSI driver name. @@ -561,6 +583,7 @@ func NewCSI(_ context.Context, _ runtime.Object, handle fwk.Handle, fts feature. scLister: scLister, vaLister: vaLister, enableCSIMigrationPortworx: fts.EnableCSIMigrationPortworx, + enableVolumeLimitScaling: fts.EnableVolumeLimitScaling, randomVolumeIDPrefix: rand.String(32), translator: csiTranslator, }, nil diff --git a/pkg/scheduler/framework/plugins/nodevolumelimits/csi_test.go b/pkg/scheduler/framework/plugins/nodevolumelimits/csi_test.go index b6e34cc16f2..d241d0c261f 100644 --- a/pkg/scheduler/framework/plugins/nodevolumelimits/csi_test.go +++ b/pkg/scheduler/framework/plugins/nodevolumelimits/csi_test.go @@ -639,7 +639,7 @@ func TestCSILimits(t *testing.T) { } csiTranslator := csitrans.New() p := &CSILimits{ - csiNodeLister: getFakeCSINodeLister(csiNode), + csiManager: NewCSIManager(getFakeCSINodeLister(csiNode)), pvLister: getFakeCSIPVLister(test.filterName, test.driverNames...), pvcLister: append(getFakeCSIPVCLister(test.filterName, scName, test.driverNames...), test.extraClaims...), scLister: getFakeCSIStorageClassLister(scName, test.driverNames[0]), @@ -1247,3 +1247,68 @@ func getNodeWithPodAndVolumeLimits(limitSource string, pods []*v1.Pod, limit int nodeInfo.SetNode(node) return nodeInfo, csiNode } + +func TestVolumeLimitScalingGate(t *testing.T) { + // Pod uses a PVC that resolves to the EBS CSI driver via PV + newPod := st.MakePod().PVC("csi-ebs.csi.aws.com-0").Obj() + + cases := []struct { + name string + enableVolumeLimitScaling bool + limitSource string + limit int32 + wantStatus *fwk.Status + }{ + { + name: "gate enabled - fail when driver not installed", + enableVolumeLimitScaling: true, + limitSource: "no-csi-driver", + limit: 0, + wantStatus: fwk.NewStatus(fwk.Unschedulable, fmt.Sprintf("%s CSI driver is not installed on the node", ebsCSIDriverName)), + }, + { + name: "gate disabled - skip driver presence check", + enableVolumeLimitScaling: false, + limitSource: "no-csi-driver", + limit: 0, + wantStatus: nil, + }, + { + name: "gate enabled - driver installed within limit", + enableVolumeLimitScaling: true, + limitSource: "csinode", + limit: 2, + wantStatus: nil, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + node, csiNode := getNodeWithPodAndVolumeLimits(tt.limitSource, []*v1.Pod{}, tt.limit, ebsCSIDriverName) + + csiTranslator := csitrans.New() + p := &CSILimits{ + csiManager: NewCSIManager(getFakeCSINodeLister(csiNode)), + pvLister: getFakeCSIPVLister("csi", ebsCSIDriverName), + pvcLister: getFakeCSIPVCLister("csi", scName, ebsCSIDriverName), + scLister: getFakeCSIStorageClassLister(scName, ebsCSIDriverName), + vaLister: getFakeVolumeAttachmentLister(0, ebsCSIDriverName), + enableVolumeLimitScaling: tt.enableVolumeLimitScaling, + randomVolumeIDPrefix: rand.String(32), + translator: csiTranslator, + } + + _, ctx := ktesting.NewTestContext(t) + // Ensure PreFilter doesn't skip + _, preStatus := p.PreFilter(ctx, nil, newPod, nil) + if preStatus.Code() == fwk.Skip { + t.Fatalf("unexpected PreFilter Skip") + } + gotStatus := p.Filter(ctx, nil, newPod, node) + + if diff := cmp.Diff(tt.wantStatus, gotStatus, statusCmpOpts...); diff != "" { + t.Errorf("Filter status does not match (-want, +got):\n%s", diff) + } + }) + } +} diff --git a/test/compatibility_lifecycle/reference/versioned_feature_list.yaml b/test/compatibility_lifecycle/reference/versioned_feature_list.yaml index 972f7ce2c44..55030e3637c 100644 --- a/test/compatibility_lifecycle/reference/versioned_feature_list.yaml +++ b/test/compatibility_lifecycle/reference/versioned_feature_list.yaml @@ -1853,6 +1853,12 @@ lockToDefault: false preRelease: GA version: "1.34" +- name: VolumeLimitScaling + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.35" - name: WatchCacheInitializationPostStartHook versionedSpecs: - default: false