From b522e95aaebba9bc02cb3d438cd8e410ec681f8c Mon Sep 17 00:00:00 2001 From: Yecheng Fu Date: Sun, 20 Jun 2021 10:00:51 +0800 Subject: [PATCH] Prioritizing nodes based on volume capacity: API changes --- .../apis/config/scheme/scheme_test.go | 45 +++++++ pkg/scheduler/apis/config/types_pluginargs.go | 15 +++ pkg/scheduler/apis/config/v1beta1/defaults.go | 12 ++ .../apis/config/v1beta1/defaults_test.go | 24 ++++ .../config/v1beta1/zz_generated.conversion.go | 2 + pkg/scheduler/apis/config/v1beta2/defaults.go | 12 ++ .../apis/config/v1beta2/defaults_test.go | 24 ++++ .../config/v1beta2/zz_generated.conversion.go | 2 + .../validation/validation_pluginargs.go | 16 ++- .../validation/validation_pluginargs_test.go | 116 ++++++++++++++++- .../apis/config/zz_generated.deepcopy.go | 5 + .../plugins/volumebinding/volume_binding.go | 18 +-- .../volumebinding/volume_binding_test.go | 121 +++++++++++++++++- .../config/v1beta1/types_pluginargs.go | 16 +++ .../config/v1beta1/zz_generated.deepcopy.go | 5 + .../config/v1beta2/types_pluginargs.go | 16 +++ .../config/v1beta2/zz_generated.deepcopy.go | 5 + 17 files changed, 426 insertions(+), 28 deletions(-) diff --git a/pkg/scheduler/apis/config/scheme/scheme_test.go b/pkg/scheduler/apis/config/scheme/scheme_test.go index 564cdec4cd8..9bdff1a9b3b 100644 --- a/pkg/scheduler/apis/config/scheme/scheme_test.go +++ b/pkg/scheduler/apis/config/scheme/scheme_test.go @@ -733,6 +733,16 @@ func TestCodecsEncodePluginConfig(t *testing.T) { Args: runtime.RawExtension{ Object: &v1beta1.VolumeBindingArgs{ BindTimeoutSeconds: pointer.Int64Ptr(300), + Shape: []v1beta1.UtilizationShapePoint{ + { + Utilization: 0, + Score: 0, + }, + { + Utilization: 100, + Score: 10, + }, + }, }, }, }, @@ -804,6 +814,11 @@ profiles: apiVersion: kubescheduler.config.k8s.io/v1beta1 bindTimeoutSeconds: 300 kind: VolumeBindingArgs + shape: + - score: 0 + utilization: 0 + - score: 10 + utilization: 100 name: VolumeBinding - args: apiVersion: kubescheduler.config.k8s.io/v1beta1 @@ -855,6 +870,16 @@ profiles: Name: "VolumeBinding", Args: &config.VolumeBindingArgs{ BindTimeoutSeconds: 300, + Shape: []config.UtilizationShapePoint{ + { + Utilization: 0, + Score: 0, + }, + { + Utilization: 100, + Score: 10, + }, + }, }, }, { @@ -913,6 +938,11 @@ profiles: apiVersion: kubescheduler.config.k8s.io/v1beta1 bindTimeoutSeconds: 300 kind: VolumeBindingArgs + shape: + - score: 0 + utilization: 0 + - score: 10 + utilization: 100 name: VolumeBinding - args: apiVersion: kubescheduler.config.k8s.io/v1beta1 @@ -945,6 +975,16 @@ profiles: Args: runtime.RawExtension{ Object: &v1beta2.VolumeBindingArgs{ BindTimeoutSeconds: pointer.Int64Ptr(300), + Shape: []v1beta2.UtilizationShapePoint{ + { + Utilization: 0, + Score: 0, + }, + { + Utilization: 100, + Score: 10, + }, + }, }, }, }, @@ -1009,6 +1049,11 @@ profiles: apiVersion: kubescheduler.config.k8s.io/v1beta2 bindTimeoutSeconds: 300 kind: VolumeBindingArgs + shape: + - score: 0 + utilization: 0 + - score: 10 + utilization: 100 name: VolumeBinding - args: apiVersion: kubescheduler.config.k8s.io/v1beta2 diff --git a/pkg/scheduler/apis/config/types_pluginargs.go b/pkg/scheduler/apis/config/types_pluginargs.go index de3abacb4a3..c7af8f6ad1f 100644 --- a/pkg/scheduler/apis/config/types_pluginargs.go +++ b/pkg/scheduler/apis/config/types_pluginargs.go @@ -214,6 +214,21 @@ type VolumeBindingArgs struct { // Value must be non-negative integer. The value zero indicates no waiting. // If this value is nil, the default value will be used. BindTimeoutSeconds int64 + + // Shape specifies the points defining the score function shape, which is + // used to score nodes based on the utilization of statically provisioned + // PVs. The utilization is calculated by dividing the total requested + // storage of the pod by the total capacity of feasible PVs on each node. + // Each point contains utilization (ranges from 0 to 100) and its + // associated score (ranges from 0 to 10). You can turn the priority by + // specifying different scores for different utilization numbers. + // The default shape points are: + // 1) 0 for 0 utilization + // 2) 10 for 100 utilization + // All points must be sorted in increasing order by utilization. + // +featureGate=VolumeCapacityPriority + // +optional + Shape []UtilizationShapePoint } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/scheduler/apis/config/v1beta1/defaults.go b/pkg/scheduler/apis/config/v1beta1/defaults.go index b5961b5d7de..5cdd3fe36a9 100644 --- a/pkg/scheduler/apis/config/v1beta1/defaults.go +++ b/pkg/scheduler/apis/config/v1beta1/defaults.go @@ -280,6 +280,18 @@ func SetDefaults_VolumeBindingArgs(obj *v1beta1.VolumeBindingArgs) { if obj.BindTimeoutSeconds == nil { obj.BindTimeoutSeconds = pointer.Int64Ptr(600) } + if len(obj.Shape) == 0 && feature.DefaultFeatureGate.Enabled(features.VolumeCapacityPriority) { + obj.Shape = []v1beta1.UtilizationShapePoint{ + { + Utilization: 0, + Score: 0, + }, + { + Utilization: 100, + Score: int32(config.MaxCustomPriorityScore), + }, + } + } } func SetDefaults_PodTopologySpreadArgs(obj *v1beta1.PodTopologySpreadArgs) { diff --git a/pkg/scheduler/apis/config/v1beta1/defaults_test.go b/pkg/scheduler/apis/config/v1beta1/defaults_test.go index b094603ecce..307f5cac182 100644 --- a/pkg/scheduler/apis/config/v1beta1/defaults_test.go +++ b/pkg/scheduler/apis/config/v1beta1/defaults_test.go @@ -700,6 +700,30 @@ func TestPluginArgsDefaults(t *testing.T) { }, }, }, + { + name: "VolumeBindingArgs empty, VolumeCapacityPriority disabled", + features: map[featuregate.Feature]bool{ + features.VolumeCapacityPriority: false, + }, + in: &v1beta1.VolumeBindingArgs{}, + want: &v1beta1.VolumeBindingArgs{ + BindTimeoutSeconds: pointer.Int64Ptr(600), + }, + }, + { + name: "VolumeBindingArgs empty, VolumeCapacityPriority enabled", + features: map[featuregate.Feature]bool{ + features.VolumeCapacityPriority: true, + }, + in: &v1beta1.VolumeBindingArgs{}, + want: &v1beta1.VolumeBindingArgs{ + BindTimeoutSeconds: pointer.Int64Ptr(600), + Shape: []v1beta1.UtilizationShapePoint{ + {Utilization: 0, Score: 0}, + {Utilization: 100, Score: 10}, + }, + }, + }, } for _, tc := range tests { scheme := runtime.NewScheme() diff --git a/pkg/scheduler/apis/config/v1beta1/zz_generated.conversion.go b/pkg/scheduler/apis/config/v1beta1/zz_generated.conversion.go index 863bb6fcbb4..0e73d44edc1 100644 --- a/pkg/scheduler/apis/config/v1beta1/zz_generated.conversion.go +++ b/pkg/scheduler/apis/config/v1beta1/zz_generated.conversion.go @@ -916,6 +916,7 @@ func autoConvert_v1beta1_VolumeBindingArgs_To_config_VolumeBindingArgs(in *v1bet if err := v1.Convert_Pointer_int64_To_int64(&in.BindTimeoutSeconds, &out.BindTimeoutSeconds, s); err != nil { return err } + out.Shape = *(*[]config.UtilizationShapePoint)(unsafe.Pointer(&in.Shape)) return nil } @@ -928,6 +929,7 @@ func autoConvert_config_VolumeBindingArgs_To_v1beta1_VolumeBindingArgs(in *confi if err := v1.Convert_int64_To_Pointer_int64(&in.BindTimeoutSeconds, &out.BindTimeoutSeconds, s); err != nil { return err } + out.Shape = *(*[]v1beta1.UtilizationShapePoint)(unsafe.Pointer(&in.Shape)) return nil } diff --git a/pkg/scheduler/apis/config/v1beta2/defaults.go b/pkg/scheduler/apis/config/v1beta2/defaults.go index fd62188fbbd..f3accf0fcdd 100644 --- a/pkg/scheduler/apis/config/v1beta2/defaults.go +++ b/pkg/scheduler/apis/config/v1beta2/defaults.go @@ -245,6 +245,18 @@ func SetDefaults_VolumeBindingArgs(obj *v1beta2.VolumeBindingArgs) { if obj.BindTimeoutSeconds == nil { obj.BindTimeoutSeconds = pointer.Int64Ptr(600) } + if len(obj.Shape) == 0 && feature.DefaultFeatureGate.Enabled(features.VolumeCapacityPriority) { + obj.Shape = []v1beta2.UtilizationShapePoint{ + { + Utilization: 0, + Score: 0, + }, + { + Utilization: 100, + Score: int32(config.MaxCustomPriorityScore), + }, + } + } } func SetDefaults_PodTopologySpreadArgs(obj *v1beta2.PodTopologySpreadArgs) { diff --git a/pkg/scheduler/apis/config/v1beta2/defaults_test.go b/pkg/scheduler/apis/config/v1beta2/defaults_test.go index 42a84030c07..17105785ba0 100644 --- a/pkg/scheduler/apis/config/v1beta2/defaults_test.go +++ b/pkg/scheduler/apis/config/v1beta2/defaults_test.go @@ -674,6 +674,30 @@ func TestPluginArgsDefaults(t *testing.T) { }, }, }, + { + name: "VolumeBindingArgs empty, VolumeCapacityPriority disabled", + features: map[featuregate.Feature]bool{ + features.VolumeCapacityPriority: false, + }, + in: &v1beta2.VolumeBindingArgs{}, + want: &v1beta2.VolumeBindingArgs{ + BindTimeoutSeconds: pointer.Int64Ptr(600), + }, + }, + { + name: "VolumeBindingArgs empty, VolumeCapacityPriority enabled", + features: map[featuregate.Feature]bool{ + features.VolumeCapacityPriority: true, + }, + in: &v1beta2.VolumeBindingArgs{}, + want: &v1beta2.VolumeBindingArgs{ + BindTimeoutSeconds: pointer.Int64Ptr(600), + Shape: []v1beta2.UtilizationShapePoint{ + {Utilization: 0, Score: 0}, + {Utilization: 100, Score: 10}, + }, + }, + }, } for _, tc := range tests { scheme := runtime.NewScheme() diff --git a/pkg/scheduler/apis/config/v1beta2/zz_generated.conversion.go b/pkg/scheduler/apis/config/v1beta2/zz_generated.conversion.go index 76656e11de3..263c95368bd 100644 --- a/pkg/scheduler/apis/config/v1beta2/zz_generated.conversion.go +++ b/pkg/scheduler/apis/config/v1beta2/zz_generated.conversion.go @@ -815,6 +815,7 @@ func autoConvert_v1beta2_VolumeBindingArgs_To_config_VolumeBindingArgs(in *v1bet if err := v1.Convert_Pointer_int64_To_int64(&in.BindTimeoutSeconds, &out.BindTimeoutSeconds, s); err != nil { return err } + out.Shape = *(*[]config.UtilizationShapePoint)(unsafe.Pointer(&in.Shape)) return nil } @@ -827,6 +828,7 @@ func autoConvert_config_VolumeBindingArgs_To_v1beta2_VolumeBindingArgs(in *confi if err := v1.Convert_int64_To_Pointer_int64(&in.BindTimeoutSeconds, &out.BindTimeoutSeconds, s); err != nil { return err } + out.Shape = *(*[]v1beta2.UtilizationShapePoint)(unsafe.Pointer(&in.Shape)) return nil } diff --git a/pkg/scheduler/apis/config/validation/validation_pluginargs.go b/pkg/scheduler/apis/config/validation/validation_pluginargs.go index 9fb3855ea29..ba2f316cf4c 100644 --- a/pkg/scheduler/apis/config/validation/validation_pluginargs.go +++ b/pkg/scheduler/apis/config/validation/validation_pluginargs.go @@ -25,7 +25,9 @@ import ( "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/component-helpers/scheduling/corev1/nodeaffinity" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/apis/config" ) @@ -294,13 +296,21 @@ func ValidateNodeAffinityArgs(path *field.Path, args *config.NodeAffinityArgs) e // ValidateVolumeBindingArgs validates that VolumeBindingArgs are set correctly. func ValidateVolumeBindingArgs(path *field.Path, args *config.VolumeBindingArgs) error { - var err error + var allErrs field.ErrorList if args.BindTimeoutSeconds < 0 { - err = field.Invalid(path.Child("bindTimeoutSeconds"), args.BindTimeoutSeconds, "invalid BindTimeoutSeconds, should not be a negative value") + allErrs = append(allErrs, field.Invalid(path.Child("bindTimeoutSeconds"), args.BindTimeoutSeconds, "invalid BindTimeoutSeconds, should not be a negative value")) } - return err + if utilfeature.DefaultFeatureGate.Enabled(features.VolumeCapacityPriority) { + allErrs = append(allErrs, validateFunctionShape(args.Shape, path.Child("shape"))...) + } else if args.Shape != nil { + // When the feature is off, return an error if the config is not nil. + // This prevents unexpected configuration from taking effect when the + // feature turns on in the future. + allErrs = append(allErrs, field.Invalid(path.Child("shape"), args.Shape, "unexpected field `shape`, remove it or turn on the feature gate VolumeCapacityPriority")) + } + return allErrs.ToAggregate() } func ValidateNodeResourcesFitArgs(path *field.Path, args *config.NodeResourcesFitArgs) error { diff --git a/pkg/scheduler/apis/config/validation/validation_pluginargs_test.go b/pkg/scheduler/apis/config/validation/validation_pluginargs_test.go index 7751ead0971..4790ecf2e4c 100644 --- a/pkg/scheduler/apis/config/validation/validation_pluginargs_test.go +++ b/pkg/scheduler/apis/config/validation/validation_pluginargs_test.go @@ -25,7 +25,12 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/apiserver/pkg/util/feature" + "k8s.io/component-base/featuregate" + featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/apis/config" ) @@ -950,9 +955,10 @@ func TestValidateNodeAffinityArgs(t *testing.T) { func TestValidateVolumeBindingArgs(t *testing.T) { cases := []struct { - name string - args config.VolumeBindingArgs - wantErr error + name string + args config.VolumeBindingArgs + features map[featuregate.Feature]bool + wantErr error }{ { name: "zero is a valid config", @@ -971,14 +977,112 @@ func TestValidateVolumeBindingArgs(t *testing.T) { args: config.VolumeBindingArgs{ BindTimeoutSeconds: -10, }, - wantErr: &field.Error{ - Type: field.ErrorTypeInvalid, - Field: "bindTimeoutSeconds", + wantErr: errors.NewAggregate([]error{&field.Error{ + Type: field.ErrorTypeInvalid, + Field: "bindTimeoutSeconds", + BadValue: int64(-10), + Detail: "invalid BindTimeoutSeconds, should not be a negative value", + }}), + }, + { + name: "[VolumeCapacityPriority=off] shape should be nil when the feature is off", + features: map[featuregate.Feature]bool{ + features.VolumeCapacityPriority: false, + }, + args: config.VolumeBindingArgs{ + BindTimeoutSeconds: 10, + Shape: nil, }, }, + { + name: "[VolumeCapacityPriority=off] error if the shape is not nil when the feature is off", + features: map[featuregate.Feature]bool{ + features.VolumeCapacityPriority: false, + }, + args: config.VolumeBindingArgs{ + BindTimeoutSeconds: 10, + Shape: []config.UtilizationShapePoint{ + {Utilization: 1, Score: 1}, + {Utilization: 3, Score: 3}, + }, + }, + wantErr: errors.NewAggregate([]error{&field.Error{ + Type: field.ErrorTypeInvalid, + Field: "shape", + }}), + }, + { + name: "[VolumeCapacityPriority=on] shape should not be empty", + features: map[featuregate.Feature]bool{ + features.VolumeCapacityPriority: true, + }, + args: config.VolumeBindingArgs{ + BindTimeoutSeconds: 10, + Shape: []config.UtilizationShapePoint{}, + }, + wantErr: errors.NewAggregate([]error{&field.Error{ + Type: field.ErrorTypeRequired, + Field: "shape", + }}), + }, + { + name: "[VolumeCapacityPriority=on] shape points must be sorted in increasing order", + features: map[featuregate.Feature]bool{ + features.VolumeCapacityPriority: true, + }, + args: config.VolumeBindingArgs{ + BindTimeoutSeconds: 10, + Shape: []config.UtilizationShapePoint{ + {Utilization: 3, Score: 3}, + {Utilization: 1, Score: 1}, + }, + }, + wantErr: errors.NewAggregate([]error{&field.Error{ + Type: field.ErrorTypeInvalid, + Field: "shape[1].utilization", + Detail: "Invalid value: 1: utilization values must be sorted in increasing order", + }}), + }, + { + name: "[VolumeCapacityPriority=on] shape point: invalid utilization and score", + features: map[featuregate.Feature]bool{ + features.VolumeCapacityPriority: true, + }, + args: config.VolumeBindingArgs{ + BindTimeoutSeconds: 10, + Shape: []config.UtilizationShapePoint{ + {Utilization: -1, Score: 1}, + {Utilization: 10, Score: -1}, + {Utilization: 20, Score: 11}, + {Utilization: 101, Score: 1}, + }, + }, + wantErr: errors.NewAggregate([]error{ + &field.Error{ + Type: field.ErrorTypeInvalid, + Field: "shape[0].utilization", + }, + &field.Error{ + Type: field.ErrorTypeInvalid, + Field: "shape[1].score", + }, + &field.Error{ + Type: field.ErrorTypeInvalid, + Field: "shape[2].score", + }, + &field.Error{ + Type: field.ErrorTypeInvalid, + Field: "shape[3].utilization", + }, + }), + }, } + for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { + for k, v := range tc.features { + defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, k, v)() + } err := ValidateVolumeBindingArgs(nil, &tc.args) if diff := cmp.Diff(tc.wantErr, err, ignoreBadValueDetail); diff != "" { t.Errorf("ValidateVolumeBindingArgs returned err (-want,+got):\n%s", diff) diff --git a/pkg/scheduler/apis/config/zz_generated.deepcopy.go b/pkg/scheduler/apis/config/zz_generated.deepcopy.go index 2abc6ff48a9..e79203588c4 100644 --- a/pkg/scheduler/apis/config/zz_generated.deepcopy.go +++ b/pkg/scheduler/apis/config/zz_generated.deepcopy.go @@ -970,6 +970,11 @@ func (in *UtilizationShapePoint) DeepCopy() *UtilizationShapePoint { func (in *VolumeBindingArgs) DeepCopyInto(out *VolumeBindingArgs) { *out = *in out.TypeMeta = in.TypeMeta + if in.Shape != nil { + in, out := &in.Shape, &out.Shape + *out = make([]UtilizationShapePoint, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/scheduler/framework/plugins/volumebinding/volume_binding.go b/pkg/scheduler/framework/plugins/volumebinding/volume_binding.go index 5736dc036a7..9b9e9b19a32 100644 --- a/pkg/scheduler/framework/plugins/volumebinding/volume_binding.go +++ b/pkg/scheduler/framework/plugins/volumebinding/volume_binding.go @@ -221,20 +221,6 @@ func (pl *VolumeBinding) Filter(ctx context.Context, cs *framework.CycleState, p return nil } -var ( - // TODO (for alpha) make it configurable in config.VolumeBindingArgs - defaultShapePoint = []config.UtilizationShapePoint{ - { - Utilization: 0, - Score: 0, - }, - { - Utilization: 100, - Score: int32(config.MaxCustomPriorityScore), - }, - } -) - // Score invoked at the score extension point. func (pl *VolumeBinding) Score(ctx context.Context, cs *framework.CycleState, pod *v1.Pod, nodeName string) (int64, *framework.Status) { if pl.scorer == nil { @@ -363,8 +349,8 @@ func New(plArgs runtime.Object, fh framework.Handle) (framework.Plugin, error) { // build score function var scorer volumeCapacityScorer if utilfeature.DefaultFeatureGate.Enabled(features.VolumeCapacityPriority) { - shape := make(helper.FunctionShape, 0, len(defaultShapePoint)) - for _, point := range defaultShapePoint { + shape := make(helper.FunctionShape, 0, len(args.Shape)) + for _, point := range args.Shape { shape = append(shape, helper.FunctionShapePoint{ Utilization: int64(point.Utilization), Score: int64(point.Score) * (framework.MaxNodeScore / config.MaxCustomPriorityScore), diff --git a/pkg/scheduler/framework/plugins/volumebinding/volume_binding_test.go b/pkg/scheduler/framework/plugins/volumebinding/volume_binding_test.go index e1df85f4589..9268c2498e0 100644 --- a/pkg/scheduler/framework/plugins/volumebinding/volume_binding_test.go +++ b/pkg/scheduler/framework/plugins/volumebinding/volume_binding_test.go @@ -62,6 +62,17 @@ var ( }, VolumeBindingMode: &waitForFirstConsumer, } + + defaultShapePoint = []config.UtilizationShapePoint{ + { + Utilization: 0, + Score: 0, + }, + { + Utilization: 100, + Score: int32(config.MaxCustomPriorityScore), + }, + } ) func makeNode(name string) *v1.Node { @@ -178,6 +189,7 @@ func TestVolumeBinding(t *testing.T) { pvcs []*v1.PersistentVolumeClaim pvs []*v1.PersistentVolume feature featuregate.Feature + args *config.VolumeBindingArgs wantPreFilterStatus *framework.Status wantStateAfterPreFilter *stateData wantFilterStatus []*framework.Status @@ -524,6 +536,99 @@ func TestVolumeBinding(t *testing.T) { 0, }, }, + { + name: "zonal volumes with close capacity are preferred (custom shape)", + pod: makePod("pod-a", []string{"pvc-a"}), + nodes: []*v1.Node{ + mergeNodeLabels(makeNode("zone-a-node-a"), map[string]string{ + "topology.kubernetes.io/region": "region-a", + "topology.kubernetes.io/zone": "zone-a", + }), + mergeNodeLabels(makeNode("zone-a-node-b"), map[string]string{ + "topology.kubernetes.io/region": "region-a", + "topology.kubernetes.io/zone": "zone-a", + }), + mergeNodeLabels(makeNode("zone-b-node-a"), map[string]string{ + "topology.kubernetes.io/region": "region-b", + "topology.kubernetes.io/zone": "zone-b", + }), + mergeNodeLabels(makeNode("zone-b-node-b"), map[string]string{ + "topology.kubernetes.io/region": "region-b", + "topology.kubernetes.io/zone": "zone-b", + }), + mergeNodeLabels(makeNode("zone-c-node-a"), map[string]string{ + "topology.kubernetes.io/region": "region-c", + "topology.kubernetes.io/zone": "zone-c", + }), + mergeNodeLabels(makeNode("zone-c-node-b"), map[string]string{ + "topology.kubernetes.io/region": "region-c", + "topology.kubernetes.io/zone": "zone-c", + }), + }, + pvcs: []*v1.PersistentVolumeClaim{ + setPVCRequestStorage(makePVC("pvc-a", "", waitSC.Name), resource.MustParse("50Gi")), + }, + pvs: []*v1.PersistentVolume{ + setPVNodeAffinity(setPVCapacity(makePV("pv-a-0", waitSC.Name), resource.MustParse("200Gi")), map[string][]string{ + "topology.kubernetes.io/region": {"region-a"}, + "topology.kubernetes.io/zone": {"zone-a"}, + }), + setPVNodeAffinity(setPVCapacity(makePV("pv-a-1", waitSC.Name), resource.MustParse("200Gi")), map[string][]string{ + "topology.kubernetes.io/region": {"region-a"}, + "topology.kubernetes.io/zone": {"zone-a"}, + }), + setPVNodeAffinity(setPVCapacity(makePV("pv-b-0", waitSC.Name), resource.MustParse("100Gi")), map[string][]string{ + "topology.kubernetes.io/region": {"region-b"}, + "topology.kubernetes.io/zone": {"zone-b"}, + }), + setPVNodeAffinity(setPVCapacity(makePV("pv-b-1", waitSC.Name), resource.MustParse("100Gi")), map[string][]string{ + "topology.kubernetes.io/region": {"region-b"}, + "topology.kubernetes.io/zone": {"zone-b"}, + }), + }, + feature: features.VolumeCapacityPriority, + args: &config.VolumeBindingArgs{ + BindTimeoutSeconds: 300, + Shape: []config.UtilizationShapePoint{ + { + Utilization: 0, + Score: 0, + }, + { + Utilization: 50, + Score: 3, + }, + { + Utilization: 100, + Score: 5, + }, + }, + }, + wantPreFilterStatus: nil, + wantStateAfterPreFilter: &stateData{ + boundClaims: []*v1.PersistentVolumeClaim{}, + claimsToBind: []*v1.PersistentVolumeClaim{ + setPVCRequestStorage(makePVC("pvc-a", "", waitSC.Name), resource.MustParse("50Gi")), + }, + podVolumesByNode: map[string]*scheduling.PodVolumes{}, + }, + wantFilterStatus: []*framework.Status{ + nil, + nil, + nil, + nil, + framework.NewStatus(framework.UnschedulableAndUnresolvable, `node(s) didn't find available persistent volumes to bind`), + framework.NewStatus(framework.UnschedulableAndUnresolvable, `node(s) didn't find available persistent volumes to bind`), + }, + wantScores: []int64{ + 15, + 15, + 30, + 30, + 0, + 0, + }, + }, } for _, item := range table { @@ -543,9 +648,19 @@ func TestVolumeBinding(t *testing.T) { if err != nil { t.Fatal(err) } - pl, err := New(&config.VolumeBindingArgs{ - BindTimeoutSeconds: 300, - }, fh) + + args := item.args + if args == nil { + // default args if the args is not specified in test cases + args = &config.VolumeBindingArgs{ + BindTimeoutSeconds: 300, + } + if utilfeature.DefaultFeatureGate.Enabled(features.VolumeCapacityPriority) { + args.Shape = defaultShapePoint + } + } + + pl, err := New(args, fh) if err != nil { t.Fatal(err) } diff --git a/staging/src/k8s.io/kube-scheduler/config/v1beta1/types_pluginargs.go b/staging/src/k8s.io/kube-scheduler/config/v1beta1/types_pluginargs.go index bcdab36fa59..d5c4fc961a8 100644 --- a/staging/src/k8s.io/kube-scheduler/config/v1beta1/types_pluginargs.go +++ b/staging/src/k8s.io/kube-scheduler/config/v1beta1/types_pluginargs.go @@ -226,6 +226,22 @@ type VolumeBindingArgs struct { // Value must be non-negative integer. The value zero indicates no waiting. // If this value is nil, the default value (600) will be used. BindTimeoutSeconds *int64 `json:"bindTimeoutSeconds,omitempty"` + + // Shape specifies the points defining the score function shape, which is + // used to score nodes based on the utilization of statically provisioned + // PVs. The utilization is calculated by dividing the total requested + // storage of the pod by the total capacity of feasible PVs on each node. + // Each point contains utilization (ranges from 0 to 100) and its + // associated score (ranges from 0 to 10). You can turn the priority by + // specifying different scores for different utilization numbers. + // The default shape points are: + // 1) 0 for 0 utilization + // 2) 10 for 100 utilization + // All points must be sorted in increasing order by utilization. + // +featureGate=VolumeCapacityPriority + // +optional + // +listType=atomic + Shape []UtilizationShapePoint `json:"shape,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/staging/src/k8s.io/kube-scheduler/config/v1beta1/zz_generated.deepcopy.go b/staging/src/k8s.io/kube-scheduler/config/v1beta1/zz_generated.deepcopy.go index 8748a8f62b1..dbf5a1a1f32 100644 --- a/staging/src/k8s.io/kube-scheduler/config/v1beta1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/kube-scheduler/config/v1beta1/zz_generated.deepcopy.go @@ -727,6 +727,11 @@ func (in *VolumeBindingArgs) DeepCopyInto(out *VolumeBindingArgs) { *out = new(int64) **out = **in } + if in.Shape != nil { + in, out := &in.Shape, &out.Shape + *out = make([]UtilizationShapePoint, len(*in)) + copy(*out, *in) + } return } diff --git a/staging/src/k8s.io/kube-scheduler/config/v1beta2/types_pluginargs.go b/staging/src/k8s.io/kube-scheduler/config/v1beta2/types_pluginargs.go index 69a09c721f8..cc34d5ee681 100644 --- a/staging/src/k8s.io/kube-scheduler/config/v1beta2/types_pluginargs.go +++ b/staging/src/k8s.io/kube-scheduler/config/v1beta2/types_pluginargs.go @@ -142,6 +142,22 @@ type VolumeBindingArgs struct { // Value must be non-negative integer. The value zero indicates no waiting. // If this value is nil, the default value (600) will be used. BindTimeoutSeconds *int64 `json:"bindTimeoutSeconds,omitempty"` + + // Shape specifies the points defining the score function shape, which is + // used to score nodes based on the utilization of statically provisioned + // PVs. The utilization is calculated by dividing the total requested + // storage of the pod by the total capacity of feasible PVs on each node. + // Each point contains utilization (ranges from 0 to 100) and its + // associated score (ranges from 0 to 10). You can turn the priority by + // specifying different scores for different utilization numbers. + // The default shape points are: + // 1) 0 for 0 utilization + // 2) 10 for 100 utilization + // All points must be sorted in increasing order by utilization. + // +featureGate=VolumeCapacityPriority + // +optional + // +listType=atomic + Shape []UtilizationShapePoint `json:"shape,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/staging/src/k8s.io/kube-scheduler/config/v1beta2/zz_generated.deepcopy.go b/staging/src/k8s.io/kube-scheduler/config/v1beta2/zz_generated.deepcopy.go index 40eca816332..3ef22c8f2bd 100644 --- a/staging/src/k8s.io/kube-scheduler/config/v1beta2/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/kube-scheduler/config/v1beta2/zz_generated.deepcopy.go @@ -508,6 +508,11 @@ func (in *VolumeBindingArgs) DeepCopyInto(out *VolumeBindingArgs) { *out = new(int64) **out = **in } + if in.Shape != nil { + in, out := &in.Shape, &out.Shape + *out = make([]UtilizationShapePoint, len(*in)) + copy(*out, *in) + } return }