github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-21 19:01:49 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #101946 from chendave/balance_allocation

Browse Source 
Support extended resource in NodeResourcesBalancedAllocation plugin
This commit is contained in:
Kubernetes Prow Robot 2021-07-06 10:42:19 -07:00 committed by GitHub
commit ea3bcbc205
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 814 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
cmd/kube-scheduler/app/options/options_test.go
View File

@ -747,6 +747,12 @@ profiles:
Name: "NodeAffinity",
Args: &kubeschedulerconfig.NodeAffinityArgs{},
},
{
Name: "NodeResourcesBalancedAllocation",
Args: &kubeschedulerconfig.NodeResourcesBalancedAllocationArgs{
Resources: []kubeschedulerconfig.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}},
},
},
{
Name: "NodeResourcesFit",
Args: &kubeschedulerconfig.NodeResourcesFitArgs{
@ -864,6 +870,12 @@ profiles:
Name: "NodeAffinity",
Args: &kubeschedulerconfig.NodeAffinityArgs{},
},
{
Name: "NodeResourcesBalancedAllocation",
Args: &kubeschedulerconfig.NodeResourcesBalancedAllocationArgs{
Resources: []kubeschedulerconfig.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}},
},
},
{
Name: "NodeResourcesFit",
Args: &kubeschedulerconfig.NodeResourcesFitArgs{
@ -986,6 +998,12 @@ profiles:
Name: "NodeAffinity",
Args: &kubeschedulerconfig.NodeAffinityArgs{},
},
{
Name: "NodeResourcesBalancedAllocation",
Args: &kubeschedulerconfig.NodeResourcesBalancedAllocationArgs{
Resources: []kubeschedulerconfig.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}},
},
},
{
Name: "NodeResourcesFit",
Args: &kubeschedulerconfig.NodeResourcesFitArgs{
@ -1102,6 +1120,12 @@ profiles:
Name: "NodeAffinity",
Args: &kubeschedulerconfig.NodeAffinityArgs{},
},
{
Name: "NodeResourcesBalancedAllocation",
Args: &kubeschedulerconfig.NodeResourcesBalancedAllocationArgs{
Resources: []kubeschedulerconfig.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}},
},
},
{
Name: "NodeResourcesFit",
Args: &kubeschedulerconfig.NodeResourcesFitArgs{

1
pkg/scheduler/apis/config/register.go
View File

@ -49,6 +49,7 @@ func addKnownTypes(scheme *runtime.Scheme) error {
&VolumeBindingArgs{},
&NodeResourcesLeastAllocatedArgs{},
&NodeResourcesMostAllocatedArgs{},
&NodeResourcesBalancedAllocationArgs{},
&NodeAffinityArgs{},
)
scheme.AddKnownTypes(schema.GroupVersion{Group: "", Version: runtime.APIVersionInternal}, &Policy{})

88
pkg/scheduler/apis/config/scheme/scheme_test.go
View File

@ -87,6 +87,19 @@ profiles:
resources:
- name: memory
weight: 1
- name: NodeResourcesBalancedAllocation
args:
resources:
- name: cpu # default weight(1) will be set.
- name: memory # weight 0 will be replaced by 1.
weight: 0
- name: scalar0
weight: 1
- name: scalar1 # default weight(1) will be set for scalar1
- name: scalar2 # weight 0 will be replaced by 1.
weight: 0
- name: scalar3
weight: 2
- name: VolumeBinding
args:
bindTimeoutSeconds: 300
@ -164,6 +177,18 @@ profiles:
Resources: []config.ResourceSpec{{Name: "memory", Weight: 1}},
},
},
{
Name: "NodeResourcesBalancedAllocation",
Args: &config.NodeResourcesBalancedAllocationArgs{
Resources: []config.ResourceSpec{
{Name: "cpu", Weight: 1},
{Name: "memory", Weight: 1},
{Name: "scalar0", Weight: 1},
{Name: "scalar1", Weight: 1},
{Name: "scalar2", Weight: 1},
{Name: "scalar3", Weight: 2}},
},
},
{
Name: "VolumeBinding",
Args: &config.VolumeBindingArgs{
@ -312,6 +337,8 @@ profiles:
args:
- name: NodeResourcesMostAllocated
args:
- name: NodeResourcesBalancedAllocation
args:
- name: VolumeBinding
args:
- name: PodTopologySpread
@ -357,6 +384,12 @@ profiles:
Resources: []config.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}},
},
},
{
Name: "NodeResourcesBalancedAllocation",
Args: &config.NodeResourcesBalancedAllocationArgs{
Resources: []config.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}},
},
},
{
Name: "VolumeBinding",
Args: &config.VolumeBindingArgs{
@ -413,6 +446,19 @@ profiles:
- key: foo
operator: In
values: ["bar"]
- name: NodeResourcesBalancedAllocation
args:
resources:
- name: cpu # default weight(1) will be set.
- name: memory # weight 0 will be replaced by 1.
weight: 0
- name: scalar0
weight: 1
- name: scalar1 # default weight(1) will be set for scalar1
- name: scalar2 # weight 0 will be replaced by 1.
weight: 0
- name: scalar3
weight: 2
`),
wantProfiles: []config.KubeSchedulerProfile{
{
@ -475,6 +521,18 @@ profiles:
},
},
},
{
Name: "NodeResourcesBalancedAllocation",
Args: &config.NodeResourcesBalancedAllocationArgs{
Resources: []config.ResourceSpec{
{Name: "cpu", Weight: 1},
{Name: "memory", Weight: 1},
{Name: "scalar0", Weight: 1},
{Name: "scalar1", Weight: 1},
{Name: "scalar2", Weight: 1},
{Name: "scalar3", Weight: 2}},
},
},
},
},
},
@ -511,6 +569,12 @@ profiles:
Name: "NodeAffinity",
Args: &config.NodeAffinityArgs{},
},
{
Name: "NodeResourcesBalancedAllocation",
Args: &config.NodeResourcesBalancedAllocationArgs{
Resources: []config.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}},
},
},
{
Name: "NodeResourcesFit",
Args: &config.NodeResourcesFitArgs{
@ -631,6 +695,7 @@ profiles:
args:
- name: PodTopologySpread
- name: NodeAffinity
- name: NodeResourcesBalancedAllocation
`),
wantProfiles: []config.KubeSchedulerProfile{
{
@ -676,6 +741,12 @@ profiles:
Name: "NodeAffinity",
Args: &config.NodeAffinityArgs{},
},
{
Name: "NodeResourcesBalancedAllocation",
Args: &config.NodeResourcesBalancedAllocationArgs{
Resources: []config.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}},
},
},
},
},
},
@ -769,6 +840,16 @@ func TestCodecsEncodePluginConfig(t *testing.T) {
},
},
},
{
Name: "NodeResourcesBalancedAllocation",
Args: runtime.RawExtension{
Object: &v1beta1.NodeResourcesBalancedAllocationArgs{
Resources: []v1beta1.ResourceSpec{
{Name: "mem", Weight: 1},
},
},
},
},
{
Name: "PodTopologySpread",
Args: runtime.RawExtension{
@ -837,6 +918,13 @@ profiles:
- name: mem
weight: 2
name: NodeResourcesLeastAllocated
- args:
apiVersion: kubescheduler.config.k8s.io/v1beta1
kind: NodeResourcesBalancedAllocationArgs
resources:
- name: mem
weight: 1
name: NodeResourcesBalancedAllocation
- args:
apiVersion: kubescheduler.config.k8s.io/v1beta1
kind: PodTopologySpreadArgs

12
pkg/scheduler/apis/config/testing/defaults/defaults.go
View File

@ -123,6 +123,12 @@ var PluginConfigsV1beta1 = []config.PluginConfig{
Name: "NodeAffinity",
Args: &config.NodeAffinityArgs{},
},
{
Name: "NodeResourcesBalancedAllocation",
Args: &config.NodeResourcesBalancedAllocationArgs{
Resources: []config.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}},
},
},
{
Name: "NodeResourcesFit",
Args: &config.NodeResourcesFitArgs{
@ -253,6 +259,12 @@ var PluginConfigsV1beta2 = []config.PluginConfig{
Name: "NodeAffinity",
Args: &config.NodeAffinityArgs{},
},
{
Name: "NodeResourcesBalancedAllocation",
Args: &config.NodeResourcesBalancedAllocationArgs{
Resources: []config.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}},
},
},
{
Name: "NodeResourcesFit",
Args: &config.NodeResourcesFitArgs{

11
pkg/scheduler/apis/config/types_pluginargs.go
View File

@ -169,6 +169,17 @@ type NodeResourcesMostAllocatedArgs struct {
Resources []ResourceSpec
}
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// NodeResourcesBalancedAllocationArgs holds arguments used to configure NodeResourcesBalancedAllocation plugin.
type NodeResourcesBalancedAllocationArgs struct {
metav1.TypeMeta
// Resources to be considered when scoring.
// The default resource set includes "cpu" and "memory", only valid weight is 1.
Resources []ResourceSpec
}
// UtilizationShapePoint represents a single point of a priority function shape.
type UtilizationShapePoint struct {
// Utilization (x axis). Valid values are 0 to 100. Fully utilized node maps to 100.

15
pkg/scheduler/apis/config/v1beta1/defaults.go
View File

@ -294,6 +294,21 @@ func SetDefaults_VolumeBindingArgs(obj *v1beta1.VolumeBindingArgs) {
}
}
func SetDefaults_NodeResourcesBalancedAllocationArgs(obj *v1beta1.NodeResourcesBalancedAllocationArgs) {
if len(obj.Resources) == 0 {
obj.Resources = append(obj.Resources,
v1beta1.ResourceSpec{Name: string(corev1.ResourceCPU), Weight: 1},
v1beta1.ResourceSpec{Name: string(corev1.ResourceMemory), Weight: 1},
)
}
// If the weight is not set or it is explicitly set to 0, then apply the default weight(1) instead.
for i := range obj.Resources {
if obj.Resources[i].Weight == 0 {
obj.Resources[i].Weight = 1
}
}
}
func SetDefaults_PodTopologySpreadArgs(obj *v1beta1.PodTopologySpreadArgs) {
if feature.DefaultFeatureGate.Enabled(features.DefaultPodTopologySpread) {
if obj.DefaultingType == "" {

74
pkg/scheduler/apis/config/v1beta1/defaults_test.go
View File

@ -68,6 +68,16 @@ var pluginConfigs = []v1beta1.PluginConfig{
},
}},
},
{
Name: "NodeResourcesBalancedAllocation",
Args: runtime.RawExtension{Object: &v1beta1.NodeResourcesBalancedAllocationArgs{
TypeMeta: metav1.TypeMeta{
Kind: "NodeResourcesBalancedAllocationArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta1",
},
Resources: []v1beta1.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}},
}},
},
{
Name: "NodeResourcesFit",
Args: runtime.RawExtension{Object: &v1beta1.NodeResourcesFitArgs{
@ -280,6 +290,16 @@ func TestSchedulerDefaults(t *testing.T) {
},
}},
},
{
Name: "NodeResourcesBalancedAllocation",
Args: runtime.RawExtension{Object: &v1beta1.NodeResourcesBalancedAllocationArgs{
TypeMeta: metav1.TypeMeta{
Kind: "NodeResourcesBalancedAllocationArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta1",
},
Resources: []v1beta1.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}},
}},
},
{
Name: "NodeResourcesFit",
Args: runtime.RawExtension{Object: &v1beta1.NodeResourcesFitArgs{
@ -637,6 +657,60 @@ func TestPluginArgsDefaults(t *testing.T) {
},
},
},
{
name: "NodeResourcesBalancedAllocationArgs resources empty",
in: &v1beta1.NodeResourcesBalancedAllocationArgs{},
want: &v1beta1.NodeResourcesBalancedAllocationArgs{
Resources: []v1beta1.ResourceSpec{
{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1},
},
},
},
{
name: "NodeResourcesBalancedAllocationArgs with scalar resource",
in: &v1beta1.NodeResourcesBalancedAllocationArgs{
Resources: []v1beta1.ResourceSpec{
{Name: "scalar.io/scalar1", Weight: 1},
},
},
want: &v1beta1.NodeResourcesBalancedAllocationArgs{
Resources: []v1beta1.ResourceSpec{
{Name: "scalar.io/scalar1", Weight: 1},
},
},
},
{
name: "NodeResourcesBalancedAllocationArgs with mixed resources",
in: &v1beta1.NodeResourcesBalancedAllocationArgs{
Resources: []v1beta1.ResourceSpec{
{Name: string(v1.ResourceCPU), Weight: 1},
{Name: "scalar.io/scalar1", Weight: 1},
},
},
want: &v1beta1.NodeResourcesBalancedAllocationArgs{
Resources: []v1beta1.ResourceSpec{
{Name: string(v1.ResourceCPU), Weight: 1},
{Name: "scalar.io/scalar1", Weight: 1},
},
},
},
{
name: "NodeResourcesBalancedAllocationArgs have resource no weight",
in: &v1beta1.NodeResourcesBalancedAllocationArgs{
Resources: []v1beta1.ResourceSpec{
{Name: string(v1.ResourceCPU)},
{Name: "scalar.io/scalar0"},
{Name: "scalar.io/scalar1", Weight: 1},
},
},
want: &v1beta1.NodeResourcesBalancedAllocationArgs{
Resources: []v1beta1.ResourceSpec{
{Name: string(v1.ResourceCPU), Weight: 1},
{Name: "scalar.io/scalar0", Weight: 1},
{Name: "scalar.io/scalar1", Weight: 1},
},
},
},
{
name: "PodTopologySpreadArgs resources empty",
in: &v1beta1.PodTopologySpreadArgs{},

30
pkg/scheduler/apis/config/v1beta1/zz_generated.conversion.go generated
View File

@ -100,6 +100,16 @@ func RegisterConversions(s *runtime.Scheme) error {
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*v1beta1.NodeResourcesBalancedAllocationArgs)(nil), (*config.NodeResourcesBalancedAllocationArgs)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_v1beta1_NodeResourcesBalancedAllocationArgs_To_config_NodeResourcesBalancedAllocationArgs(a.(*v1beta1.NodeResourcesBalancedAllocationArgs), b.(*config.NodeResourcesBalancedAllocationArgs), scope)
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*config.NodeResourcesBalancedAllocationArgs)(nil), (*v1beta1.NodeResourcesBalancedAllocationArgs)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_config_NodeResourcesBalancedAllocationArgs_To_v1beta1_NodeResourcesBalancedAllocationArgs(a.(*config.NodeResourcesBalancedAllocationArgs), b.(*v1beta1.NodeResourcesBalancedAllocationArgs), scope)
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*v1beta1.NodeResourcesFitArgs)(nil), (*config.NodeResourcesFitArgs)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_v1beta1_NodeResourcesFitArgs_To_config_NodeResourcesFitArgs(a.(*v1beta1.NodeResourcesFitArgs), b.(*config.NodeResourcesFitArgs), scope)
}); err != nil {
@ -555,6 +565,26 @@ func Convert_config_NodeLabelArgs_To_v1beta1_NodeLabelArgs(in *config.NodeLabelA
return autoConvert_config_NodeLabelArgs_To_v1beta1_NodeLabelArgs(in, out, s)
}
func autoConvert_v1beta1_NodeResourcesBalancedAllocationArgs_To_config_NodeResourcesBalancedAllocationArgs(in *v1beta1.NodeResourcesBalancedAllocationArgs, out *config.NodeResourcesBalancedAllocationArgs, s conversion.Scope) error {
out.Resources = *(*[]config.ResourceSpec)(unsafe.Pointer(&in.Resources))
return nil
}
// Convert_v1beta1_NodeResourcesBalancedAllocationArgs_To_config_NodeResourcesBalancedAllocationArgs is an autogenerated conversion function.
func Convert_v1beta1_NodeResourcesBalancedAllocationArgs_To_config_NodeResourcesBalancedAllocationArgs(in *v1beta1.NodeResourcesBalancedAllocationArgs, out *config.NodeResourcesBalancedAllocationArgs, s conversion.Scope) error {
return autoConvert_v1beta1_NodeResourcesBalancedAllocationArgs_To_config_NodeResourcesBalancedAllocationArgs(in, out, s)
}
func autoConvert_config_NodeResourcesBalancedAllocationArgs_To_v1beta1_NodeResourcesBalancedAllocationArgs(in *config.NodeResourcesBalancedAllocationArgs, out *v1beta1.NodeResourcesBalancedAllocationArgs, s conversion.Scope) error {
out.Resources = *(*[]v1beta1.ResourceSpec)(unsafe.Pointer(&in.Resources))
return nil
}
// Convert_config_NodeResourcesBalancedAllocationArgs_To_v1beta1_NodeResourcesBalancedAllocationArgs is an autogenerated conversion function.
func Convert_config_NodeResourcesBalancedAllocationArgs_To_v1beta1_NodeResourcesBalancedAllocationArgs(in *config.NodeResourcesBalancedAllocationArgs, out *v1beta1.NodeResourcesBalancedAllocationArgs, s conversion.Scope) error {
return autoConvert_config_NodeResourcesBalancedAllocationArgs_To_v1beta1_NodeResourcesBalancedAllocationArgs(in, out, s)
}
func autoConvert_v1beta1_NodeResourcesFitArgs_To_config_NodeResourcesFitArgs(in *v1beta1.NodeResourcesFitArgs, out *config.NodeResourcesFitArgs, s conversion.Scope) error {
out.IgnoredResources = *(*[]string)(unsafe.Pointer(&in.IgnoredResources))
out.IgnoredResourceGroups = *(*[]string)(unsafe.Pointer(&in.IgnoredResourceGroups))

7
pkg/scheduler/apis/config/v1beta1/zz_generated.defaults.go generated
View File

@ -34,6 +34,9 @@ func RegisterDefaults(scheme *runtime.Scheme) error {
scheme.AddTypeDefaultingFunc(&v1beta1.KubeSchedulerConfiguration{}, func(obj interface{}) {
SetObjectDefaults_KubeSchedulerConfiguration(obj.(*v1beta1.KubeSchedulerConfiguration))
})
scheme.AddTypeDefaultingFunc(&v1beta1.NodeResourcesBalancedAllocationArgs{}, func(obj interface{}) {
SetObjectDefaults_NodeResourcesBalancedAllocationArgs(obj.(*v1beta1.NodeResourcesBalancedAllocationArgs))
})
scheme.AddTypeDefaultingFunc(&v1beta1.NodeResourcesFitArgs{}, func(obj interface{}) { SetObjectDefaults_NodeResourcesFitArgs(obj.(*v1beta1.NodeResourcesFitArgs)) })
scheme.AddTypeDefaultingFunc(&v1beta1.NodeResourcesLeastAllocatedArgs{}, func(obj interface{}) {
SetObjectDefaults_NodeResourcesLeastAllocatedArgs(obj.(*v1beta1.NodeResourcesLeastAllocatedArgs))
@ -61,6 +64,10 @@ func SetObjectDefaults_KubeSchedulerConfiguration(in *v1beta1.KubeSchedulerConfi
SetDefaults_KubeSchedulerConfiguration(in)
}
func SetObjectDefaults_NodeResourcesBalancedAllocationArgs(in *v1beta1.NodeResourcesBalancedAllocationArgs) {
SetDefaults_NodeResourcesBalancedAllocationArgs(in)
}
func SetObjectDefaults_NodeResourcesFitArgs(in *v1beta1.NodeResourcesFitArgs) {
SetDefaults_NodeResourcesFitArgs(in)
}

20
pkg/scheduler/apis/config/v1beta2/defaults.go
View File

@ -20,7 +20,6 @@ import (
"net"
"strconv"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/sets"
@ -33,8 +32,8 @@ import (
)
var defaultResourceSpec = []v1beta2.ResourceSpec{
{Name: string(corev1.ResourceCPU), Weight: 1},
{Name: string(corev1.ResourceMemory), Weight: 1},
{Name: string(v1.ResourceCPU), Weight: 1},
{Name: string(v1.ResourceMemory), Weight: 1},
}
func addDefaultingFuncs(scheme *runtime.Scheme) error {
@ -259,6 +258,21 @@ func SetDefaults_VolumeBindingArgs(obj *v1beta2.VolumeBindingArgs) {
}
}
func SetDefaults_NodeResourcesBalancedAllocationArgs(obj *v1beta2.NodeResourcesBalancedAllocationArgs) {
if len(obj.Resources) == 0 {
obj.Resources = append(obj.Resources,
v1beta2.ResourceSpec{Name: string(v1.ResourceCPU), Weight: 1},
v1beta2.ResourceSpec{Name: string(v1.ResourceMemory), Weight: 1},
)
}
// If the weight is not set or it is explicitly set to 0, then apply the default weight(1) instead.
for i := range obj.Resources {
if obj.Resources[i].Weight == 0 {
obj.Resources[i].Weight = 1
}
}
}
func SetDefaults_PodTopologySpreadArgs(obj *v1beta2.PodTopologySpreadArgs) {
if feature.DefaultFeatureGate.Enabled(features.DefaultPodTopologySpread) {
if obj.DefaultingType == "" {

74
pkg/scheduler/apis/config/v1beta2/defaults_test.go
View File

@ -69,6 +69,16 @@ var pluginConfigs = []v1beta2.PluginConfig{
},
}},
},
{
Name: "NodeResourcesBalancedAllocation",
Args: runtime.RawExtension{Object: &v1beta2.NodeResourcesBalancedAllocationArgs{
TypeMeta: metav1.TypeMeta{
Kind: "NodeResourcesBalancedAllocationArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta2",
},
Resources: []v1beta2.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}},
}},
},
{
Name: "NodeResourcesFit",
Args: runtime.RawExtension{Object: &v1beta2.NodeResourcesFitArgs{
@ -273,6 +283,16 @@ func TestSchedulerDefaults(t *testing.T) {
},
}},
},
{
Name: "NodeResourcesBalancedAllocation",
Args: runtime.RawExtension{Object: &v1beta2.NodeResourcesBalancedAllocationArgs{
TypeMeta: metav1.TypeMeta{
Kind: "NodeResourcesBalancedAllocationArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta2",
},
Resources: []v1beta2.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}},
}},
},
{
Name: "NodeResourcesFit",
Args: runtime.RawExtension{Object: &v1beta2.NodeResourcesFitArgs{
@ -612,6 +632,60 @@ func TestPluginArgsDefaults(t *testing.T) {
HardPodAffinityWeight: pointer.Int32Ptr(5),
},
},
{
name: "NodeResourcesBalancedAllocationArgs resources empty",
in: &v1beta2.NodeResourcesBalancedAllocationArgs{},
want: &v1beta2.NodeResourcesBalancedAllocationArgs{
Resources: []v1beta2.ResourceSpec{
{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1},
},
},
},
{
name: "NodeResourcesBalancedAllocationArgs with scalar resource",
in: &v1beta2.NodeResourcesBalancedAllocationArgs{
Resources: []v1beta2.ResourceSpec{
{Name: "scalar.io/scalar1", Weight: 1},
},
},
want: &v1beta2.NodeResourcesBalancedAllocationArgs{
Resources: []v1beta2.ResourceSpec{
{Name: "scalar.io/scalar1", Weight: 1},
},
},
},
{
name: "NodeResourcesBalancedAllocationArgs with mixed resources",
in: &v1beta2.NodeResourcesBalancedAllocationArgs{
Resources: []v1beta2.ResourceSpec{
{Name: string(v1.ResourceCPU), Weight: 1},
{Name: "scalar.io/scalar1", Weight: 1},
},
},
want: &v1beta2.NodeResourcesBalancedAllocationArgs{
Resources: []v1beta2.ResourceSpec{
{Name: string(v1.ResourceCPU), Weight: 1},
{Name: "scalar.io/scalar1", Weight: 1},
},
},
},
{
name: "NodeResourcesBalancedAllocationArgs have resource no weight",
in: &v1beta2.NodeResourcesBalancedAllocationArgs{
Resources: []v1beta2.ResourceSpec{
{Name: string(v1.ResourceCPU)},
{Name: "scalar.io/scalar0"},
{Name: "scalar.io/scalar1", Weight: 1},
},
},
want: &v1beta2.NodeResourcesBalancedAllocationArgs{
Resources: []v1beta2.ResourceSpec{
{Name: string(v1.ResourceCPU), Weight: 1},
{Name: "scalar.io/scalar0", Weight: 1},
{Name: "scalar.io/scalar1", Weight: 1},
},
},
},
{
name: "PodTopologySpreadArgs resources empty",
in: &v1beta2.PodTopologySpreadArgs{},

30
pkg/scheduler/apis/config/v1beta2/zz_generated.conversion.go generated
View File

@ -90,6 +90,16 @@ func RegisterConversions(s *runtime.Scheme) error {
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*v1beta2.NodeResourcesBalancedAllocationArgs)(nil), (*config.NodeResourcesBalancedAllocationArgs)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_v1beta2_NodeResourcesBalancedAllocationArgs_To_config_NodeResourcesBalancedAllocationArgs(a.(*v1beta2.NodeResourcesBalancedAllocationArgs), b.(*config.NodeResourcesBalancedAllocationArgs), scope)
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*config.NodeResourcesBalancedAllocationArgs)(nil), (*v1beta2.NodeResourcesBalancedAllocationArgs)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_config_NodeResourcesBalancedAllocationArgs_To_v1beta2_NodeResourcesBalancedAllocationArgs(a.(*config.NodeResourcesBalancedAllocationArgs), b.(*v1beta2.NodeResourcesBalancedAllocationArgs), scope)
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*v1beta2.NodeResourcesFitArgs)(nil), (*config.NodeResourcesFitArgs)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_v1beta2_NodeResourcesFitArgs_To_config_NodeResourcesFitArgs(a.(*v1beta2.NodeResourcesFitArgs), b.(*config.NodeResourcesFitArgs), scope)
}); err != nil {
@ -479,6 +489,26 @@ func Convert_config_NodeAffinityArgs_To_v1beta2_NodeAffinityArgs(in *config.Node
return autoConvert_config_NodeAffinityArgs_To_v1beta2_NodeAffinityArgs(in, out, s)
}
func autoConvert_v1beta2_NodeResourcesBalancedAllocationArgs_To_config_NodeResourcesBalancedAllocationArgs(in *v1beta2.NodeResourcesBalancedAllocationArgs, out *config.NodeResourcesBalancedAllocationArgs, s conversion.Scope) error {
out.Resources = *(*[]config.ResourceSpec)(unsafe.Pointer(&in.Resources))
return nil
}
// Convert_v1beta2_NodeResourcesBalancedAllocationArgs_To_config_NodeResourcesBalancedAllocationArgs is an autogenerated conversion function.
func Convert_v1beta2_NodeResourcesBalancedAllocationArgs_To_config_NodeResourcesBalancedAllocationArgs(in *v1beta2.NodeResourcesBalancedAllocationArgs, out *config.NodeResourcesBalancedAllocationArgs, s conversion.Scope) error {
return autoConvert_v1beta2_NodeResourcesBalancedAllocationArgs_To_config_NodeResourcesBalancedAllocationArgs(in, out, s)
}
func autoConvert_config_NodeResourcesBalancedAllocationArgs_To_v1beta2_NodeResourcesBalancedAllocationArgs(in *config.NodeResourcesBalancedAllocationArgs, out *v1beta2.NodeResourcesBalancedAllocationArgs, s conversion.Scope) error {
out.Resources = *(*[]v1beta2.ResourceSpec)(unsafe.Pointer(&in.Resources))
return nil
}
// Convert_config_NodeResourcesBalancedAllocationArgs_To_v1beta2_NodeResourcesBalancedAllocationArgs is an autogenerated conversion function.
func Convert_config_NodeResourcesBalancedAllocationArgs_To_v1beta2_NodeResourcesBalancedAllocationArgs(in *config.NodeResourcesBalancedAllocationArgs, out *v1beta2.NodeResourcesBalancedAllocationArgs, s conversion.Scope) error {
return autoConvert_config_NodeResourcesBalancedAllocationArgs_To_v1beta2_NodeResourcesBalancedAllocationArgs(in, out, s)
}
func autoConvert_v1beta2_NodeResourcesFitArgs_To_config_NodeResourcesFitArgs(in *v1beta2.NodeResourcesFitArgs, out *config.NodeResourcesFitArgs, s conversion.Scope) error {
out.IgnoredResources = *(*[]string)(unsafe.Pointer(&in.IgnoredResources))
out.IgnoredResourceGroups = *(*[]string)(unsafe.Pointer(&in.IgnoredResourceGroups))

7
pkg/scheduler/apis/config/v1beta2/zz_generated.defaults.go generated
View File

@ -34,6 +34,9 @@ func RegisterDefaults(scheme *runtime.Scheme) error {
scheme.AddTypeDefaultingFunc(&v1beta2.KubeSchedulerConfiguration{}, func(obj interface{}) {
SetObjectDefaults_KubeSchedulerConfiguration(obj.(*v1beta2.KubeSchedulerConfiguration))
})
scheme.AddTypeDefaultingFunc(&v1beta2.NodeResourcesBalancedAllocationArgs{}, func(obj interface{}) {
SetObjectDefaults_NodeResourcesBalancedAllocationArgs(obj.(*v1beta2.NodeResourcesBalancedAllocationArgs))
})
scheme.AddTypeDefaultingFunc(&v1beta2.NodeResourcesFitArgs{}, func(obj interface{}) { SetObjectDefaults_NodeResourcesFitArgs(obj.(*v1beta2.NodeResourcesFitArgs)) })
scheme.AddTypeDefaultingFunc(&v1beta2.PodTopologySpreadArgs{}, func(obj interface{}) { SetObjectDefaults_PodTopologySpreadArgs(obj.(*v1beta2.PodTopologySpreadArgs)) })
scheme.AddTypeDefaultingFunc(&v1beta2.VolumeBindingArgs{}, func(obj interface{}) { SetObjectDefaults_VolumeBindingArgs(obj.(*v1beta2.VolumeBindingArgs)) })
@ -52,6 +55,10 @@ func SetObjectDefaults_KubeSchedulerConfiguration(in *v1beta2.KubeSchedulerConfi
SetDefaults_KubeSchedulerConfiguration(in)
}
func SetObjectDefaults_NodeResourcesBalancedAllocationArgs(in *v1beta2.NodeResourcesBalancedAllocationArgs) {
SetDefaults_NodeResourcesBalancedAllocationArgs(in)
}
func SetObjectDefaults_NodeResourcesFitArgs(in *v1beta2.NodeResourcesFitArgs) {
SetDefaults_NodeResourcesFitArgs(in)
}

21
pkg/scheduler/apis/config/validation/validation.go
View File

@ -189,16 +189,17 @@ func validateKubeSchedulerProfile(path *field.Path, apiVersion string, profile *
func validatePluginConfig(path *field.Path, apiVersion string, profile *config.KubeSchedulerProfile) []error {
var errs []error
m := map[string]interface{}{
"DefaultPreemption": ValidateDefaultPreemptionArgs,
"InterPodAffinity": ValidateInterPodAffinityArgs,
"NodeAffinity": ValidateNodeAffinityArgs,
"NodeLabel": ValidateNodeLabelArgs,
"NodeResourcesFitArgs": ValidateNodeResourcesFitArgs,
"NodeResourcesLeastAllocated": ValidateNodeResourcesLeastAllocatedArgs,
"NodeResourcesMostAllocated": ValidateNodeResourcesMostAllocatedArgs,
"PodTopologySpread": ValidatePodTopologySpreadArgs,
"RequestedToCapacityRatio": ValidateRequestedToCapacityRatioArgs,
"VolumeBinding": ValidateVolumeBindingArgs,
"DefaultPreemption": ValidateDefaultPreemptionArgs,
"InterPodAffinity": ValidateInterPodAffinityArgs,
"NodeAffinity": ValidateNodeAffinityArgs,
"NodeLabel": ValidateNodeLabelArgs,
"NodeResourcesBalancedAllocation": ValidateNodeResourcesBalancedAllocationArgs,
"NodeResourcesFitArgs": ValidateNodeResourcesFitArgs,
"NodeResourcesLeastAllocated": ValidateNodeResourcesLeastAllocatedArgs,
"NodeResourcesMostAllocated": ValidateNodeResourcesMostAllocatedArgs,
"PodTopologySpread": ValidatePodTopologySpreadArgs,
"RequestedToCapacityRatio": ValidateRequestedToCapacityRatioArgs,
"VolumeBinding": ValidateVolumeBindingArgs,
}
seenPluginConfig := make(sets.String)

17
pkg/scheduler/apis/config/validation/validation_pluginargs.go
View File

@ -271,6 +271,23 @@ func validateResources(resources []config.ResourceSpec, p *field.Path) field.Err
return allErrs
}
// ValidateNodeResourcesBalancedAllocationArgs validates that NodeResourcesBalancedAllocationArgs are set correctly.
func ValidateNodeResourcesBalancedAllocationArgs(path *field.Path, args *config.NodeResourcesBalancedAllocationArgs) error {
var allErrs field.ErrorList
seenResources := sets.NewString()
for i, resource := range args.Resources {
if seenResources.Has(resource.Name) {
allErrs = append(allErrs, field.Duplicate(path.Child("resources").Index(i).Child("name"), resource.Name))
} else {
seenResources.Insert(resource.Name)
}
if resource.Weight != 1 {
allErrs = append(allErrs, field.Invalid(path.Child("resources").Index(i).Child("weight"), resource.Weight, "must be 1"))
}
}
return allErrs.ToAggregate()
}
// ValidateNodeAffinityArgs validates that NodeAffinityArgs are correct.
func ValidateNodeAffinityArgs(path *field.Path, args *config.NodeAffinityArgs) error {
if args.AddedAffinity == nil {

71
pkg/scheduler/apis/config/validation/validation_pluginargs_test.go
View File

@ -855,6 +855,77 @@ func TestValidateNodeResourcesMostAllocatedArgs(t *testing.T) {
}
}
func TestValidateNodeResourcesBalancedAllocationArgs(t *testing.T) {
cases := map[string]struct {
args *config.NodeResourcesBalancedAllocationArgs
wantErrs field.ErrorList
}{
"valid config": {
args: &config.NodeResourcesBalancedAllocationArgs{
Resources: []config.ResourceSpec{
{
Name: "cpu",
Weight: 1,
},
{
Name: "memory",
Weight: 1,
},
},
},
},
"invalid config": {
args: &config.NodeResourcesBalancedAllocationArgs{
Resources: []config.ResourceSpec{
{
Name: "cpu",
Weight: 2,
},
{
Name: "memory",
Weight: 1,
},
},
},
wantErrs: field.ErrorList{
&field.Error{
Type: field.ErrorTypeInvalid,
Field: "resources[0].weight",
},
},
},
"repeated resources": {
args: &config.NodeResourcesBalancedAllocationArgs{
Resources: []config.ResourceSpec{
{
Name: "cpu",
Weight: 1,
},
{
Name: "cpu",
Weight: 1,
},
},
},
wantErrs: field.ErrorList{
&field.Error{
Type: field.ErrorTypeDuplicate,
Field: "resources[1].name",
},
},
},
}
for name, tc := range cases {
t.Run(name, func(t *testing.T) {
err := ValidateNodeResourcesBalancedAllocationArgs(nil, tc.args)
if diff := cmp.Diff(tc.wantErrs.ToAggregate(), err, ignoreBadValueDetail); diff != "" {
t.Errorf("ValidateNodeResourcesBalancedAllocationArgs returned err (-want,+got):\n%s", diff)
}
})
}
}
func TestValidateNodeAffinityArgs(t *testing.T) {
cases := []struct {
name string

30
pkg/scheduler/apis/config/zz_generated.deepcopy.go generated
View File

@ -331,6 +331,36 @@ func (in *NodeLabelArgs) DeepCopyObject() runtime.Object {
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NodeResourcesBalancedAllocationArgs) DeepCopyInto(out *NodeResourcesBalancedAllocationArgs) {
*out = *in
out.TypeMeta = in.TypeMeta
if in.Resources != nil {
in, out := &in.Resources, &out.Resources
*out = make([]ResourceSpec, len(*in))
copy(*out, *in)
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeResourcesBalancedAllocationArgs.
func (in *NodeResourcesBalancedAllocationArgs) DeepCopy() *NodeResourcesBalancedAllocationArgs {
if in == nil {
return nil
}
out := new(NodeResourcesBalancedAllocationArgs)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *NodeResourcesBalancedAllocationArgs) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NodeResourcesFitArgs) DeepCopyInto(out *NodeResourcesFitArgs) {
*out = *in

6
pkg/scheduler/factory_test.go
View File

@ -124,6 +124,12 @@ func TestCreateFromConfig(t *testing.T) {
Name: nodeaffinity.Name,
Args: &schedulerapi.NodeAffinityArgs{},
},
{
Name: noderesources.BalancedAllocationName,
Args: &schedulerapi.NodeResourcesBalancedAllocationArgs{
Resources: []schedulerapi.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}},
},
},
{
Name: noderesources.FitName,
Args: &schedulerapi.NodeResourcesFitArgs{

78
pkg/scheduler/framework/plugins/noderesources/balanced_allocation.go
View File

@ -23,6 +23,8 @@ import (
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/kubernetes/pkg/scheduler/apis/config"
"k8s.io/kubernetes/pkg/scheduler/apis/config/validation"
"k8s.io/kubernetes/pkg/scheduler/framework"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/names"
@ -53,11 +55,10 @@ func (ba *BalancedAllocation) Score(ctx context.Context, state *framework.CycleS
}
// ba.score favors nodes with balanced resource usage rate.
// It calculates the difference between the cpu and memory fraction of capacity,
// and prioritizes the host based on how close the two metrics are to each other.
// Detail: score = (1 - variance(cpuFraction,memoryFraction,volumeFraction)) * MaxNodeScore. The algorithm is partly inspired by:
// "Wei Huang et al. An Energy Efficient Virtual Machine Placement Algorithm with Balanced
// Resource Utilization"
// It calculates the standard deviation for those resources and prioritizes the node based on how close the usage of those resources is to each other.
// Detail: score = (1 - std) * MaxNodeScore, where std is calculated by the root square of Σ((fraction(i)-mean)^2)/len(resources)
// The algorithm is partly inspired by:
// "Wei Huang et al. An Energy Efficient Virtual Machine Placement Algorithm with Balanced Resource Utilization"
return ba.score(pod, nodeInfo)
}
@ -67,42 +68,63 @@ func (ba *BalancedAllocation) ScoreExtensions() framework.ScoreExtensions {
}
// NewBalancedAllocation initializes a new plugin and returns it.
func NewBalancedAllocation(_ runtime.Object, h framework.Handle, fts feature.Features) (framework.Plugin, error) {
func NewBalancedAllocation(baArgs runtime.Object, h framework.Handle, fts feature.Features) (framework.Plugin, error) {
args, ok := baArgs.(*config.NodeResourcesBalancedAllocationArgs)
if !ok {
return nil, fmt.Errorf("want args to be of type NodeResourcesBalancedAllocationArgs, got %T", baArgs)
}
if err := validation.ValidateNodeResourcesBalancedAllocationArgs(nil, args); err != nil {
return nil, err
}
resToWeightMap := make(resourceToWeightMap)
for _, resource := range args.Resources {
resToWeightMap[v1.ResourceName(resource.Name)] = resource.Weight
}
return &BalancedAllocation{
handle: h,
resourceAllocationScorer: resourceAllocationScorer{
Name: BalancedAllocationName,
scorer: balancedResourceScorer,
resourceToWeightMap: defaultRequestedRatioResources,
resourceToWeightMap: resToWeightMap,
enablePodOverhead: fts.EnablePodOverhead,
},
}, nil
}
// todo: use resource weights in the scorer function
func balancedResourceScorer(requested, allocable resourceToValueMap) int64 {
cpuFraction := fractionOfCapacity(requested[v1.ResourceCPU], allocable[v1.ResourceCPU])
memoryFraction := fractionOfCapacity(requested[v1.ResourceMemory], allocable[v1.ResourceMemory])
// fractions might be greater than 1 because pods with no requests get minimum
// values.
if cpuFraction > 1 {
cpuFraction = 1
}
if memoryFraction > 1 {
memoryFraction = 1
var resourceToFractions []float64
var totalFraction float64
for name, value := range requested {
fraction := float64(value) / float64(allocable[name])
if fraction > 1 {
fraction = 1
}
totalFraction += fraction
resourceToFractions = append(resourceToFractions, fraction)
}
// Upper and lower boundary of difference between cpuFraction and memoryFraction are -1 and 1
// respectively. Multiplying the absolute value of the difference by `MaxNodeScore` scales the value to
// 0-MaxNodeScore with 0 representing well balanced allocation and `MaxNodeScore` poorly balanced. Subtracting it from
// `MaxNodeScore` leads to the score which also scales from 0 to `MaxNodeScore` while `MaxNodeScore` representing well balanced.
diff := math.Abs(cpuFraction - memoryFraction)
return int64((1 - diff) * float64(framework.MaxNodeScore))
}
std := 0.0
func fractionOfCapacity(requested, capacity int64) float64 {
if capacity == 0 {
return 1
// For most cases, resources are limited to cpu and memory, the std could be simplified to std := (fraction1-fraction2)/2
// len(fractions) > 2: calculate std based on the well-known formula - root square of Σ((fraction(i)-mean)^2)/len(fractions)
// Otherwise, set the std to zero is enough.
if len(resourceToFractions) == 2 {
std = math.Abs((resourceToFractions[0] - resourceToFractions[1]) / 2)
} else if len(resourceToFractions) > 2 {
mean := totalFraction / float64(len(resourceToFractions))
var sum float64
for _, fraction := range resourceToFractions {
sum = sum + (fraction-mean)*(fraction-mean)
}
std = math.Sqrt(sum / float64(len(resourceToFractions)))
}
return float64(requested) / float64(capacity)
// STD (standard deviation) is always a positive value. 1-deviation lets the score to be higher for node which has least deviation and
// multiplying it with `MaxNodeScore` provides the scaling factor needed.
return int64((1 - std) * float64(framework.MaxNodeScore))
}

173
pkg/scheduler/framework/plugins/noderesources/balanced_allocation_test.go
View File

@ -24,6 +24,7 @@ import (
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/pkg/scheduler/apis/config"
"k8s.io/kubernetes/pkg/scheduler/framework"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature"
"k8s.io/kubernetes/pkg/scheduler/framework/runtime"
@ -31,6 +32,28 @@ import (
)
func TestNodeResourcesBalancedAllocation(t *testing.T) {
cpuAndMemoryAndGPU := v1.PodSpec{
Containers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("1000m"),
v1.ResourceMemory: resource.MustParse("2000"),
},
},
},
{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("2000m"),
v1.ResourceMemory: resource.MustParse("3000"),
"nvidia.com/gpu": resource.MustParse("3"),
},
},
},
},
NodeName: "machine1",
}
labels1 := map[string]string{
"foo": "bar",
"baz": "blah",
@ -99,91 +122,113 @@ func TestNodeResourcesBalancedAllocation(t *testing.T) {
NodeName: "machine1",
Containers: []v1.Container{{}},
}
defaultResourceBalancedAllocationSet := []config.ResourceSpec{
{Name: string(v1.ResourceCPU), Weight: 1},
{Name: string(v1.ResourceMemory), Weight: 1},
}
scalarResource := map[string]int64{
"nvidia.com/gpu": 8,
}
tests := []struct {
pod *v1.Pod
pods []*v1.Pod
nodes []*v1.Node
expectedList framework.NodeScoreList
name string
args config.NodeResourcesBalancedAllocationArgs
}{
{
// Node1 scores (remaining resources) on 0-MaxNodeScore scale
// CPU Fraction: 0 / 4000 = 0%
// Memory Fraction: 0 / 10000 = 0%
// Node1 Score: MaxNodeScore - (0-0)*MaxNodeScore = MaxNodeScore
// Node1 Score: (1-0) * MaxNodeScore = MaxNodeScore
// Node2 scores (remaining resources) on 0-MaxNodeScore scale
// CPU Fraction: 0 / 4000 = 0 %
// Memory Fraction: 0 / 10000 = 0%
// Node2 Score: MaxNodeScore - (0-0)*MaxNodeScore = MaxNodeScore
// Node2 Score: (1-0) * MaxNodeScore = MaxNodeScore
pod: &v1.Pod{Spec: noResources},
nodes: []*v1.Node{makeNode("machine1", 4000, 10000), makeNode("machine2", 4000, 10000)},
expectedList: []framework.NodeScore{{Name: "machine1", Score: framework.MaxNodeScore}, {Name: "machine2", Score: framework.MaxNodeScore}},
name: "nothing scheduled, nothing requested",
args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
},
{
// Node1 scores on 0-MaxNodeScore scale
// CPU Fraction: 3000 / 4000= 75%
// Memory Fraction: 5000 / 10000 = 50%
// Node1 Score: MaxNodeScore - (0.75-0.5)*MaxNodeScore = 75
// Node1 std: (0.75 - 0.5) / 2 = 0.125
// Node1 Score: (1 - 0.125)*MaxNodeScore = 87
// Node2 scores on 0-MaxNodeScore scale
// CPU Fraction: 3000 / 6000= 50%
// Memory Fraction: 5000/10000 = 50%
// Node2 Score: MaxNodeScore - (0.5-0.5)*MaxNodeScore = MaxNodeScore
// Node2 std: 0
// Node2 Score: (1-0) * MaxNodeScore = MaxNodeScore
pod: &v1.Pod{Spec: cpuAndMemory},
nodes: []*v1.Node{makeNode("machine1", 4000, 10000), makeNode("machine2", 6000, 10000)},
expectedList: []framework.NodeScore{{Name: "machine1", Score: 75}, {Name: "machine2", Score: framework.MaxNodeScore}},
expectedList: []framework.NodeScore{{Name: "machine1", Score: 87}, {Name: "machine2", Score: framework.MaxNodeScore}},
name: "nothing scheduled, resources requested, differently sized machines",
args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
},
{
// Node1 scores on 0-MaxNodeScore scale
// CPU Fraction: 0 / 4000= 0%
// Memory Fraction: 0 / 10000 = 0%
// Node1 Score: MaxNodeScore - (0-0)*MaxNodeScore = MaxNodeScore
// Node1 std: 0
// Node1 Score: (1-0) * MaxNodeScore = MaxNodeScore
// Node2 scores on 0-MaxNodeScore scale
// CPU Fraction: 0 / 4000= 0%
// Memory Fraction: 0 / 10000 = 0%
// Node2 Score: MaxNodeScore - (0-0)*MaxNodeScore= MaxNodeScore
// Node2 std: 0
// Node2 Score: (1-0) * MaxNodeScore = MaxNodeScore
pod: &v1.Pod{Spec: noResources},
nodes: []*v1.Node{makeNode("machine1", 4000, 10000), makeNode("machine2", 4000, 10000)},
expectedList: []framework.NodeScore{{Name: "machine1", Score: framework.MaxNodeScore}, {Name: "machine2", Score: framework.MaxNodeScore}},
name: "no resources requested, pods scheduled",
name: "no resources requested, pods without container scheduled",
pods: []*v1.Pod{
{Spec: machine1Spec, ObjectMeta: metav1.ObjectMeta{Labels: labels2}},
{Spec: machine1Spec, ObjectMeta: metav1.ObjectMeta{Labels: labels1}},
{Spec: machine2Spec, ObjectMeta: metav1.ObjectMeta{Labels: labels1}},
{Spec: machine2Spec, ObjectMeta: metav1.ObjectMeta{Labels: labels1}},
},
args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
},
{
// Node1 scores on 0-MaxNodeScore scale
// CPU Fraction: 300 / 250 = 100%
// Memory Fraction: 600 / 10000 = 60%
// Node1 Score: MaxNodeScore - (100-60)*MaxNodeScore = 60
// Node1 std: (1 - 0.6) / 2 = 0.2
// Node1 Score: (1 - 0.2)*MaxNodeScore = 80
// Node2 scores on 0-MaxNodeScore scale
// CPU Fraction: 100 / 250 = 40%
// Memory Fraction: 200 / 10000 = 20%
// Node2 Score: MaxNodeScore - (40-20)*MaxNodeScore= 80
// Node2 std: (0.4 - 0.2) / 2 = 0.1
// Node2 Score: (1 - 0.1)*MaxNodeScore = 90
pod: &v1.Pod{Spec: nonZeroContainer},
nodes: []*v1.Node{makeNode("machine1", 250, 1000*1024*1024), makeNode("machine2", 250, 1000*1024*1024)},
expectedList: []framework.NodeScore{{Name: "machine1", Score: 60}, {Name: "machine2", Score: 80}},
name: "no resources requested, pods scheduled",
expectedList: []framework.NodeScore{{Name: "machine1", Score: 80}, {Name: "machine2", Score: 90}},
name: "no resources requested, pods with container scheduled",
pods: []*v1.Pod{
{Spec: nonZeroContainer1},
{Spec: nonZeroContainer1},
},
args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
},
{
// Node1 scores on 0-MaxNodeScore scale
// CPU Fraction: 6000 / 10000 = 60%
// Memory Fraction: 0 / 20000 = 0%
// Node1 Score: MaxNodeScore - (0.6-0)*MaxNodeScore = 40
// Node1 std: (0.6 - 0) / 2 = 0.3
// Node1 Score: (1 - 0.3)*MaxNodeScore = 70
// Node2 scores on 0-MaxNodeScore scale
// CPU Fraction: 6000 / 10000 = 60%
// Memory Fraction: 5000 / 20000 = 25%
// Node2 Score: MaxNodeScore - (0.6-0.25)*MaxNodeScore = 65
// Node2 std: (0.6 - 0.25) / 2 = 0.175
// Node2 Score: (1 - 0.175)*MaxNodeScore = 82
pod: &v1.Pod{Spec: noResources},
nodes: []*v1.Node{makeNode("machine1", 10000, 20000), makeNode("machine2", 10000, 20000)},
expectedList: []framework.NodeScore{{Name: "machine1", Score: 40}, {Name: "machine2", Score: 65}},
expectedList: []framework.NodeScore{{Name: "machine1", Score: 70}, {Name: "machine2", Score: 82}},
name: "no resources requested, pods scheduled with resources",
pods: []*v1.Pod{
{Spec: cpuOnly, ObjectMeta: metav1.ObjectMeta{Labels: labels2}},
@ -191,60 +236,139 @@ func TestNodeResourcesBalancedAllocation(t *testing.T) {
{Spec: cpuOnly2, ObjectMeta: metav1.ObjectMeta{Labels: labels1}},
{Spec: cpuAndMemory, ObjectMeta: metav1.ObjectMeta{Labels: labels1}},
},
args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
},
{
// Node1 scores on 0-MaxNodeScore scale
// CPU Fraction: 6000 / 10000 = 60%
// Memory Fraction: 5000 / 20000 = 25%
// Node1 Score: MaxNodeScore - (0.6-0.25)*MaxNodeScore = 65
// Node1 std: (0.6 - 0.25) / 2 = 0.175
// Node1 Score: (1 - 0.175)*MaxNodeScore = 82
// Node2 scores on 0-MaxNodeScore scale
// CPU Fraction: 6000 / 10000 = 60%
// Memory Fraction: 10000 / 20000 = 50%
// Node2 Score: MaxNodeScore - (0.6-0.5)*MaxNodeScore = 90
// Node2 std: (0.6 - 0.5) / 2 = 0.05
// Node2 Score: (1 - 0.05)*MaxNodeScore = 95
pod: &v1.Pod{Spec: cpuAndMemory},
nodes: []*v1.Node{makeNode("machine1", 10000, 20000), makeNode("machine2", 10000, 20000)},
expectedList: []framework.NodeScore{{Name: "machine1", Score: 65}, {Name: "machine2", Score: 90}},
expectedList: []framework.NodeScore{{Name: "machine1", Score: 82}, {Name: "machine2", Score: 95}},
name: "resources requested, pods scheduled with resources",
pods: []*v1.Pod{
{Spec: cpuOnly},
{Spec: cpuAndMemory},
},
args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
},
{
// Node1 scores on 0-MaxNodeScore scale
// CPU Fraction: 6000 / 10000 = 60%
// Memory Fraction: 5000 / 20000 = 25%
// Node1 Score: MaxNodeScore - (0.6-0.25)*MaxNodeScore = 65
// Node1 std: (0.6 - 0.25) / 2 = 0.175
// Node1 Score: (1 - 0.175)*MaxNodeScore = 82
// Node2 scores on 0-MaxNodeScore scale
// CPU Fraction: 6000 / 10000 = 60%
// Memory Fraction: 10000 / 50000 = 20%
// Node2 Score: MaxNodeScore - (0.6-0.2)*MaxNodeScore = 60
// Node2 std: (0.6 - 0.2) / 2 = 0.2
// Node2 Score: (1 - 0.2)*MaxNodeScore = 80
pod: &v1.Pod{Spec: cpuAndMemory},
nodes: []*v1.Node{makeNode("machine1", 10000, 20000), makeNode("machine2", 10000, 50000)},
expectedList: []framework.NodeScore{{Name: "machine1", Score: 65}, {Name: "machine2", Score: 60}},
expectedList: []framework.NodeScore{{Name: "machine1", Score: 82}, {Name: "machine2", Score: 80}},
name: "resources requested, pods scheduled with resources, differently sized machines",
pods: []*v1.Pod{
{Spec: cpuOnly},
{Spec: cpuAndMemory},
},
args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
},
{
// Node1 scores on 0-MaxNodeScore scale
// CPU Fraction: 6000 / 6000 = 1
// Memory Fraction: 0 / 10000 = 0
// Node1 std: (1 - 0) / 2 = 0.5
// Node1 Score: (1 - 0.5)*MaxNodeScore = 50
// Node1 Score: MaxNodeScore - (1 - 0) * MaxNodeScore = 0
// Node2 scores on 0-MaxNodeScore scale
// CPU Fraction: 6000 / 6000 = 1
// Memory Fraction 5000 / 10000 = 50%
// Node2 Score: MaxNodeScore - (1 - 0.5) * MaxNodeScore = 50
// Node2 std: (1 - 0.5) / 2 = 0.25
// Node2 Score: (1 - 0.25)*MaxNodeScore = 75
pod: &v1.Pod{Spec: cpuOnly},
nodes: []*v1.Node{makeNode("machine1", 6000, 10000), makeNode("machine2", 6000, 10000)},
expectedList: []framework.NodeScore{{Name: "machine1", Score: 0}, {Name: "machine2", Score: 50}},
expectedList: []framework.NodeScore{{Name: "machine1", Score: 50}, {Name: "machine2", Score: 75}},
name: "requested resources at node capacity",
pods: []*v1.Pod{
{Spec: cpuOnly},
{Spec: cpuAndMemory},
},
args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
},
{
pod: &v1.Pod{Spec: noResources},
nodes: []*v1.Node{makeNode("machine1", 0, 0), makeNode("machine2", 0, 0)},
expectedList: []framework.NodeScore{{Name: "machine1", Score: 100}, {Name: "machine2", Score: 100}},
name: "zero node resources, pods scheduled with resources",
pods: []*v1.Pod{
{Spec: cpuOnly},
{Spec: cpuAndMemory},
},
args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
},
// Node1 scores on 0-MaxNodeScore scale
// CPU Fraction: 3100 / 3500 = 88.57%
// Memory Fraction: 5000 / 40000 = 12.5%
// GPU Fraction: 4 / 8 = 0.5%
// Node1 std: sqrt(((0.8857 - 0.503) * (0.8857 - 0.503) + (0.503 - 0.125) * (0.503 - 0.125) + (0.503 - 0.5) * (0.503 - 0.5)) / 3) = 0.3105
// Node1 Score: (1 - 0.3105)*MaxNodeScore = 68
// Node2 scores on 0-MaxNodeScore scale
// CPU Fraction: 3100 / 3500 = 88.57%
// Memory Fraction: 5000 / 40000 = 12.5%
// GPU Fraction: 1 / 8 = 12.5%
// Node2 std: sqrt(((0.8875 - 0.378) * (0.8875 - 0.378) + (0.378 - 0.125) * (0.378 - 0.125)) + (0.378 - 0.125) * (0.378 - 0.125)) / 3) = 0.358
// Node2 Score: (1 - 0.358)*MaxNodeScore = 64
{
pod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("0"),
"nvidia.com/gpu": resource.MustParse("1"),
},
},
},
},
},
},
nodes: []*v1.Node{makeNodeWithExtendedResource("machine1", 3500, 40000, scalarResource), makeNodeWithExtendedResource("machine2", 3500, 40000, scalarResource)},
expectedList: []framework.NodeScore{{Name: "machine1", Score: 68}, {Name: "machine2", Score: 64}},
name: "include scalar resource on a node for balanced resource allocation",
pods: []*v1.Pod{
{Spec: cpuAndMemory},
{Spec: cpuAndMemoryAndGPU},
},
args: config.NodeResourcesBalancedAllocationArgs{Resources: []config.ResourceSpec{
{Name: string(v1.ResourceCPU), Weight: 1},
{Name: string(v1.ResourceMemory), Weight: 1},
{Name: "nvidia.com/gpu", Weight: 1},
}},
},
// Only one node (machine1) has the scalar resource, pod doesn't request the scalar resource and the scalar resource should be skipped for consideration.
// Node1: std = 0, score = 100
// Node2: std = 0, score = 100
{
pod: &v1.Pod{Spec: v1.PodSpec{Containers: []v1.Container{{}}}},
nodes: []*v1.Node{makeNodeWithExtendedResource("machine1", 3500, 40000, scalarResource), makeNode("machine2", 3500, 40000)},
expectedList: []framework.NodeScore{{Name: "machine1", Score: 100}, {Name: "machine2", Score: 100}},
name: "node without the scalar resource results to a higher score",
pods: []*v1.Pod{
{Spec: cpuOnly},
{Spec: cpuOnly2},
},
args: config.NodeResourcesBalancedAllocationArgs{Resources: []config.ResourceSpec{
{Name: string(v1.ResourceCPU), Weight: 1},
{Name: "nvidia.com/gpu", Weight: 1},
}},
},
}
@ -252,8 +376,7 @@ func TestNodeResourcesBalancedAllocation(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
snapshot := cache.NewSnapshot(test.pods, test.nodes)
fh, _ := runtime.NewFramework(nil, nil, runtime.WithSnapshotSharedLister(snapshot))
p, _ := NewBalancedAllocation(nil, fh, feature.Features{EnablePodOverhead: true})
p, _ := NewBalancedAllocation(&test.args, fh, feature.Features{EnablePodOverhead: true})
for i := range test.nodes {
hostResult, err := p.(framework.ScorePlugin).Score(context.Background(), nil, test.pod, test.nodes[i].Name)
if err != nil {

5
pkg/scheduler/framework/plugins/noderesources/resource_allocation.go
View File

@ -30,9 +30,6 @@ type resourceToWeightMap map[v1.ResourceName]int64
// scorer is decorator for resourceAllocationScorer
type scorer func(args *config.NodeResourcesFitArgs) *resourceAllocationScorer
// defaultRequestedRatioResources is used to set default requestToWeight map for CPU and memory
var defaultRequestedRatioResources = resourceToWeightMap{v1.ResourceMemory: 1, v1.ResourceCPU: 1}
// resourceAllocationScorer contains information to calculate resource allocation score.
type resourceAllocationScorer struct {
Name string
@ -42,7 +39,7 @@ type resourceAllocationScorer struct {
enablePodOverhead bool
}
// resourceToValueMap contains resource name and score.
// resourceToValueMap is keyed with resource name and valued with quantity.
type resourceToValueMap map[v1.ResourceName]int64
// score will use `scorer` function to calculate the score.

1
staging/src/k8s.io/kube-scheduler/config/v1beta1/register.go
View File

@ -45,6 +45,7 @@ func addKnownTypes(scheme *runtime.Scheme) error {
&PodTopologySpreadArgs{},
&RequestedToCapacityRatioArgs{},
&ServiceAffinityArgs{},
&NodeResourcesBalancedAllocationArgs{},
&NodeResourcesLeastAllocatedArgs{},
&NodeResourcesMostAllocatedArgs{},
&VolumeBindingArgs{},

16
staging/src/k8s.io/kube-scheduler/config/v1beta1/types_pluginargs.go
View File

@ -180,6 +180,18 @@ type NodeResourcesMostAllocatedArgs struct {
Resources []ResourceSpec `json:"resources,omitempty"`
}
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// NodeResourcesBalancedAllocationArgs holds arguments used to configure NodeResourcesBalancedAllocation plugin.
type NodeResourcesBalancedAllocationArgs struct {
metav1.TypeMeta `json:",inline"`
// Resources to be managed, the default is "cpu" and "memory" if not specified.
// +listType=map
// +listMapKey=name
Resources []ResourceSpec `json:"resources,omitempty"`
}
// UtilizationShapePoint represents single point of priority function shape.
type UtilizationShapePoint struct {
// Utilization (x axis). Valid values are 0 to 100. Fully utilized node maps to 100.
@ -188,9 +200,9 @@ type UtilizationShapePoint struct {
Score int32 `json:"score"`
}
// ResourceSpec represents single resource and weight for bin packing of priority RequestedToCapacityRatioArguments.
// ResourceSpec represents a single resource.
type ResourceSpec struct {
// Name of the resource to be managed by RequestedToCapacityRatio function.
// Name of the resource.
Name string `json:"name"`
// Weight of the resource.
Weight int64 `json:"weight,omitempty"`

30
staging/src/k8s.io/kube-scheduler/config/v1beta1/zz_generated.deepcopy.go generated
View File

@ -298,6 +298,36 @@ func (in *NodeLabelArgs) DeepCopyObject() runtime.Object {
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NodeResourcesBalancedAllocationArgs) DeepCopyInto(out *NodeResourcesBalancedAllocationArgs) {
*out = *in
out.TypeMeta = in.TypeMeta
if in.Resources != nil {
in, out := &in.Resources, &out.Resources
*out = make([]ResourceSpec, len(*in))
copy(*out, *in)
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeResourcesBalancedAllocationArgs.
func (in *NodeResourcesBalancedAllocationArgs) DeepCopy() *NodeResourcesBalancedAllocationArgs {
if in == nil {
return nil
}
out := new(NodeResourcesBalancedAllocationArgs)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *NodeResourcesBalancedAllocationArgs) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NodeResourcesFitArgs) DeepCopyInto(out *NodeResourcesFitArgs) {
*out = *in

1
staging/src/k8s.io/kube-scheduler/config/v1beta2/register.go
View File

@ -40,6 +40,7 @@ func addKnownTypes(scheme *runtime.Scheme) error {
&KubeSchedulerConfiguration{},
&DefaultPreemptionArgs{},
&InterPodAffinityArgs{},
&NodeResourcesBalancedAllocationArgs{},
&NodeResourcesFitArgs{},
&PodTopologySpreadArgs{},
&VolumeBindingArgs{},

16
staging/src/k8s.io/kube-scheduler/config/v1beta2/types_pluginargs.go
View File

@ -116,6 +116,18 @@ type PodTopologySpreadArgs struct {
DefaultingType PodTopologySpreadConstraintsDefaulting `json:"defaultingType,omitempty"`
}
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// NodeResourcesBalancedAllocationArgs holds arguments used to configure NodeResourcesBalancedAllocation plugin.
type NodeResourcesBalancedAllocationArgs struct {
metav1.TypeMeta `json:",inline"`
// Resources to be managed, the default is "cpu" and "memory" if not specified.
// +listType=map
// +listMapKey=name
Resources []ResourceSpec `json:"resources,omitempty"`
}
// UtilizationShapePoint represents single point of priority function shape.
type UtilizationShapePoint struct {
// Utilization (x axis). Valid values are 0 to 100. Fully utilized node maps to 100.
@ -124,9 +136,9 @@ type UtilizationShapePoint struct {
Score int32 `json:"score"`
}
// ResourceSpec represents single resource and weight for bin packing of priority RequestedToCapacityRatioArguments.
// ResourceSpec represents a single resource.
type ResourceSpec struct {
// Name of the resource to be managed by RequestedToCapacityRatio function.
// Name of the resource.
Name string `json:"name"`
// Weight of the resource.
Weight int64 `json:"weight,omitempty"`

30
staging/src/k8s.io/kube-scheduler/config/v1beta2/zz_generated.deepcopy.go generated
View File

@ -253,6 +253,36 @@ func (in *NodeAffinityArgs) DeepCopyObject() runtime.Object {
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NodeResourcesBalancedAllocationArgs) DeepCopyInto(out *NodeResourcesBalancedAllocationArgs) {
*out = *in
out.TypeMeta = in.TypeMeta
if in.Resources != nil {
in, out := &in.Resources, &out.Resources
*out = make([]ResourceSpec, len(*in))
copy(*out, *in)
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeResourcesBalancedAllocationArgs.
func (in *NodeResourcesBalancedAllocationArgs) DeepCopy() *NodeResourcesBalancedAllocationArgs {
if in == nil {
return nil
}
out := new(NodeResourcesBalancedAllocationArgs)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *NodeResourcesBalancedAllocationArgs) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NodeResourcesFitArgs) DeepCopyInto(out *NodeResourcesFitArgs) {
*out = *in