github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2026-01-05 23:47:50 +00:00
Code Issues Projects Releases Wiki Activity

Configurable weight on the CPU and memory

Browse Source 
This change also make it possible to score the resources beyond the "cpu"
and "memory" which is currently listed in "defaultRequestedRatioResources".

Signed-off-by: Dave Chen <dave.chen@arm.com>
This commit is contained in:
Dave Chen
2020-04-28 15:47:03 +08:00
parent c096a37226
commit 621c73b984
15 changed files with 581 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
pkg/scheduler/framework/plugins/noderesources/least_allocated.go
View File

@@ -22,6 +22,7 @@ import (
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/kubernetes/pkg/scheduler/apis/config"
framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
)
@@ -63,25 +64,43 @@ func (la *LeastAllocated) ScoreExtensions() framework.ScoreExtensions {
}
// NewLeastAllocated initializes a new plugin and returns it.
func NewLeastAllocated(_ runtime.Object, h framework.FrameworkHandle) (framework.Plugin, error) {
func NewLeastAllocated(laArgs runtime.Object, h framework.FrameworkHandle) (framework.Plugin, error) {
args, ok := laArgs.(*config.NodeResourcesLeastAllocatedArgs)
if !ok {
return nil, fmt.Errorf("want args to be of type NodeResourcesLeastAllocatedArgs, got %T", laArgs)
}
resToWeightMap := make(resourceToWeightMap)
for _, resource := range (*args).Resources {
if resource.Weight <= 0 {
return nil, fmt.Errorf("resource Weight of %v should be a positive value, got %v", resource.Name, resource.Weight)
}
if resource.Weight > framework.MaxNodeScore {
return nil, fmt.Errorf("resource Weight of %v should be less than 100, got %v", resource.Name, resource.Weight)
}
resToWeightMap[v1.ResourceName(resource.Name)] = resource.Weight
}
return &LeastAllocated{
handle: h,
resourceAllocationScorer: resourceAllocationScorer{
LeastAllocatedName,
leastResourceScorer,
defaultRequestedRatioResources,
Name: LeastAllocatedName,
scorer: leastResourceScorer(resToWeightMap),
resourceToWeightMap: resToWeightMap,
},
}, nil
}
func leastResourceScorer(requested, allocable resourceToValueMap, includeVolumes bool, requestedVolumes int, allocatableVolumes int) int64 {
var nodeScore, weightSum int64
for resource, weight := range defaultRequestedRatioResources {
resourceScore := leastRequestedScore(requested[resource], allocable[resource])
nodeScore += resourceScore * weight
weightSum += weight
func leastResourceScorer(resToWeightMap resourceToWeightMap) func(resourceToValueMap, resourceToValueMap, bool, int, int) int64 {
return func(requested, allocable resourceToValueMap, includeVolumes bool, requestedVolumes int, allocatableVolumes int) int64 {
var nodeScore, weightSum int64
for resource, weight := range resToWeightMap {
resourceScore := leastRequestedScore(requested[resource], allocable[resource])
nodeScore += resourceScore * weight
weightSum += weight
}
return nodeScore / weightSum
}
return nodeScore / weightSum
}
// The unused capacity is calculated on a scale of 0-MaxNodeScore

68
pkg/scheduler/framework/plugins/noderesources/least_allocated_test.go
View File

@@ -24,6 +24,7 @@ import (
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/pkg/scheduler/apis/config"
framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
"k8s.io/kubernetes/pkg/scheduler/internal/cache"
)
@@ -90,10 +91,16 @@ func TestNodeResourcesLeastAllocated(t *testing.T) {
},
},
}
defaultResourceLeastAllocatedSet := []config.ResourceSpec{
{Name: string(v1.ResourceCPU), Weight: 1},
{Name: string(v1.ResourceMemory), Weight: 1},
}
tests := []struct {
pod *v1.Pod
pods []*v1.Pod
nodes []*v1.Node
args config.NodeResourcesLeastAllocatedArgs
wantErr string
expectedList framework.NodeScoreList
name string
}{
@@ -108,6 +115,7 @@ func TestNodeResourcesLeastAllocated(t *testing.T) {
// Node2 Score: (100 + 100) / 2 = 100
pod: &v1.Pod{Spec: noResources},
nodes: []*v1.Node{makeNode("machine1", 4000, 10000), makeNode("machine2", 4000, 10000)},
args: config.NodeResourcesLeastAllocatedArgs{Resources: defaultResourceLeastAllocatedSet},
expectedList: []framework.NodeScore{{Name: "machine1", Score: framework.MaxNodeScore}, {Name: "machine2", Score: framework.MaxNodeScore}},
name: "nothing scheduled, nothing requested",
},
@@ -122,6 +130,7 @@ func TestNodeResourcesLeastAllocated(t *testing.T) {
// Node2 Score: (50 + 50) / 2 = 50
pod: &v1.Pod{Spec: cpuAndMemory},
nodes: []*v1.Node{makeNode("machine1", 4000, 10000), makeNode("machine2", 6000, 10000)},
args: config.NodeResourcesLeastAllocatedArgs{Resources: defaultResourceLeastAllocatedSet},
expectedList: []framework.NodeScore{{Name: "machine1", Score: 37}, {Name: "machine2", Score: 50}},
name: "nothing scheduled, resources requested, differently sized machines",
},
@@ -136,6 +145,7 @@ func TestNodeResourcesLeastAllocated(t *testing.T) {
// Node2 Score: (100 + 100) / 2 = 100
pod: &v1.Pod{Spec: noResources},
nodes: []*v1.Node{makeNode("machine1", 4000, 10000), makeNode("machine2", 4000, 10000)},
args: config.NodeResourcesLeastAllocatedArgs{Resources: defaultResourceLeastAllocatedSet},
expectedList: []framework.NodeScore{{Name: "machine1", Score: framework.MaxNodeScore}, {Name: "machine2", Score: framework.MaxNodeScore}},
name: "no resources requested, pods scheduled",
pods: []*v1.Pod{
@@ -156,6 +166,7 @@ func TestNodeResourcesLeastAllocated(t *testing.T) {
// Node2 Score: (40 + 75) / 2 = 57
pod: &v1.Pod{Spec: noResources},
nodes: []*v1.Node{makeNode("machine1", 10000, 20000), makeNode("machine2", 10000, 20000)},
args: config.NodeResourcesLeastAllocatedArgs{Resources: defaultResourceLeastAllocatedSet},
expectedList: []framework.NodeScore{{Name: "machine1", Score: 70}, {Name: "machine2", Score: 57}},
name: "no resources requested, pods scheduled with resources",
pods: []*v1.Pod{
@@ -176,6 +187,7 @@ func TestNodeResourcesLeastAllocated(t *testing.T) {
// Node2 Score: (40 + 50) / 2 = 45
pod: &v1.Pod{Spec: cpuAndMemory},
nodes: []*v1.Node{makeNode("machine1", 10000, 20000), makeNode("machine2", 10000, 20000)},
args: config.NodeResourcesLeastAllocatedArgs{Resources: defaultResourceLeastAllocatedSet},
expectedList: []framework.NodeScore{{Name: "machine1", Score: 57}, {Name: "machine2", Score: 45}},
name: "resources requested, pods scheduled with resources",
pods: []*v1.Pod{
@@ -194,6 +206,7 @@ func TestNodeResourcesLeastAllocated(t *testing.T) {
// Node2 Score: (40 + 80) / 2 = 60
pod: &v1.Pod{Spec: cpuAndMemory},
nodes: []*v1.Node{makeNode("machine1", 10000, 20000), makeNode("machine2", 10000, 50000)},
args: config.NodeResourcesLeastAllocatedArgs{Resources: defaultResourceLeastAllocatedSet},
expectedList: []framework.NodeScore{{Name: "machine1", Score: 57}, {Name: "machine2", Score: 60}},
name: "resources requested, pods scheduled with resources, differently sized machines",
pods: []*v1.Pod{
@@ -212,6 +225,7 @@ func TestNodeResourcesLeastAllocated(t *testing.T) {
// Node2 Score: (0 + 50) / 2 = 25
pod: &v1.Pod{Spec: cpuOnly},
nodes: []*v1.Node{makeNode("machine1", 4000, 10000), makeNode("machine2", 4000, 10000)},
args: config.NodeResourcesLeastAllocatedArgs{Resources: defaultResourceLeastAllocatedSet},
expectedList: []framework.NodeScore{{Name: "machine1", Score: 50}, {Name: "machine2", Score: 25}},
name: "requested resources exceed node capacity",
pods: []*v1.Pod{
@@ -222,6 +236,7 @@ func TestNodeResourcesLeastAllocated(t *testing.T) {
{
pod: &v1.Pod{Spec: noResources},
nodes: []*v1.Node{makeNode("machine1", 0, 0), makeNode("machine2", 0, 0)},
args: config.NodeResourcesLeastAllocatedArgs{Resources: defaultResourceLeastAllocatedSet},
expectedList: []framework.NodeScore{{Name: "machine1", Score: 0}, {Name: "machine2", Score: 0}},
name: "zero node resources, pods scheduled with resources",
pods: []*v1.Pod{
@@ -229,13 +244,64 @@ func TestNodeResourcesLeastAllocated(t *testing.T) {
{Spec: cpuAndMemory},
},
},
{
// CPU Score: ((4000 - 3000) *100) / 4000 = 25
// Memory Score: ((10000 - 5000) *100) / 10000 = 50
// Node1 Score: (25 * 1 + 50 * 2) / (1 + 2) = 41
// CPU Score: ((6000 - 3000) *100) / 6000 = 50
// Memory Score: ((10000 - 5000) *100) / 10000 = 50
// Node2 Score: (50 * 1 + 50 * 2) / (1 + 2) = 50
pod: &v1.Pod{Spec: cpuAndMemory},
nodes: []*v1.Node{makeNode("machine1", 4000, 10000), makeNode("machine2", 6000, 10000)},
args: config.NodeResourcesLeastAllocatedArgs{Resources: []config.ResourceSpec{{Name: "memory", Weight: 2}, {Name: "cpu", Weight: 1}}},
expectedList: []framework.NodeScore{{Name: "machine1", Score: 41}, {Name: "machine2", Score: 50}},
name: "nothing scheduled, resources requested with different weight on CPU and memory, differently sized machines",
},
{
// resource with negtive weight is not allowed
pod: &v1.Pod{Spec: cpuAndMemory},
nodes: []*v1.Node{makeNode("machine", 4000, 10000)},
args: config.NodeResourcesLeastAllocatedArgs{Resources: []config.ResourceSpec{{Name: "memory", Weight: -1}, {Name: "cpu", Weight: 1}}},
wantErr: "resource Weight of memory should be a positive value, got -1",
name: "resource with negtive weight",
},
{
// resource with zero weight is not allowed
pod: &v1.Pod{Spec: cpuAndMemory},
nodes: []*v1.Node{makeNode("machine", 4000, 10000)},
args: config.NodeResourcesLeastAllocatedArgs{Resources: []config.ResourceSpec{{Name: "memory", Weight: 1}, {Name: "cpu", Weight: 0}}},
wantErr: "resource Weight of cpu should be a positive value, got 0",
name: "resource with zero weight",
},
{
// resource weight should be less than MaxNodeScore
pod: &v1.Pod{Spec: cpuAndMemory},
nodes: []*v1.Node{makeNode("machine", 4000, 10000)},
args: config.NodeResourcesLeastAllocatedArgs{Resources: []config.ResourceSpec{{Name: "memory", Weight: 120}}},
wantErr: "resource Weight of memory should be less than 100, got 120",
name: "resource weight larger than MaxNodeScore",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
snapshot := cache.NewSnapshot(test.pods, test.nodes)
fh, _ := framework.NewFramework(nil, nil, nil, framework.WithSnapshotSharedLister(snapshot))
p, _ := NewLeastAllocated(nil, fh)
p, err := NewLeastAllocated(&test.args, fh)
if len(test.wantErr) != 0 {
if err != nil && test.wantErr != err.Error() {
t.Fatalf("got err %w, want %w", err.Error(), test.wantErr)
} else if err == nil {
t.Fatal("no error produced, wanted %w", test.wantErr)
}
return
}
if err != nil && len(test.wantErr) == 0 {
t.Fatalf("failed to initialize plugin NodeResourcesLeastAllocated, got error: %v", err)
}
for i := range test.nodes {
hostResult, err := p.(framework.ScorePlugin).Score(context.Background(), nil, test.pod, test.nodes[i].Name)
if err != nil {

43
pkg/scheduler/framework/plugins/noderesources/most_allocated.go
View File

@@ -22,6 +22,7 @@ import (
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/kubernetes/pkg/scheduler/apis/config"
framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
)
@@ -61,26 +62,44 @@ func (ma *MostAllocated) ScoreExtensions() framework.ScoreExtensions {
}
// NewMostAllocated initializes a new plugin and returns it.
func NewMostAllocated(_ runtime.Object, h framework.FrameworkHandle) (framework.Plugin, error) {
func NewMostAllocated(maArgs runtime.Object, h framework.FrameworkHandle) (framework.Plugin, error) {
args, ok := maArgs.(*config.NodeResourcesMostAllocatedArgs)
if !ok {
return nil, fmt.Errorf("want args to be of type NodeResourcesMostAllocatedArgs, got %T", args)
}
resToWeightMap := make(resourceToWeightMap)
for _, resource := range (*args).Resources {
if resource.Weight <= 0 {
return nil, fmt.Errorf("resource Weight of %v should be a positive value, got %v", resource.Name, resource.Weight)
}
if resource.Weight > framework.MaxNodeScore {
return nil, fmt.Errorf("resource Weight of %v should be less than 100, got %v", resource.Name, resource.Weight)
}
resToWeightMap[v1.ResourceName(resource.Name)] = resource.Weight
}
return &MostAllocated{
handle: h,
resourceAllocationScorer: resourceAllocationScorer{
MostAllocatedName,
mostResourceScorer,
defaultRequestedRatioResources,
Name: MostAllocatedName,
scorer: mostResourceScorer(resToWeightMap),
resourceToWeightMap: resToWeightMap,
},
}, nil
}
func mostResourceScorer(requested, allocable resourceToValueMap, includeVolumes bool, requestedVolumes int, allocatableVolumes int) int64 {
var nodeScore, weightSum int64
for resource, weight := range defaultRequestedRatioResources {
resourceScore := mostRequestedScore(requested[resource], allocable[resource])
nodeScore += resourceScore * weight
weightSum += weight
func mostResourceScorer(resToWeightMap resourceToWeightMap) func(requested, allocable resourceToValueMap, includeVolumes bool, requestedVolumes int, allocatableVolumes int) int64 {
return func(requested, allocable resourceToValueMap, includeVolumes bool, requestedVolumes int, allocatableVolumes int) int64 {
var nodeScore, weightSum int64
for resource, weight := range resToWeightMap {
resourceScore := mostRequestedScore(requested[resource], allocable[resource])
nodeScore += resourceScore * weight
weightSum += weight
}
return (nodeScore / weightSum)
}
return (nodeScore / weightSum)
}
// The used capacity is calculated on a scale of 0-10

65
pkg/scheduler/framework/plugins/noderesources/most_allocated_test.go
View File

@@ -24,6 +24,7 @@ import (
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/pkg/scheduler/apis/config"
framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
"k8s.io/kubernetes/pkg/scheduler/internal/cache"
)
@@ -105,10 +106,16 @@ func TestNodeResourcesMostAllocated(t *testing.T) {
},
},
}
defaultResourceMostAllocatedSet := []config.ResourceSpec{
{Name: string(v1.ResourceCPU), Weight: 1},
{Name: string(v1.ResourceMemory), Weight: 1},
}
tests := []struct {
pod *v1.Pod
pods []*v1.Pod
nodes []*v1.Node
args config.NodeResourcesMostAllocatedArgs
wantErr string
expectedList framework.NodeScoreList
name string
}{
@@ -123,6 +130,7 @@ func TestNodeResourcesMostAllocated(t *testing.T) {
// Node2 Score: (0 + 0) / 2 = 0
pod: &v1.Pod{Spec: noResources},
nodes: []*v1.Node{makeNode("machine1", 4000, 10000), makeNode("machine2", 4000, 10000)},
args: config.NodeResourcesMostAllocatedArgs{Resources: defaultResourceMostAllocatedSet},
expectedList: []framework.NodeScore{{Name: "machine1", Score: 0}, {Name: "machine2", Score: 0}},
name: "nothing scheduled, nothing requested",
},
@@ -137,6 +145,7 @@ func TestNodeResourcesMostAllocated(t *testing.T) {
// Node2 Score: (50 + 50) / 2 = 50
pod: &v1.Pod{Spec: cpuAndMemory},
nodes: []*v1.Node{makeNode("machine1", 4000, 10000), makeNode("machine2", 6000, 10000)},
args: config.NodeResourcesMostAllocatedArgs{Resources: defaultResourceMostAllocatedSet},
expectedList: []framework.NodeScore{{Name: "machine1", Score: 62}, {Name: "machine2", Score: 50}},
name: "nothing scheduled, resources requested, differently sized machines",
},
@@ -151,6 +160,7 @@ func TestNodeResourcesMostAllocated(t *testing.T) {
// Node2 Score: (60 + 25) / 2 = 42
pod: &v1.Pod{Spec: noResources},
nodes: []*v1.Node{makeNode("machine1", 10000, 20000), makeNode("machine2", 10000, 20000)},
args: config.NodeResourcesMostAllocatedArgs{Resources: defaultResourceMostAllocatedSet},
expectedList: []framework.NodeScore{{Name: "machine1", Score: 30}, {Name: "machine2", Score: 42}},
name: "no resources requested, pods scheduled with resources",
pods: []*v1.Pod{
@@ -171,6 +181,7 @@ func TestNodeResourcesMostAllocated(t *testing.T) {
// Node2 Score: (60 + 50) / 2 = 55
pod: &v1.Pod{Spec: cpuAndMemory},
nodes: []*v1.Node{makeNode("machine1", 10000, 20000), makeNode("machine2", 10000, 20000)},
args: config.NodeResourcesMostAllocatedArgs{Resources: defaultResourceMostAllocatedSet},
expectedList: []framework.NodeScore{{Name: "machine1", Score: 42}, {Name: "machine2", Score: 55}},
name: "resources requested, pods scheduled with resources",
pods: []*v1.Pod{
@@ -189,16 +200,68 @@ func TestNodeResourcesMostAllocated(t *testing.T) {
// Node2 Score: (50 + 0) / 2 = 25
pod: &v1.Pod{Spec: bigCPUAndMemory},
nodes: []*v1.Node{makeNode("machine1", 4000, 10000), makeNode("machine2", 10000, 8000)},
args: config.NodeResourcesMostAllocatedArgs{Resources: defaultResourceMostAllocatedSet},
expectedList: []framework.NodeScore{{Name: "machine1", Score: 45}, {Name: "machine2", Score: 25}},
name: "resources requested with more than the node, pods scheduled with resources",
},
{
// CPU Score: (3000 *100) / 4000 = 75
// Memory Score: (5000 *100) / 10000 = 50
// Node1 Score: (75 * 1 + 50 * 2) / (1 + 2) = 58
// CPU Score: (3000 *100) / 6000 = 50
// Memory Score: (5000 *100) / 10000 = 50
// Node2 Score: (50 * 1 + 50 * 2) / (1 + 2) = 50
pod: &v1.Pod{Spec: cpuAndMemory},
nodes: []*v1.Node{makeNode("machine1", 4000, 10000), makeNode("machine2", 6000, 10000)},
args: config.NodeResourcesMostAllocatedArgs{Resources: []config.ResourceSpec{{Name: "memory", Weight: 2}, {Name: "cpu", Weight: 1}}},
expectedList: []framework.NodeScore{{Name: "machine1", Score: 58}, {Name: "machine2", Score: 50}},
name: "nothing scheduled, resources requested, differently sized machines",
},
{
// resource with negtive weight is not allowed
pod: &v1.Pod{Spec: cpuAndMemory},
nodes: []*v1.Node{makeNode("machine", 4000, 10000)},
args: config.NodeResourcesMostAllocatedArgs{Resources: []config.ResourceSpec{{Name: "memory", Weight: -1}, {Name: "cpu", Weight: 1}}},
wantErr: "resource Weight of memory should be a positive value, got -1",
name: "resource with negtive weight",
},
{
// resource with zero weight is not allowed
pod: &v1.Pod{Spec: cpuAndMemory},
nodes: []*v1.Node{makeNode("machine", 4000, 10000)},
args: config.NodeResourcesMostAllocatedArgs{Resources: []config.ResourceSpec{{Name: "memory", Weight: 1}, {Name: "cpu", Weight: 0}}},
wantErr: "resource Weight of cpu should be a positive value, got 0",
name: "resource with zero weight",
},
{
// resource weight should be less than MaxNodeScore
pod: &v1.Pod{Spec: cpuAndMemory},
nodes: []*v1.Node{makeNode("machine", 4000, 10000)},
args: config.NodeResourcesMostAllocatedArgs{Resources: []config.ResourceSpec{{Name: "memory", Weight: 120}}},
wantErr: "resource Weight of memory should be less than 100, got 120",
name: "resource weight larger than MaxNodeScore",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
snapshot := cache.NewSnapshot(test.pods, test.nodes)
fh, _ := framework.NewFramework(nil, nil, nil, framework.WithSnapshotSharedLister(snapshot))
p, _ := NewMostAllocated(nil, fh)
p, err := NewMostAllocated(&test.args, fh)
if len(test.wantErr) != 0 {
if err != nil && test.wantErr != err.Error() {
t.Fatalf("got err %w, want %w", err.Error(), test.wantErr)
} else if err == nil {
t.Fatal("no error produced, wanted %w", test.wantErr)
}
return
}
if err != nil && len(test.wantErr) == 0 {
t.Fatalf("failed to initialize plugin NodeResourcesMostAllocated, got error: %v", err)
}
for i := range test.nodes {
hostResult, err := p.(framework.ScorePlugin).Score(context.Background(), nil, test.pod, test.nodes[i].Name)
if err != nil {