mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-23 19:56:01 +00:00
Merge pull request #88842 from angao/fit-arg
add args for NodeResourcesFit plugin
This commit is contained in:
commit
70e09f2c24
@ -19,7 +19,7 @@ package lifecycle
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/klog/v2"
|
||||
v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
|
||||
"k8s.io/kubernetes/pkg/kubelet/util/format"
|
||||
@ -227,7 +227,7 @@ func GeneralPredicates(pod *v1.Pod, nodeInfo *schedulerframework.NodeInfo) ([]Pr
|
||||
}
|
||||
|
||||
var reasons []PredicateFailureReason
|
||||
for _, r := range noderesources.Fits(pod, nodeInfo, nil) {
|
||||
for _, r := range noderesources.Fits(pod, nodeInfo) {
|
||||
reasons = append(reasons, &InsufficientResourceError{
|
||||
ResourceName: r.ResourceName,
|
||||
Requested: r.Requested,
|
||||
|
@ -57,6 +57,11 @@ type NodeResourcesFitArgs struct {
|
||||
// IgnoredResources is the list of resources that NodeResources fit filter
|
||||
// should ignore.
|
||||
IgnoredResources []string
|
||||
// IgnoredResourceGroups defines the list of resource groups that NodeResources fit filter should ignore.
|
||||
// e.g. if group is ["example.com"], it will ignore all resource names that begin
|
||||
// with "example.com", such as "example.com/aaa" and "example.com/bbb".
|
||||
// A resource group name can't contain '/'.
|
||||
IgnoredResourceGroups []string
|
||||
}
|
||||
|
||||
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
|
||||
|
@ -462,6 +462,7 @@ func Convert_config_NodeLabelArgs_To_v1beta1_NodeLabelArgs(in *config.NodeLabelA
|
||||
|
||||
func autoConvert_v1beta1_NodeResourcesFitArgs_To_config_NodeResourcesFitArgs(in *v1beta1.NodeResourcesFitArgs, out *config.NodeResourcesFitArgs, s conversion.Scope) error {
|
||||
out.IgnoredResources = *(*[]string)(unsafe.Pointer(&in.IgnoredResources))
|
||||
out.IgnoredResourceGroups = *(*[]string)(unsafe.Pointer(&in.IgnoredResourceGroups))
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -472,6 +473,7 @@ func Convert_v1beta1_NodeResourcesFitArgs_To_config_NodeResourcesFitArgs(in *v1b
|
||||
|
||||
func autoConvert_config_NodeResourcesFitArgs_To_v1beta1_NodeResourcesFitArgs(in *config.NodeResourcesFitArgs, out *v1beta1.NodeResourcesFitArgs, s conversion.Scope) error {
|
||||
out.IgnoredResources = *(*[]string)(unsafe.Pointer(&in.IgnoredResources))
|
||||
out.IgnoredResourceGroups = *(*[]string)(unsafe.Pointer(&in.IgnoredResourceGroups))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
@ -286,6 +286,11 @@ func (in *NodeResourcesFitArgs) DeepCopyInto(out *NodeResourcesFitArgs) {
|
||||
*out = make([]string, len(*in))
|
||||
copy(*out, *in)
|
||||
}
|
||||
if in.IgnoredResourceGroups != nil {
|
||||
in, out := &in.IgnoredResourceGroups, &out.IgnoredResourceGroups
|
||||
*out = make([]string, len(*in))
|
||||
copy(*out, *in)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
|
@ -23,8 +23,10 @@ go_library(
|
||||
"//staging/src/k8s.io/api/core/v1:go_default_library",
|
||||
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
|
||||
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/validation:go_default_library",
|
||||
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
|
||||
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
|
||||
"//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
|
||||
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
|
||||
"//vendor/k8s.io/klog/v2:go_default_library",
|
||||
],
|
||||
|
@ -19,10 +19,13 @@ package noderesources
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1validation "k8s.io/apimachinery/pkg/apis/meta/v1/validation"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
"k8s.io/apimachinery/pkg/util/validation/field"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
|
||||
"k8s.io/kubernetes/pkg/features"
|
||||
@ -44,7 +47,8 @@ const (
|
||||
|
||||
// Fit is a plugin that checks if a node has sufficient resources.
|
||||
type Fit struct {
|
||||
ignoredResources sets.String
|
||||
ignoredResources sets.String
|
||||
ignoredResourceGroups sets.String
|
||||
}
|
||||
|
||||
// preFilterState computed at PreFilter and used at Filter.
|
||||
@ -62,16 +66,48 @@ func (f *Fit) Name() string {
|
||||
return FitName
|
||||
}
|
||||
|
||||
func validateFitArgs(args config.NodeResourcesFitArgs) error {
|
||||
var allErrs field.ErrorList
|
||||
resPath := field.NewPath("ignoredResources")
|
||||
for i, res := range args.IgnoredResources {
|
||||
path := resPath.Index(i)
|
||||
if errs := metav1validation.ValidateLabelName(res, path); len(errs) != 0 {
|
||||
allErrs = append(allErrs, errs...)
|
||||
}
|
||||
}
|
||||
|
||||
groupPath := field.NewPath("ignoredResourceGroups")
|
||||
for i, group := range args.IgnoredResourceGroups {
|
||||
path := groupPath.Index(i)
|
||||
if strings.Contains(group, "/") {
|
||||
allErrs = append(allErrs, field.Invalid(path, group, "resource group name can't contain '/'"))
|
||||
}
|
||||
if errs := metav1validation.ValidateLabelName(group, path); len(errs) != 0 {
|
||||
allErrs = append(allErrs, errs...)
|
||||
}
|
||||
}
|
||||
|
||||
if len(allErrs) == 0 {
|
||||
return nil
|
||||
}
|
||||
return allErrs.ToAggregate()
|
||||
}
|
||||
|
||||
// NewFit initializes a new plugin and returns it.
|
||||
func NewFit(plArgs runtime.Object, _ framework.FrameworkHandle) (framework.Plugin, error) {
|
||||
args, err := getFitArgs(plArgs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fit := &Fit{
|
||||
ignoredResources: sets.NewString(args.IgnoredResources...),
|
||||
|
||||
if err := validateFitArgs(args); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fit, nil
|
||||
|
||||
return &Fit{
|
||||
ignoredResources: sets.NewString(args.IgnoredResources...),
|
||||
ignoredResourceGroups: sets.NewString(args.IgnoredResourceGroups...),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func getFitArgs(obj runtime.Object) (config.NodeResourcesFitArgs, error) {
|
||||
@ -162,7 +198,7 @@ func (f *Fit) Filter(ctx context.Context, cycleState *framework.CycleState, pod
|
||||
return framework.NewStatus(framework.Error, err.Error())
|
||||
}
|
||||
|
||||
insufficientResources := fitsRequest(s, nodeInfo, f.ignoredResources)
|
||||
insufficientResources := fitsRequest(s, nodeInfo, f.ignoredResources, f.ignoredResourceGroups)
|
||||
|
||||
if len(insufficientResources) != 0 {
|
||||
// We will keep all failure reasons.
|
||||
@ -187,11 +223,11 @@ type InsufficientResource struct {
|
||||
}
|
||||
|
||||
// Fits checks if node have enough resources to host the pod.
|
||||
func Fits(pod *v1.Pod, nodeInfo *framework.NodeInfo, ignoredExtendedResources sets.String) []InsufficientResource {
|
||||
return fitsRequest(computePodResourceRequest(pod), nodeInfo, ignoredExtendedResources)
|
||||
func Fits(pod *v1.Pod, nodeInfo *framework.NodeInfo) []InsufficientResource {
|
||||
return fitsRequest(computePodResourceRequest(pod), nodeInfo, nil, nil)
|
||||
}
|
||||
|
||||
func fitsRequest(podRequest *preFilterState, nodeInfo *framework.NodeInfo, ignoredExtendedResources sets.String) []InsufficientResource {
|
||||
func fitsRequest(podRequest *preFilterState, nodeInfo *framework.NodeInfo, ignoredExtendedResources, ignoredResourceGroups sets.String) []InsufficientResource {
|
||||
insufficientResources := make([]InsufficientResource, 0, 4)
|
||||
|
||||
allowedPodNumber := nodeInfo.Allocatable.AllowedPodNumber
|
||||
@ -205,10 +241,6 @@ func fitsRequest(podRequest *preFilterState, nodeInfo *framework.NodeInfo, ignor
|
||||
})
|
||||
}
|
||||
|
||||
if ignoredExtendedResources == nil {
|
||||
ignoredExtendedResources = sets.NewString()
|
||||
}
|
||||
|
||||
if podRequest.MilliCPU == 0 &&
|
||||
podRequest.Memory == 0 &&
|
||||
podRequest.EphemeralStorage == 0 &&
|
||||
@ -246,9 +278,13 @@ func fitsRequest(podRequest *preFilterState, nodeInfo *framework.NodeInfo, ignor
|
||||
|
||||
for rName, rQuant := range podRequest.ScalarResources {
|
||||
if v1helper.IsExtendedResourceName(rName) {
|
||||
// If this resource is one of the extended resources that should be
|
||||
// ignored, we will skip checking it.
|
||||
if ignoredExtendedResources.Has(string(rName)) {
|
||||
// If this resource is one of the extended resources that should be ignored, we will skip checking it.
|
||||
// rName is guaranteed to have a slash due to API validation.
|
||||
var rNamePrefix string
|
||||
if ignoredResourceGroups.Len() > 0 {
|
||||
rNamePrefix = strings.Split(string(rName), "/")[0]
|
||||
}
|
||||
if ignoredExtendedResources.Has(string(rName)) || ignoredResourceGroups.Has(rNamePrefix) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
@ -20,13 +20,13 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"k8s.io/kubernetes/pkg/scheduler/apis/config"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
|
||||
"k8s.io/kubernetes/pkg/scheduler/apis/config"
|
||||
framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
|
||||
)
|
||||
|
||||
@ -367,6 +367,31 @@ func TestEnoughRequests(t *testing.T) {
|
||||
wantStatus: framework.NewStatus(framework.Unschedulable, getErrReason(v1.ResourceMemory)),
|
||||
wantInsufficientResources: []InsufficientResource{{v1.ResourceMemory, getErrReason(v1.ResourceMemory), 16, 5, 20}},
|
||||
},
|
||||
{
|
||||
pod: newResourcePod(
|
||||
framework.Resource{
|
||||
MilliCPU: 1,
|
||||
Memory: 1,
|
||||
ScalarResources: map[v1.ResourceName]int64{
|
||||
extendedResourceB: 1,
|
||||
kubernetesIOResourceA: 1,
|
||||
}}),
|
||||
nodeInfo: framework.NewNodeInfo(newResourcePod(framework.Resource{MilliCPU: 0, Memory: 0})),
|
||||
args: config.NodeResourcesFitArgs{
|
||||
IgnoredResourceGroups: []string{"example.com"},
|
||||
},
|
||||
name: "skip checking ignored extended resource via resource groups",
|
||||
wantStatus: framework.NewStatus(framework.Unschedulable, fmt.Sprintf("Insufficient %v", kubernetesIOResourceA)),
|
||||
wantInsufficientResources: []InsufficientResource{
|
||||
{
|
||||
ResourceName: kubernetesIOResourceA,
|
||||
Reason: fmt.Sprintf("Insufficient %v", kubernetesIOResourceA),
|
||||
Requested: 1,
|
||||
Used: 0,
|
||||
Capacity: 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range enoughPodsTests {
|
||||
@ -389,9 +414,9 @@ func TestEnoughRequests(t *testing.T) {
|
||||
t.Errorf("status does not match: %v, want: %v", gotStatus, test.wantStatus)
|
||||
}
|
||||
|
||||
gotInsufficientResources := Fits(test.pod, test.nodeInfo, p.(*Fit).ignoredResources)
|
||||
gotInsufficientResources := fitsRequest(computePodResourceRequest(test.pod), test.nodeInfo, p.(*Fit).ignoredResources, p.(*Fit).ignoredResourceGroups)
|
||||
if !reflect.DeepEqual(gotInsufficientResources, test.wantInsufficientResources) {
|
||||
t.Errorf("insufficient resources do not match: %v, want: %v", gotInsufficientResources, test.wantInsufficientResources)
|
||||
t.Errorf("insufficient resources do not match: %+v, want: %v", gotInsufficientResources, test.wantInsufficientResources)
|
||||
}
|
||||
})
|
||||
}
|
||||
@ -529,3 +554,72 @@ func TestStorageRequests(t *testing.T) {
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestValidateFitArgs(t *testing.T) {
|
||||
argsTest := []struct {
|
||||
name string
|
||||
args config.NodeResourcesFitArgs
|
||||
expect string
|
||||
}{
|
||||
{
|
||||
name: "IgnoredResources: too long value",
|
||||
args: config.NodeResourcesFitArgs{
|
||||
IgnoredResources: []string{fmt.Sprintf("longvalue%s", strings.Repeat("a", 64))},
|
||||
},
|
||||
expect: "name part must be no more than 63 characters",
|
||||
},
|
||||
{
|
||||
name: "IgnoredResources: name is empty",
|
||||
args: config.NodeResourcesFitArgs{
|
||||
IgnoredResources: []string{"example.com/"},
|
||||
},
|
||||
expect: "name part must be non-empty",
|
||||
},
|
||||
{
|
||||
name: "IgnoredResources: name has too many slash",
|
||||
args: config.NodeResourcesFitArgs{
|
||||
IgnoredResources: []string{"example.com/aaa/bbb"},
|
||||
},
|
||||
expect: "a qualified name must consist of alphanumeric characters",
|
||||
},
|
||||
{
|
||||
name: "IgnoredResources: valid args",
|
||||
args: config.NodeResourcesFitArgs{
|
||||
IgnoredResources: []string{"example.com"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "IgnoredResourceGroups: valid args ",
|
||||
args: config.NodeResourcesFitArgs{
|
||||
IgnoredResourceGroups: []string{"example.com"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "IgnoredResourceGroups: illegal args",
|
||||
args: config.NodeResourcesFitArgs{
|
||||
IgnoredResourceGroups: []string{"example.com/"},
|
||||
},
|
||||
expect: "name part must be non-empty",
|
||||
},
|
||||
{
|
||||
name: "IgnoredResourceGroups: name is too long",
|
||||
args: config.NodeResourcesFitArgs{
|
||||
IgnoredResourceGroups: []string{strings.Repeat("a", 64)},
|
||||
},
|
||||
expect: "name part must be no more than 63 characters",
|
||||
},
|
||||
{
|
||||
name: "IgnoredResourceGroups: name cannot be contain slash",
|
||||
args: config.NodeResourcesFitArgs{
|
||||
IgnoredResourceGroups: []string{"example.com/aa"},
|
||||
},
|
||||
expect: "resource group name can't contain '/'",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range argsTest {
|
||||
if err := validateFitArgs(test.args); err != nil && !strings.Contains(err.Error(), test.expect) {
|
||||
t.Errorf("case[%v]: error details do not include %v", test.name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -62,6 +62,12 @@ type NodeResourcesFitArgs struct {
|
||||
// should ignore.
|
||||
// +listType=atomic
|
||||
IgnoredResources []string `json:"ignoredResources,omitempty"`
|
||||
// IgnoredResourceGroups defines the list of resource groups that NodeResources fit filter should ignore.
|
||||
// e.g. if group is ["example.com"], it will ignore all resource names that begin
|
||||
// with "example.com", such as "example.com/aaa" and "example.com/bbb".
|
||||
// A resource group name can't contain '/'.
|
||||
// +listType=atomic
|
||||
IgnoredResourceGroups []string `json:"ignoredResourceGroups,omitempty"`
|
||||
}
|
||||
|
||||
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
|
||||
|
@ -237,6 +237,11 @@ func (in *NodeResourcesFitArgs) DeepCopyInto(out *NodeResourcesFitArgs) {
|
||||
*out = make([]string, len(*in))
|
||||
copy(*out, *in)
|
||||
}
|
||||
if in.IgnoredResourceGroups != nil {
|
||||
in, out := &in.IgnoredResourceGroups, &out.IgnoredResourceGroups
|
||||
*out = make([]string, len(*in))
|
||||
copy(*out, *in)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user