mirror of
https://github.com/k3s-io/kubernetes.git
synced 2026-08-08 23:37:11 +00:00
scheduler: Add BindingTimeout args to DynamicResources plugin
Add a new `bindingTimeout` field to DynamicResources plugin args and wire it into PreBind. Changes: - API: add `bindingTimeout` to DynamicResourcesArgs (staging + internal types). - Defaults: default to 600 seconds when BOTH DRADeviceBindingConditions and DRAResourceClaimDeviceStatus are enabled. - Validation: require >= 1s; forbid when either feature gate is disabled. - Plugin: plumbs args into `pl.bindingTimeout` and uses it in `wait.PollUntilContextTimeout` for binding-condition wait logic. - Plugin: remove legacy `BindingTimeoutDefaultSeconds`. Tests: - Add/adjust unit tests for validation and PreBind timeout path. - Ensure <1s and negative values are rejected; forbids when gates disabled.
This commit is contained in:
6
pkg/generated/openapi/zz_generated.openapi.go
generated
6
pkg/generated/openapi/zz_generated.openapi.go
generated
@@ -68495,6 +68495,12 @@ func schema_k8sio_kube_scheduler_config_v1_DynamicResourcesArgs(ref common.Refer
|
||||
Ref: ref(metav1.Duration{}.OpenAPIModelName()),
|
||||
},
|
||||
},
|
||||
"bindingTimeout": {
|
||||
SchemaProps: spec.SchemaProps{
|
||||
Description: "BindingTimeout limits how long the PreBind extension point may wait for ResourceClaim device BindingConditions to become satisfied when such conditions are present. While waiting, the scheduler periodically checks device status. If the timeout elapses before all required conditions are true (or any bindingFailureConditions become true), the allocation is cleared and the Pod re-enters scheduling queue. Note that the same or other node may be chosen if feasible; otherwise the Pod is placed in the unschedulable queue and retried based on cluster changes and backoff.\n\nDefaults & feature gates:\n - Defaults to 10 minutes when the DRADeviceBindingConditions feature gate is enabled.\n - Has effect only when BOTH DRADeviceBindingConditions and\n DRAResourceClaimDeviceStatus are enabled; otherwise omit this field.\n - When DRADeviceBindingConditions is disabled, setting this field is considered an error.\n\nValid values:\n - >=1s (non-zero). No upper bound is enforced.\n\nTuning guidance:\n - Lower values reduce time-to-retry when devices aren’t ready but can\n increase churn if drivers typically need longer to report readiness.\n - Review scheduler latency metrics (e.g. PreBind duration in\n `scheduler_framework_extension_point_duration_seconds`) and driver\n readiness behavior before tightening this timeout.",
|
||||
Ref: ref(metav1.Duration{}.OpenAPIModelName()),
|
||||
},
|
||||
},
|
||||
},
|
||||
Required: []string{"filterTimeout"},
|
||||
},
|
||||
|
||||
@@ -57,6 +57,7 @@ profiles:
|
||||
- name: DynamicResources
|
||||
args:
|
||||
filterTimeout: 10s
|
||||
bindingTimeout: 30s
|
||||
- name: InterPodAffinity
|
||||
args:
|
||||
hardPodAffinityWeight: 5
|
||||
@@ -107,7 +108,10 @@ profiles:
|
||||
},
|
||||
{
|
||||
Name: "DynamicResources",
|
||||
Args: &config.DynamicResourcesArgs{FilterTimeout: &metav1.Duration{Duration: 10 * time.Second}},
|
||||
Args: &config.DynamicResourcesArgs{
|
||||
FilterTimeout: &metav1.Duration{Duration: 10 * time.Second},
|
||||
BindingTimeout: &metav1.Duration{Duration: 30 * time.Second},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "InterPodAffinity",
|
||||
|
||||
@@ -257,6 +257,33 @@ type DynamicResourcesArgs struct {
|
||||
//
|
||||
// Setting it to zero completely disables the timeout.
|
||||
FilterTimeout *metav1.Duration
|
||||
|
||||
// BindingTimeout limits how long the PreBind extension point may wait for
|
||||
// ResourceClaim device BindingConditions to become satisfied when such
|
||||
// conditions are present. While waiting, the scheduler periodically checks
|
||||
// device status. If the timeout elapses before all required conditions are
|
||||
// true (or any bindingFailureConditions become true), the allocation is
|
||||
// cleared and the Pod re-enters scheduling queue. Note that the same or other node may be
|
||||
// chosen if feasible; otherwise the Pod is placed in the unschedulable queue and
|
||||
// retried based on cluster changes and backoff.
|
||||
//
|
||||
// Defaults & feature gates:
|
||||
// - Defaults to 10 minutes when the DRADeviceBindingConditions feature gate is enabled.
|
||||
// - Has effect only when BOTH DRADeviceBindingConditions and
|
||||
// DRAResourceClaimDeviceStatus are enabled; otherwise omit this field.
|
||||
// - When DRADeviceBindingConditions is disabled, setting this field is considered an error.
|
||||
//
|
||||
// Valid values:
|
||||
// - >=1s (non-zero). No upper bound is enforced.
|
||||
//
|
||||
// Tuning guidance:
|
||||
// - Lower values reduce time-to-retry when devices aren’t ready but can
|
||||
// increase churn if drivers typically need longer to report readiness.
|
||||
// - Review scheduler latency metrics (e.g. PreBind duration in
|
||||
// `scheduler_framework_extension_point_duration_seconds`) and driver
|
||||
// readiness behavior before tightening this timeout.
|
||||
BindingTimeout *metav1.Duration
|
||||
}
|
||||
|
||||
const DynamicResourcesFilterTimeoutDefault = 10 * time.Second
|
||||
const DynamicResourcesBindingTimeoutDefault = 600 * time.Second
|
||||
|
||||
@@ -248,4 +248,10 @@ func SetDefaults_DynamicResourcesArgs(obj *configv1.DynamicResourcesArgs) {
|
||||
if obj.FilterTimeout == nil && feature.DefaultFeatureGate.Enabled(features.DRASchedulerFilterTimeout) {
|
||||
obj.FilterTimeout = &metav1.Duration{Duration: configv1.DynamicResourcesFilterTimeoutDefault}
|
||||
}
|
||||
|
||||
if obj.BindingTimeout == nil &&
|
||||
feature.DefaultFeatureGate.Enabled(features.DRADeviceBindingConditions) &&
|
||||
feature.DefaultFeatureGate.Enabled(features.DRAResourceClaimDeviceStatus) {
|
||||
obj.BindingTimeout = &metav1.Duration{Duration: configv1.DynamicResourcesBindingTimeoutDefault}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -892,6 +892,18 @@ func TestPluginArgsDefaults(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "DynamicResourcesArgs defaults, DRADeviceBindingConditions enabled",
|
||||
features: map[featuregate.Feature]bool{
|
||||
features.DRADeviceBindingConditions: true,
|
||||
features.DRAResourceClaimDeviceStatus: true,
|
||||
},
|
||||
in: &configv1.DynamicResourcesArgs{},
|
||||
want: &configv1.DynamicResourcesArgs{
|
||||
FilterTimeout: &metav1.Duration{Duration: 10 * time.Second},
|
||||
BindingTimeout: &metav1.Duration{Duration: 600 * time.Second},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
scheme := runtime.NewScheme()
|
||||
|
||||
@@ -285,6 +285,7 @@ func Convert_config_DefaultPreemptionArgs_To_v1_DefaultPreemptionArgs(in *config
|
||||
|
||||
func autoConvert_v1_DynamicResourcesArgs_To_config_DynamicResourcesArgs(in *configv1.DynamicResourcesArgs, out *config.DynamicResourcesArgs, s conversion.Scope) error {
|
||||
out.FilterTimeout = (*metav1.Duration)(unsafe.Pointer(in.FilterTimeout))
|
||||
out.BindingTimeout = (*metav1.Duration)(unsafe.Pointer(in.BindingTimeout))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -295,6 +296,7 @@ func Convert_v1_DynamicResourcesArgs_To_config_DynamicResourcesArgs(in *configv1
|
||||
|
||||
func autoConvert_config_DynamicResourcesArgs_To_v1_DynamicResourcesArgs(in *config.DynamicResourcesArgs, out *configv1.DynamicResourcesArgs, s conversion.Scope) error {
|
||||
out.FilterTimeout = (*metav1.Duration)(unsafe.Pointer(in.FilterTimeout))
|
||||
out.BindingTimeout = (*metav1.Duration)(unsafe.Pointer(in.BindingTimeout))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ package validation
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1validation "k8s.io/apimachinery/pkg/apis/meta/v1/validation"
|
||||
@@ -343,5 +344,22 @@ func ValidateDynamicResourcesArgs(path *field.Path, args *config.DynamicResource
|
||||
allErrs = append(allErrs, field.Forbidden(path.Child("filterTimeout"), "DRASchedulingFilterTimeout feature gate is disabled"))
|
||||
}
|
||||
}
|
||||
|
||||
if fts.EnableDRADeviceBindingConditions && fts.EnableDRAResourceClaimDeviceStatus {
|
||||
if args.BindingTimeout != nil && args.BindingTimeout.Duration < 1*time.Second {
|
||||
allErrs = append(allErrs, field.Invalid(
|
||||
path.Child("bindingTimeout"),
|
||||
args.BindingTimeout,
|
||||
"must be at least 1 second",
|
||||
))
|
||||
}
|
||||
} else {
|
||||
if args.BindingTimeout != nil {
|
||||
allErrs = append(allErrs, field.Forbidden(
|
||||
path.Child("bindingTimeout"),
|
||||
"DRADeviceBindingConditions or DRAResourceClaimDeviceStatus feature gate is disabled",
|
||||
))
|
||||
}
|
||||
}
|
||||
return allErrs.ToAggregate()
|
||||
}
|
||||
|
||||
@@ -1158,13 +1158,16 @@ func TestValidateRequestedToCapacityRatioScoringStrategy(t *testing.T) {
|
||||
|
||||
func TestValidateDynamicResourcesArgs(t *testing.T) {
|
||||
cases := map[string]struct {
|
||||
args config.DynamicResourcesArgs
|
||||
wantErrs field.ErrorList
|
||||
filterTimeoutDisabled bool
|
||||
args config.DynamicResourcesArgs
|
||||
wantErrs field.ErrorList
|
||||
filterTimeoutDisabled bool
|
||||
bindingConditionsDisabled bool
|
||||
deviceStatusDisabled bool
|
||||
}{
|
||||
"valid args (default)": {
|
||||
args: config.DynamicResourcesArgs{
|
||||
FilterTimeout: &metav1.Duration{Duration: config.DynamicResourcesFilterTimeoutDefault},
|
||||
FilterTimeout: &metav1.Duration{Duration: config.DynamicResourcesFilterTimeoutDefault},
|
||||
BindingTimeout: &metav1.Duration{Duration: config.DynamicResourcesBindingTimeoutDefault},
|
||||
},
|
||||
},
|
||||
"valid args (disabled)": {
|
||||
@@ -1195,11 +1198,76 @@ func TestValidateDynamicResourcesArgs(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
// BindingTimeout tests
|
||||
"valid BindingTimeout": {
|
||||
args: config.DynamicResourcesArgs{
|
||||
BindingTimeout: &metav1.Duration{Duration: 30 * time.Second},
|
||||
},
|
||||
},
|
||||
"BindingTimeout < 1s (0s)": {
|
||||
args: config.DynamicResourcesArgs{
|
||||
BindingTimeout: &metav1.Duration{Duration: 0},
|
||||
},
|
||||
wantErrs: field.ErrorList{
|
||||
&field.Error{
|
||||
Type: field.ErrorTypeInvalid,
|
||||
Field: "bindingTimeout",
|
||||
Detail: "must be at least 1 second",
|
||||
},
|
||||
},
|
||||
},
|
||||
"BindingTimeout < 1s (negative)": {
|
||||
args: config.DynamicResourcesArgs{
|
||||
BindingTimeout: &metav1.Duration{Duration: -time.Second},
|
||||
},
|
||||
wantErrs: field.ErrorList{
|
||||
&field.Error{
|
||||
Type: field.ErrorTypeInvalid,
|
||||
Field: "bindingTimeout",
|
||||
Detail: "must be at least 1 second",
|
||||
},
|
||||
},
|
||||
},
|
||||
"BindingTimeout set but DRADeviceBindingConditions disabled": {
|
||||
args: config.DynamicResourcesArgs{
|
||||
BindingTimeout: &metav1.Duration{Duration: time.Second},
|
||||
},
|
||||
bindingConditionsDisabled: true,
|
||||
wantErrs: field.ErrorList{
|
||||
&field.Error{
|
||||
Type: field.ErrorTypeForbidden,
|
||||
Field: "bindingTimeout",
|
||||
Detail: "requires DRADeviceBindingConditions and DRAResourceClaimDeviceStatus feature gates to be enabled",
|
||||
},
|
||||
},
|
||||
},
|
||||
"BindingTimeout set but DRAResourceClaimDeviceStatus disabled": {
|
||||
args: config.DynamicResourcesArgs{
|
||||
BindingTimeout: &metav1.Duration{Duration: time.Second},
|
||||
},
|
||||
deviceStatusDisabled: true,
|
||||
wantErrs: field.ErrorList{
|
||||
&field.Error{
|
||||
Type: field.ErrorTypeForbidden,
|
||||
Field: "bindingTimeout",
|
||||
Detail: "requires DRADeviceBindingConditions and DRAResourceClaimDeviceStatus feature gates to be enabled",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for name, tc := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
err := ValidateDynamicResourcesArgs(nil, &tc.args, schedfeature.Features{EnableDRASchedulerFilterTimeout: !tc.filterTimeoutDisabled})
|
||||
err := ValidateDynamicResourcesArgs(
|
||||
nil,
|
||||
&tc.args,
|
||||
schedfeature.Features{
|
||||
EnableDRASchedulerFilterTimeout: !tc.filterTimeoutDisabled,
|
||||
EnableDRADeviceBindingConditions: !tc.bindingConditionsDisabled,
|
||||
EnableDRAResourceClaimDeviceStatus: !tc.deviceStatusDisabled,
|
||||
},
|
||||
)
|
||||
if diff := cmp.Diff(tc.wantErrs.ToAggregate(), err, ignoreBadValueDetail); diff != "" {
|
||||
t.Errorf("ValidateDynamicResourcesArgs returned err (-want,+got):\n%s", diff)
|
||||
}
|
||||
|
||||
@@ -61,6 +61,11 @@ func (in *DynamicResourcesArgs) DeepCopyInto(out *DynamicResourcesArgs) {
|
||||
*out = new(v1.Duration)
|
||||
**out = **in
|
||||
}
|
||||
if in.BindingTimeout != nil {
|
||||
in, out := &in.BindingTimeout, &out.BindingTimeout
|
||||
*out = new(v1.Duration)
|
||||
**out = **in
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -73,10 +73,6 @@ const (
|
||||
// some actual ResourceClaim in the apiserver.
|
||||
specialClaimInMemName = "<extended-resources>"
|
||||
|
||||
// BindingTimeoutDefaultSeconds is the default timeout for waiting for
|
||||
// BindingConditions to be ready.
|
||||
BindingTimeoutDefaultSeconds = 600
|
||||
|
||||
// AssumeExtendedResourceTimeoutDefaultSeconds is the default timeout for waiting
|
||||
// for the extended resource claim to be updated in assumed cache.
|
||||
AssumeExtendedResourceTimeoutDefaultSeconds = 120
|
||||
@@ -161,13 +157,14 @@ type nodeAllocation struct {
|
||||
|
||||
// DynamicResources is a plugin that ensures that ResourceClaims are allocated.
|
||||
type DynamicResources struct {
|
||||
enabled bool
|
||||
fts feature.Features
|
||||
filterTimeout time.Duration
|
||||
fh fwk.Handle
|
||||
clientset kubernetes.Interface
|
||||
celCache *cel.Cache
|
||||
draManager fwk.SharedDRAManager
|
||||
enabled bool
|
||||
fts feature.Features
|
||||
filterTimeout time.Duration
|
||||
bindingTimeout time.Duration
|
||||
fh fwk.Handle
|
||||
clientset kubernetes.Interface
|
||||
celCache *cel.Cache
|
||||
draManager fwk.SharedDRAManager
|
||||
}
|
||||
|
||||
// New initializes a new plugin and returns it.
|
||||
@@ -189,7 +186,10 @@ func New(ctx context.Context, plArgs runtime.Object, fh fwk.Handle, fts feature.
|
||||
enabled: true,
|
||||
fts: fts,
|
||||
filterTimeout: ptr.Deref(args.FilterTimeout, metav1.Duration{}).Duration,
|
||||
|
||||
bindingTimeout: ptr.Deref(
|
||||
args.BindingTimeout,
|
||||
metav1.Duration{Duration: config.DynamicResourcesBindingTimeoutDefault},
|
||||
).Duration,
|
||||
fh: fh,
|
||||
clientset: fh.ClientSet(),
|
||||
// This is a LRU cache for compiled CEL expressions. The most
|
||||
@@ -1320,7 +1320,7 @@ func (pl *DynamicResources) PreBind(ctx context.Context, cs fwk.CycleState, pod
|
||||
|
||||
// We need to wait for the device to be attached to the node.
|
||||
pl.fh.EventRecorder().Eventf(pod, nil, v1.EventTypeNormal, "BindingConditionsPending", "Scheduling", "waiting for binding conditions for device on node %s", nodeName)
|
||||
err = wait.PollUntilContextTimeout(ctx, 5*time.Second, time.Duration(BindingTimeoutDefaultSeconds)*time.Second, true,
|
||||
err = wait.PollUntilContextTimeout(ctx, 5*time.Second, pl.bindingTimeout, true,
|
||||
func(ctx context.Context) (bool, error) {
|
||||
return pl.isPodReadyForBinding(state)
|
||||
})
|
||||
@@ -1627,7 +1627,7 @@ func (pl *DynamicResources) isClaimTimeout(claim *resourceapi.ResourceClaim) boo
|
||||
if deviceRequest.BindingConditions == nil {
|
||||
continue
|
||||
}
|
||||
if claim.Status.Allocation.AllocationTimestamp.Add(time.Duration(BindingTimeoutDefaultSeconds) * time.Second).Before(time.Now()) {
|
||||
if claim.Status.Allocation.AllocationTimestamp.Add(pl.bindingTimeout).Before(time.Now()) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1797,7 +1797,10 @@ func TestPlugin(t *testing.T) {
|
||||
"prebind-fail-with-binding-timeout": {
|
||||
enableDRADeviceBindingConditions: true,
|
||||
enableDRAResourceClaimDeviceStatus: true,
|
||||
pod: podWithClaimName,
|
||||
args: &config.DynamicResourcesArgs{
|
||||
BindingTimeout: &metav1.Duration{Duration: 600 * time.Second},
|
||||
},
|
||||
pod: podWithClaimName,
|
||||
claims: func() []*resourceapi.ResourceClaim {
|
||||
claim := allocatedClaim.DeepCopy()
|
||||
claim.Status.Allocation = allocationResultWithBindingConditions.DeepCopy()
|
||||
|
||||
@@ -432,6 +432,33 @@ type DynamicResourcesArgs struct {
|
||||
//
|
||||
// Setting it to zero completely disables the timeout.
|
||||
FilterTimeout *metav1.Duration `json:"filterTimeout"`
|
||||
|
||||
// BindingTimeout limits how long the PreBind extension point may wait for
|
||||
// ResourceClaim device BindingConditions to become satisfied when such
|
||||
// conditions are present. While waiting, the scheduler periodically checks
|
||||
// device status. If the timeout elapses before all required conditions are
|
||||
// true (or any bindingFailureConditions become true), the allocation is
|
||||
// cleared and the Pod re-enters scheduling queue. Note that the same or other node may be
|
||||
// chosen if feasible; otherwise the Pod is placed in the unschedulable queue and
|
||||
// retried based on cluster changes and backoff.
|
||||
//
|
||||
// Defaults & feature gates:
|
||||
// - Defaults to 10 minutes when the DRADeviceBindingConditions feature gate is enabled.
|
||||
// - Has effect only when BOTH DRADeviceBindingConditions and
|
||||
// DRAResourceClaimDeviceStatus are enabled; otherwise omit this field.
|
||||
// - When DRADeviceBindingConditions is disabled, setting this field is considered an error.
|
||||
//
|
||||
// Valid values:
|
||||
// - >=1s (non-zero). No upper bound is enforced.
|
||||
//
|
||||
// Tuning guidance:
|
||||
// - Lower values reduce time-to-retry when devices aren’t ready but can
|
||||
// increase churn if drivers typically need longer to report readiness.
|
||||
// - Review scheduler latency metrics (e.g. PreBind duration in
|
||||
// `scheduler_framework_extension_point_duration_seconds`) and driver
|
||||
// readiness behavior before tightening this timeout.
|
||||
BindingTimeout *metav1.Duration `json:"bindingTimeout,omitempty"`
|
||||
}
|
||||
|
||||
const DynamicResourcesFilterTimeoutDefault = 10 * time.Second
|
||||
const DynamicResourcesBindingTimeoutDefault = 600 * time.Second
|
||||
|
||||
@@ -71,6 +71,11 @@ func (in *DynamicResourcesArgs) DeepCopyInto(out *DynamicResourcesArgs) {
|
||||
*out = new(metav1.Duration)
|
||||
**out = **in
|
||||
}
|
||||
if in.BindingTimeout != nil {
|
||||
in, out := &in.BindingTimeout, &out.BindingTimeout
|
||||
*out = new(metav1.Duration)
|
||||
**out = **in
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user