From 72863f65d6bb833b1899b3d58016c02cfb3ac8d5 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Wed, 15 Feb 2023 15:04:59 -0800 Subject: [PATCH 1/2] Graduate PodSchedulingReadiness to beta --- .../app/options/options_test.go | 40 +-- cmd/kube-scheduler/app/server_test.go | 242 +++++++----------- pkg/apis/core/types.go | 9 +- pkg/features/kube_features.go | 3 +- .../apis/config/testing/defaults/defaults.go | 17 ++ .../apis/config/v1/default_plugins_test.go | 4 + pkg/scheduler/apis/config/v1/defaults_test.go | 1 + .../config/v1beta2/default_plugins_test.go | 3 + .../apis/config/v1beta2/defaults_test.go | 5 + .../config/v1beta3/default_plugins_test.go | 3 + .../apis/config/v1beta3/defaults_test.go | 1 + staging/src/k8s.io/api/core/v1/types.go | 11 +- test/e2e/scheduling/predicates.go | 2 +- 13 files changed, 160 insertions(+), 181 deletions(-) diff --git a/cmd/kube-scheduler/app/options/options_test.go b/cmd/kube-scheduler/app/options/options_test.go index 8fc58613dd5..abce33634cc 100644 --- a/cmd/kube-scheduler/app/options/options_test.go +++ b/cmd/kube-scheduler/app/options/options_test.go @@ -1438,37 +1438,23 @@ profiles: Profiles: []kubeschedulerconfig.KubeSchedulerProfile{ { SchedulerName: "foo-profile", - Plugins: &kubeschedulerconfig.Plugins{ - QueueSort: defaults.PluginsV1beta2.QueueSort, - PreFilter: defaults.PluginsV1beta2.PreFilter, - Filter: defaults.PluginsV1beta2.Filter, - PostFilter: defaults.PluginsV1beta2.PostFilter, - PreScore: defaults.PluginsV1beta2.PreScore, - Score: defaults.PluginsV1beta2.Score, - Bind: defaults.PluginsV1beta2.Bind, - PreBind: defaults.PluginsV1beta2.PreBind, - Reserve: kubeschedulerconfig.PluginSet{ - Enabled: []kubeschedulerconfig.Plugin{ - {Name: "foo"}, - {Name: names.VolumeBinding}, - }, - }, - }, + Plugins: func() *kubeschedulerconfig.Plugins { + plugins := defaults.PluginsV1beta2.DeepCopy() + plugins.Reserve.Enabled = []kubeschedulerconfig.Plugin{ + {Name: "foo"}, + {Name: names.VolumeBinding}, + } + return plugins + }(), PluginConfig: defaults.PluginConfigsV1beta2, }, { SchedulerName: "bar-profile", - Plugins: &kubeschedulerconfig.Plugins{ - PreEnqueue: defaults.PluginsV1beta2.PreEnqueue, - QueueSort: defaults.PluginsV1beta2.QueueSort, - PreFilter: defaults.PluginsV1beta2.PreFilter, - Filter: defaults.PluginsV1beta2.Filter, - PostFilter: defaults.PluginsV1beta2.PostFilter, - PreScore: defaults.PluginsV1beta2.PreScore, - Score: defaults.PluginsV1beta2.Score, - Bind: defaults.PluginsV1beta2.Bind, - Reserve: defaults.PluginsV1beta2.Reserve, - }, + Plugins: func() *kubeschedulerconfig.Plugins { + plugins := defaults.PluginsV1beta2.DeepCopy() + plugins.PreBind.Enabled = nil + return plugins + }(), PluginConfig: []kubeschedulerconfig.PluginConfig{ { Name: "foo", diff --git a/cmd/kube-scheduler/app/server_test.go b/cmd/kube-scheduler/app/server_test.go index ec8733b3cc9..1024bc501ea 100644 --- a/cmd/kube-scheduler/app/server_test.go +++ b/cmd/kube-scheduler/app/server_test.go @@ -42,7 +42,6 @@ import ( "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/apis/config/testing/defaults" "k8s.io/kubernetes/pkg/scheduler/framework" - "k8s.io/kubernetes/pkg/scheduler/framework/plugins/names" ) func TestSetup(t *testing.T) { @@ -318,21 +317,33 @@ leaderElection: wantLeaderElection *componentbaseconfig.LeaderElectionConfiguration }{ { - name: "default config with two alpha features enabled", + name: "default config with an alpha feature enabled", flags: []string{ "--kubeconfig", configKubeconfig, - "--feature-gates=VolumeCapacityPriority=true,PodSchedulingReadiness=true", + "--feature-gates=VolumeCapacityPriority=true", + }, + wantPlugins: map[string]*config.Plugins{ + "default-scheduler": defaults.ExpandedPluginsV1, + }, + restoreFeatures: map[featuregate.Feature]bool{ + features.VolumeCapacityPriority: false, + }, + }, + { + name: "default config with a beta feature disabled", + flags: []string{ + "--kubeconfig", configKubeconfig, + "--feature-gates=PodSchedulingReadiness=false", }, wantPlugins: map[string]*config.Plugins{ "default-scheduler": func() *config.Plugins { plugins := defaults.ExpandedPluginsV1.DeepCopy() - plugins.PreEnqueue.Enabled = append(plugins.PreEnqueue.Enabled, config.Plugin{Name: names.SchedulingGates}) + plugins.PreEnqueue = config.PluginSet{} return plugins }(), }, restoreFeatures: map[featuregate.Feature]bool{ - features.VolumeCapacityPriority: false, - features.PodSchedulingReadiness: false, + features.PodSchedulingReadiness: true, }, }, { @@ -351,37 +362,26 @@ leaderElection: "--kubeconfig", configKubeconfig, }, wantPlugins: map[string]*config.Plugins{ - "default-scheduler": { - Bind: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultBinder"}}}, - Filter: config.PluginSet{ - Enabled: []config.Plugin{ - {Name: "NodeResourcesFit"}, - {Name: "NodePorts"}, - }, - }, - PreFilter: config.PluginSet{ - Enabled: []config.Plugin{ - {Name: "NodeResourcesFit"}, - {Name: "NodePorts"}, - }, - }, - PostFilter: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultPreemption"}}}, - PreScore: config.PluginSet{ - Enabled: []config.Plugin{ - {Name: "InterPodAffinity"}, - {Name: "TaintToleration"}, - }, - }, - QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: "PrioritySort"}}}, - Score: config.PluginSet{ - Enabled: []config.Plugin{ - {Name: "InterPodAffinity", Weight: 1}, - {Name: "TaintToleration", Weight: 1}, - }, - }, - Reserve: config.PluginSet{Enabled: []config.Plugin{{Name: "VolumeBinding"}}}, - PreBind: config.PluginSet{Enabled: []config.Plugin{{Name: "VolumeBinding"}}}, - }, + "default-scheduler": func() *config.Plugins { + plugins := defaults.PluginsV1beta2.DeepCopy() + plugins.Filter.Enabled = []config.Plugin{ + {Name: "NodeResourcesFit"}, + {Name: "NodePorts"}, + } + plugins.PreFilter.Enabled = []config.Plugin{ + {Name: "NodeResourcesFit"}, + {Name: "NodePorts"}, + } + plugins.PreScore.Enabled = []config.Plugin{ + {Name: "InterPodAffinity"}, + {Name: "TaintToleration"}, + } + plugins.Score.Enabled = []config.Plugin{ + {Name: "InterPodAffinity", Weight: 1}, + {Name: "TaintToleration", Weight: 1}, + } + return plugins + }(), }, }, { @@ -391,38 +391,26 @@ leaderElection: "--kubeconfig", configKubeconfig, }, wantPlugins: map[string]*config.Plugins{ - "default-scheduler": { - PreEnqueue: config.PluginSet{Enabled: []config.Plugin{{Name: "SchedulingGates"}}}, - Bind: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultBinder"}}}, - Filter: config.PluginSet{ - Enabled: []config.Plugin{ - {Name: "NodeResourcesFit"}, - {Name: "NodePorts"}, - }, - }, - PreFilter: config.PluginSet{ - Enabled: []config.Plugin{ - {Name: "NodeResourcesFit"}, - {Name: "NodePorts"}, - }, - }, - PostFilter: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultPreemption"}}}, - PreScore: config.PluginSet{ - Enabled: []config.Plugin{ - {Name: "InterPodAffinity"}, - {Name: "TaintToleration"}, - }, - }, - QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: "PrioritySort"}}}, - Score: config.PluginSet{ - Enabled: []config.Plugin{ - {Name: "InterPodAffinity", Weight: 1}, - {Name: "TaintToleration", Weight: 1}, - }, - }, - Reserve: config.PluginSet{Enabled: []config.Plugin{{Name: "VolumeBinding"}}}, - PreBind: config.PluginSet{Enabled: []config.Plugin{{Name: "VolumeBinding"}}}, - }, + "default-scheduler": func() *config.Plugins { + plugins := defaults.ExpandedPluginsV1beta3.DeepCopy() + plugins.Filter.Enabled = []config.Plugin{ + {Name: "NodeResourcesFit"}, + {Name: "NodePorts"}, + } + plugins.PreFilter.Enabled = []config.Plugin{ + {Name: "NodeResourcesFit"}, + {Name: "NodePorts"}, + } + plugins.PreScore.Enabled = []config.Plugin{ + {Name: "InterPodAffinity"}, + {Name: "TaintToleration"}, + } + plugins.Score.Enabled = []config.Plugin{ + {Name: "InterPodAffinity", Weight: 1}, + {Name: "TaintToleration", Weight: 1}, + } + return plugins + }(), }, }, { @@ -432,38 +420,26 @@ leaderElection: "--kubeconfig", configKubeconfig, }, wantPlugins: map[string]*config.Plugins{ - "default-scheduler": { - PreEnqueue: config.PluginSet{Enabled: []config.Plugin{{Name: "SchedulingGates"}}}, - Bind: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultBinder"}}}, - Filter: config.PluginSet{ - Enabled: []config.Plugin{ - {Name: "NodeResourcesFit"}, - {Name: "NodePorts"}, - }, - }, - PreFilter: config.PluginSet{ - Enabled: []config.Plugin{ - {Name: "NodeResourcesFit"}, - {Name: "NodePorts"}, - }, - }, - PostFilter: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultPreemption"}}}, - PreScore: config.PluginSet{ - Enabled: []config.Plugin{ - {Name: "InterPodAffinity"}, - {Name: "TaintToleration"}, - }, - }, - QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: "PrioritySort"}}}, - Score: config.PluginSet{ - Enabled: []config.Plugin{ - {Name: "InterPodAffinity", Weight: 1}, - {Name: "TaintToleration", Weight: 1}, - }, - }, - Reserve: config.PluginSet{Enabled: []config.Plugin{{Name: "VolumeBinding"}}}, - PreBind: config.PluginSet{Enabled: []config.Plugin{{Name: "VolumeBinding"}}}, - }, + "default-scheduler": func() *config.Plugins { + plugins := defaults.ExpandedPluginsV1.DeepCopy() + plugins.Filter.Enabled = []config.Plugin{ + {Name: "NodeResourcesFit"}, + {Name: "NodePorts"}, + } + plugins.PreFilter.Enabled = []config.Plugin{ + {Name: "NodeResourcesFit"}, + {Name: "NodePorts"}, + } + plugins.PreScore.Enabled = []config.Plugin{ + {Name: "InterPodAffinity"}, + {Name: "TaintToleration"}, + } + plugins.Score.Enabled = []config.Plugin{ + {Name: "InterPodAffinity", Weight: 1}, + {Name: "TaintToleration", Weight: 1}, + } + return plugins + }(), }, }, { @@ -474,21 +450,12 @@ leaderElection: }, registryOptions: []Option{WithPlugin("Foo", newFoo)}, wantPlugins: map[string]*config.Plugins{ - "default-scheduler": { - Bind: defaults.PluginsV1beta2.Bind, - Filter: config.PluginSet{ - Enabled: append(defaults.PluginsV1beta2.Filter.Enabled, config.Plugin{Name: "Foo"}), - }, - PreFilter: config.PluginSet{ - Enabled: append(defaults.PluginsV1beta2.PreFilter.Enabled, config.Plugin{Name: "Foo"}), - }, - PostFilter: defaults.PluginsV1beta2.PostFilter, - PreScore: defaults.PluginsV1beta2.PreScore, - QueueSort: defaults.PluginsV1beta2.QueueSort, - Score: defaults.PluginsV1beta2.Score, - Reserve: defaults.PluginsV1beta2.Reserve, - PreBind: defaults.PluginsV1beta2.PreBind, - }, + "default-scheduler": func() *config.Plugins { + plugins := defaults.PluginsV1beta2.DeepCopy() + plugins.PreFilter.Enabled = append(plugins.PreFilter.Enabled, config.Plugin{Name: "Foo"}) + plugins.Filter.Enabled = append(plugins.Filter.Enabled, config.Plugin{Name: "Foo"}) + return plugins + }(), }, }, { @@ -499,21 +466,12 @@ leaderElection: }, registryOptions: []Option{WithPlugin("Foo", newFoo)}, wantPlugins: map[string]*config.Plugins{ - "default-scheduler": { - Bind: defaults.ExpandedPluginsV1beta3.Bind, - Filter: config.PluginSet{ - Enabled: append(defaults.ExpandedPluginsV1beta3.Filter.Enabled, config.Plugin{Name: "Foo"}), - }, - PreFilter: config.PluginSet{ - Enabled: append(defaults.ExpandedPluginsV1beta3.PreFilter.Enabled, config.Plugin{Name: "Foo"}), - }, - PostFilter: defaults.ExpandedPluginsV1beta3.PostFilter, - PreScore: defaults.ExpandedPluginsV1beta3.PreScore, - QueueSort: defaults.ExpandedPluginsV1beta3.QueueSort, - Score: defaults.ExpandedPluginsV1beta3.Score, - Reserve: defaults.ExpandedPluginsV1beta3.Reserve, - PreBind: defaults.ExpandedPluginsV1beta3.PreBind, - }, + "default-scheduler": func() *config.Plugins { + plugins := defaults.ExpandedPluginsV1beta3.DeepCopy() + plugins.PreFilter.Enabled = append(plugins.PreFilter.Enabled, config.Plugin{Name: "Foo"}) + plugins.Filter.Enabled = append(plugins.Filter.Enabled, config.Plugin{Name: "Foo"}) + return plugins + }(), }, }, { @@ -524,22 +482,12 @@ leaderElection: }, registryOptions: []Option{WithPlugin("Foo", newFoo)}, wantPlugins: map[string]*config.Plugins{ - "default-scheduler": { - PreEnqueue: defaults.ExpandedPluginsV1.PreEnqueue, - Bind: defaults.ExpandedPluginsV1.Bind, - Filter: config.PluginSet{ - Enabled: append(defaults.ExpandedPluginsV1.Filter.Enabled, config.Plugin{Name: "Foo"}), - }, - PreFilter: config.PluginSet{ - Enabled: append(defaults.ExpandedPluginsV1.PreFilter.Enabled, config.Plugin{Name: "Foo"}), - }, - PostFilter: defaults.ExpandedPluginsV1.PostFilter, - PreScore: defaults.ExpandedPluginsV1.PreScore, - QueueSort: defaults.ExpandedPluginsV1.QueueSort, - Score: defaults.ExpandedPluginsV1.Score, - Reserve: defaults.ExpandedPluginsV1.Reserve, - PreBind: defaults.ExpandedPluginsV1.PreBind, - }, + "default-scheduler": func() *config.Plugins { + plugins := defaults.ExpandedPluginsV1.DeepCopy() + plugins.PreFilter.Enabled = append(plugins.PreFilter.Enabled, config.Plugin{Name: "Foo"}) + plugins.Filter.Enabled = append(plugins.Filter.Enabled, config.Plugin{Name: "Foo"}) + return plugins + }(), }, }, { diff --git a/pkg/apis/core/types.go b/pkg/apis/core/types.go index 45514929eba..10a6510e51f 100644 --- a/pkg/apis/core/types.go +++ b/pkg/apis/core/types.go @@ -3044,9 +3044,14 @@ type PodSpec struct { OS *PodOS // SchedulingGates is an opaque list of values that if specified will block scheduling the pod. - // More info: https://git.k8s.io/enhancements/keps/sig-scheduling/3521-pod-scheduling-readiness. + // If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the + // scheduler will not attempt to schedule the pod. // - // This is an alpha-level feature enabled by PodSchedulingReadiness feature gate. + // SchedulingGates can only be set at pod creation time, and be removed only afterwards. + // + // This is a beta feature enabled by the PodSchedulingReadiness feature gate. + // + // +featureGate=PodSchedulingReadiness // +optional SchedulingGates []PodSchedulingGate // ResourceClaims defines which ResourceClaims must be allocated diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 03a567a3b46..959106135ad 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -611,6 +611,7 @@ const ( // owner: @Huang-Wei // kep: https://kep.k8s.io/3521 // alpha: v1.26 + // beta: v1.27 // // Enable users to specify when a Pod is ready for scheduling. PodSchedulingReadiness featuregate.Feature = "PodSchedulingReadiness" @@ -999,7 +1000,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS PodHasNetworkCondition: {Default: false, PreRelease: featuregate.Alpha}, - PodSchedulingReadiness: {Default: false, PreRelease: featuregate.Alpha}, + PodSchedulingReadiness: {Default: true, PreRelease: featuregate.Beta}, PodSecurity: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, diff --git a/pkg/scheduler/apis/config/testing/defaults/defaults.go b/pkg/scheduler/apis/config/testing/defaults/defaults.go index b3492b1289f..c092e11c0e7 100644 --- a/pkg/scheduler/apis/config/testing/defaults/defaults.go +++ b/pkg/scheduler/apis/config/testing/defaults/defaults.go @@ -23,6 +23,11 @@ import ( // PluginsV1beta2 default set of v1beta2 plugins. var PluginsV1beta2 = &config.Plugins{ + PreEnqueue: config.PluginSet{ + Enabled: []config.Plugin{ + {Name: names.SchedulingGates}, + }, + }, QueueSort: config.PluginSet{ Enabled: []config.Plugin{ {Name: names.PrioritySort}, @@ -176,12 +181,18 @@ var PluginsV1beta3 = &config.Plugins{ {Name: names.NodeResourcesBalancedAllocation, Weight: 1}, {Name: names.ImageLocality, Weight: 1}, {Name: names.DefaultBinder}, + {Name: names.SchedulingGates}, }, }, } // ExpandedPluginsV1beta3 default set of v1beta3 plugins after MultiPoint expansion var ExpandedPluginsV1beta3 = &config.Plugins{ + PreEnqueue: config.PluginSet{ + Enabled: []config.Plugin{ + {Name: names.SchedulingGates}, + }, + }, QueueSort: config.PluginSet{ Enabled: []config.Plugin{ {Name: names.PrioritySort}, @@ -347,12 +358,18 @@ var PluginsV1 = &config.Plugins{ {Name: names.NodeResourcesBalancedAllocation, Weight: 1}, {Name: names.ImageLocality, Weight: 1}, {Name: names.DefaultBinder}, + {Name: names.SchedulingGates}, }, }, } // ExpandedPluginsV1 default set of v1 plugins after MultiPoint expansion var ExpandedPluginsV1 = &config.Plugins{ + PreEnqueue: config.PluginSet{ + Enabled: []config.Plugin{ + {Name: names.SchedulingGates}, + }, + }, QueueSort: config.PluginSet{ Enabled: []config.Plugin{ {Name: names.PrioritySort}, diff --git a/pkg/scheduler/apis/config/v1/default_plugins_test.go b/pkg/scheduler/apis/config/v1/default_plugins_test.go index df782e06580..a56a4306ea4 100644 --- a/pkg/scheduler/apis/config/v1/default_plugins_test.go +++ b/pkg/scheduler/apis/config/v1/default_plugins_test.go @@ -38,6 +38,9 @@ func TestApplyFeatureGates(t *testing.T) { }{ { name: "Feature gates disabled", + features: map[featuregate.Feature]bool{ + features.PodSchedulingReadiness: false, + }, wantConfig: &v1.Plugins{ MultiPoint: v1.PluginSet{ Enabled: []v1.Plugin{ @@ -127,6 +130,7 @@ func TestApplyFeatureGates(t *testing.T) { {Name: names.NodeResourcesBalancedAllocation, Weight: pointer.Int32(1)}, {Name: names.ImageLocality, Weight: pointer.Int32(1)}, {Name: names.DefaultBinder}, + {Name: names.SchedulingGates}, }, }, }, diff --git a/pkg/scheduler/apis/config/v1/defaults_test.go b/pkg/scheduler/apis/config/v1/defaults_test.go index 8702e0779b6..38ec7ef8f74 100644 --- a/pkg/scheduler/apis/config/v1/defaults_test.go +++ b/pkg/scheduler/apis/config/v1/defaults_test.go @@ -348,6 +348,7 @@ func TestSchedulerDefaults(t *testing.T) { {Name: names.NodeResourcesBalancedAllocation, Weight: pointer.Int32(1)}, {Name: names.ImageLocality, Weight: pointer.Int32(1)}, {Name: names.DefaultBinder}, + {Name: names.SchedulingGates}, }, }, Bind: configv1.PluginSet{ diff --git a/pkg/scheduler/apis/config/v1beta2/default_plugins_test.go b/pkg/scheduler/apis/config/v1beta2/default_plugins_test.go index 81bcfac7b66..a1bd23b7ded 100644 --- a/pkg/scheduler/apis/config/v1beta2/default_plugins_test.go +++ b/pkg/scheduler/apis/config/v1beta2/default_plugins_test.go @@ -37,6 +37,9 @@ func TestApplyFeatureGates(t *testing.T) { }{ { name: "Feature gates disabled", + features: map[featuregate.Feature]bool{ + features.PodSchedulingReadiness: false, + }, wantConfig: &v1beta2.Plugins{ QueueSort: v1beta2.PluginSet{ Enabled: []v1beta2.Plugin{ diff --git a/pkg/scheduler/apis/config/v1beta2/defaults_test.go b/pkg/scheduler/apis/config/v1beta2/defaults_test.go index 824959e0697..1c0caba5d26 100644 --- a/pkg/scheduler/apis/config/v1beta2/defaults_test.go +++ b/pkg/scheduler/apis/config/v1beta2/defaults_test.go @@ -326,6 +326,11 @@ func TestSchedulerDefaults(t *testing.T) { { SchedulerName: pointer.String("custom-scheduler"), Plugins: &v1beta2.Plugins{ + PreEnqueue: v1beta2.PluginSet{ + Enabled: []v1beta2.Plugin{ + {Name: "SchedulingGates"}, + }, + }, QueueSort: v1beta2.PluginSet{ Enabled: []v1beta2.Plugin{ {Name: names.PrioritySort}, diff --git a/pkg/scheduler/apis/config/v1beta3/default_plugins_test.go b/pkg/scheduler/apis/config/v1beta3/default_plugins_test.go index e1ab9d84a02..d3794398da7 100644 --- a/pkg/scheduler/apis/config/v1beta3/default_plugins_test.go +++ b/pkg/scheduler/apis/config/v1beta3/default_plugins_test.go @@ -37,6 +37,9 @@ func TestApplyFeatureGates(t *testing.T) { }{ { name: "Feature gates disabled", + features: map[featuregate.Feature]bool{ + features.PodSchedulingReadiness: false, + }, wantConfig: &v1beta3.Plugins{ MultiPoint: v1beta3.PluginSet{ Enabled: []v1beta3.Plugin{ diff --git a/pkg/scheduler/apis/config/v1beta3/defaults_test.go b/pkg/scheduler/apis/config/v1beta3/defaults_test.go index 5b555da6a89..8a546ab85dd 100644 --- a/pkg/scheduler/apis/config/v1beta3/defaults_test.go +++ b/pkg/scheduler/apis/config/v1beta3/defaults_test.go @@ -348,6 +348,7 @@ func TestSchedulerDefaults(t *testing.T) { {Name: names.NodeResourcesBalancedAllocation, Weight: pointer.Int32(1)}, {Name: names.ImageLocality, Weight: pointer.Int32(1)}, {Name: names.DefaultBinder}, + {Name: names.SchedulingGates}, }, }, Bind: v1beta3.PluginSet{ diff --git a/staging/src/k8s.io/api/core/v1/types.go b/staging/src/k8s.io/api/core/v1/types.go index 8316a82424f..d969390f0b7 100644 --- a/staging/src/k8s.io/api/core/v1/types.go +++ b/staging/src/k8s.io/api/core/v1/types.go @@ -3386,14 +3386,19 @@ type PodSpec struct { HostUsers *bool `json:"hostUsers,omitempty" protobuf:"bytes,37,opt,name=hostUsers"` // SchedulingGates is an opaque list of values that if specified will block scheduling the pod. - // More info: https://git.k8s.io/enhancements/keps/sig-scheduling/3521-pod-scheduling-readiness. + // If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the + // scheduler will not attempt to schedule the pod. + // + // SchedulingGates can only be set at pod creation time, and be removed only afterwards. + // + // This is a beta feature enabled by the PodSchedulingReadiness feature gate. // - // This is an alpha-level feature enabled by PodSchedulingReadiness feature gate. - // +optional // +patchMergeKey=name // +patchStrategy=merge // +listType=map // +listMapKey=name + // +featureGate=PodSchedulingReadiness + // +optional SchedulingGates []PodSchedulingGate `json:"schedulingGates,omitempty" patchStrategy:"merge" patchMergeKey:"name" protobuf:"bytes,38,opt,name=schedulingGates"` // ResourceClaims defines which ResourceClaims must be allocated // and reserved before the Pod is allowed to start. The resources diff --git a/test/e2e/scheduling/predicates.go b/test/e2e/scheduling/predicates.go index fd21557bb77..46e707a7f6d 100644 --- a/test/e2e/scheduling/predicates.go +++ b/test/e2e/scheduling/predicates.go @@ -806,7 +806,7 @@ var _ = SIGDescribe("SchedulerPredicates [Serial]", func() { }) }) - ginkgo.It("validates Pods with non-empty schedulingGates are blocked on scheduling [Feature:PodSchedulingReadiness] [alpha]", func(ctx context.Context) { + ginkgo.It("validates Pods with non-empty schedulingGates are blocked on scheduling", func(ctx context.Context) { podLabel := "e2e-scheduling-gates" replicas := 3 ginkgo.By(fmt.Sprintf("Creating a ReplicaSet with replicas=%v, carrying scheduling gates [foo bar]", replicas)) From 411200db4afb6ae338e8c3015d44e27773690d88 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Fri, 17 Feb 2023 23:18:45 -0800 Subject: [PATCH 2/2] generated files --- api/openapi-spec/swagger.json | 2 +- api/openapi-spec/v3/api__v1_openapi.json | 2 +- api/openapi-spec/v3/apis__apps__v1_openapi.json | 2 +- api/openapi-spec/v3/apis__batch__v1_openapi.json | 2 +- pkg/generated/openapi/zz_generated.openapi.go | 2 +- staging/src/k8s.io/api/core/v1/generated.proto | 11 ++++++++--- .../k8s.io/api/core/v1/types_swagger_doc_generated.go | 2 +- 7 files changed, 14 insertions(+), 9 deletions(-) diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index fdd41d69948..9e1cca18a28 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -7835,7 +7835,7 @@ "type": "string" }, "schedulingGates": { - "description": "SchedulingGates is an opaque list of values that if specified will block scheduling the pod. More info: https://git.k8s.io/enhancements/keps/sig-scheduling/3521-pod-scheduling-readiness.\n\nThis is an alpha-level feature enabled by PodSchedulingReadiness feature gate.", + "description": "SchedulingGates is an opaque list of values that if specified will block scheduling the pod. If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the scheduler will not attempt to schedule the pod.\n\nSchedulingGates can only be set at pod creation time, and be removed only afterwards.\n\nThis is a beta feature enabled by the PodSchedulingReadiness feature gate.", "items": { "$ref": "#/definitions/io.k8s.api.core.v1.PodSchedulingGate" }, diff --git a/api/openapi-spec/v3/api__v1_openapi.json b/api/openapi-spec/v3/api__v1_openapi.json index 0403e9f5a2c..348000f867f 100644 --- a/api/openapi-spec/v3/api__v1_openapi.json +++ b/api/openapi-spec/v3/api__v1_openapi.json @@ -5209,7 +5209,7 @@ "type": "string" }, "schedulingGates": { - "description": "SchedulingGates is an opaque list of values that if specified will block scheduling the pod. More info: https://git.k8s.io/enhancements/keps/sig-scheduling/3521-pod-scheduling-readiness.\n\nThis is an alpha-level feature enabled by PodSchedulingReadiness feature gate.", + "description": "SchedulingGates is an opaque list of values that if specified will block scheduling the pod. If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the scheduler will not attempt to schedule the pod.\n\nSchedulingGates can only be set at pod creation time, and be removed only afterwards.\n\nThis is a beta feature enabled by the PodSchedulingReadiness feature gate.", "items": { "allOf": [ { diff --git a/api/openapi-spec/v3/apis__apps__v1_openapi.json b/api/openapi-spec/v3/apis__apps__v1_openapi.json index b1f8f6fa4dd..97b6b9b37c9 100644 --- a/api/openapi-spec/v3/apis__apps__v1_openapi.json +++ b/api/openapi-spec/v3/apis__apps__v1_openapi.json @@ -3651,7 +3651,7 @@ "type": "string" }, "schedulingGates": { - "description": "SchedulingGates is an opaque list of values that if specified will block scheduling the pod. More info: https://git.k8s.io/enhancements/keps/sig-scheduling/3521-pod-scheduling-readiness.\n\nThis is an alpha-level feature enabled by PodSchedulingReadiness feature gate.", + "description": "SchedulingGates is an opaque list of values that if specified will block scheduling the pod. If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the scheduler will not attempt to schedule the pod.\n\nSchedulingGates can only be set at pod creation time, and be removed only afterwards.\n\nThis is a beta feature enabled by the PodSchedulingReadiness feature gate.", "items": { "allOf": [ { diff --git a/api/openapi-spec/v3/apis__batch__v1_openapi.json b/api/openapi-spec/v3/apis__batch__v1_openapi.json index 510e2e09934..d01ccc0b1c8 100644 --- a/api/openapi-spec/v3/apis__batch__v1_openapi.json +++ b/api/openapi-spec/v3/apis__batch__v1_openapi.json @@ -2825,7 +2825,7 @@ "type": "string" }, "schedulingGates": { - "description": "SchedulingGates is an opaque list of values that if specified will block scheduling the pod. More info: https://git.k8s.io/enhancements/keps/sig-scheduling/3521-pod-scheduling-readiness.\n\nThis is an alpha-level feature enabled by PodSchedulingReadiness feature gate.", + "description": "SchedulingGates is an opaque list of values that if specified will block scheduling the pod. If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the scheduler will not attempt to schedule the pod.\n\nSchedulingGates can only be set at pod creation time, and be removed only afterwards.\n\nThis is a beta feature enabled by the PodSchedulingReadiness feature gate.", "items": { "allOf": [ { diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index 3ffd1e40976..4147850a8d4 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -23562,7 +23562,7 @@ func schema_k8sio_api_core_v1_PodSpec(ref common.ReferenceCallback) common.OpenA }, }, SchemaProps: spec.SchemaProps{ - Description: "SchedulingGates is an opaque list of values that if specified will block scheduling the pod. More info: https://git.k8s.io/enhancements/keps/sig-scheduling/3521-pod-scheduling-readiness.\n\nThis is an alpha-level feature enabled by PodSchedulingReadiness feature gate.", + Description: "SchedulingGates is an opaque list of values that if specified will block scheduling the pod. If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the scheduler will not attempt to schedule the pod.\n\nSchedulingGates can only be set at pod creation time, and be removed only afterwards.\n\nThis is a beta feature enabled by the PodSchedulingReadiness feature gate.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ diff --git a/staging/src/k8s.io/api/core/v1/generated.proto b/staging/src/k8s.io/api/core/v1/generated.proto index 57dea388da9..3fb25dba124 100644 --- a/staging/src/k8s.io/api/core/v1/generated.proto +++ b/staging/src/k8s.io/api/core/v1/generated.proto @@ -3808,14 +3808,19 @@ message PodSpec { optional bool hostUsers = 37; // SchedulingGates is an opaque list of values that if specified will block scheduling the pod. - // More info: https://git.k8s.io/enhancements/keps/sig-scheduling/3521-pod-scheduling-readiness. + // If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the + // scheduler will not attempt to schedule the pod. + // + // SchedulingGates can only be set at pod creation time, and be removed only afterwards. + // + // This is a beta feature enabled by the PodSchedulingReadiness feature gate. // - // This is an alpha-level feature enabled by PodSchedulingReadiness feature gate. - // +optional // +patchMergeKey=name // +patchStrategy=merge // +listType=map // +listMapKey=name + // +featureGate=PodSchedulingReadiness + // +optional repeated PodSchedulingGate schedulingGates = 38; // ResourceClaims defines which ResourceClaims must be allocated diff --git a/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go b/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go index 766b1cd7b2a..707ff2f6a64 100644 --- a/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go +++ b/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go @@ -1701,7 +1701,7 @@ var map_PodSpec = map[string]string{ "setHostnameAsFQDN": "If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. If a pod does not have FQDN, this has no effect. Default to false.", "os": "Specifies the OS of the containers in the pod. Some pod and container fields are restricted if this is set.\n\nIf the OS field is set to linux, the following fields must be unset: -securityContext.windowsOptions\n\nIf the OS field is set to windows, following fields must be unset: - spec.hostPID - spec.hostIPC - spec.hostUsers - spec.securityContext.seLinuxOptions - spec.securityContext.seccompProfile - spec.securityContext.fsGroup - spec.securityContext.fsGroupChangePolicy - spec.securityContext.sysctls - spec.shareProcessNamespace - spec.securityContext.runAsUser - spec.securityContext.runAsGroup - spec.securityContext.supplementalGroups - spec.containers[*].securityContext.seLinuxOptions - spec.containers[*].securityContext.seccompProfile - spec.containers[*].securityContext.capabilities - spec.containers[*].securityContext.readOnlyRootFilesystem - spec.containers[*].securityContext.privileged - spec.containers[*].securityContext.allowPrivilegeEscalation - spec.containers[*].securityContext.procMount - spec.containers[*].securityContext.runAsUser - spec.containers[*].securityContext.runAsGroup", "hostUsers": "Use the host's user namespace. Optional: Default to true. If set to true or not present, the pod will be run in the host user namespace, useful for when the pod needs a feature only available to the host user namespace, such as loading a kernel module with CAP_SYS_MODULE. When set to false, a new userns is created for the pod. Setting false is useful for mitigating container breakout vulnerabilities even allowing users to run their containers as root without actually having root privileges on the host. This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature.", - "schedulingGates": "SchedulingGates is an opaque list of values that if specified will block scheduling the pod. More info: https://git.k8s.io/enhancements/keps/sig-scheduling/3521-pod-scheduling-readiness.\n\nThis is an alpha-level feature enabled by PodSchedulingReadiness feature gate.", + "schedulingGates": "SchedulingGates is an opaque list of values that if specified will block scheduling the pod. If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the scheduler will not attempt to schedule the pod.\n\nSchedulingGates can only be set at pod creation time, and be removed only afterwards.\n\nThis is a beta feature enabled by the PodSchedulingReadiness feature gate.", "resourceClaims": "ResourceClaims defines which ResourceClaims must be allocated and reserved before the Pod is allowed to start. The resources will be made available to those containers which consume them by name.\n\nThis is an alpha field and requires enabling the DynamicResourceAllocation feature gate.\n\nThis field is immutable.", }