From 5c7f602f48a2840beb78b63dd8ee55a050e225be Mon Sep 17 00:00:00 2001 From: ravisantoshgudimetla Date: Mon, 4 Oct 2021 13:53:32 -0400 Subject: [PATCH] Make v1beta3 default --- cmd/kube-scheduler/app/options/configfile.go | 5 +- .../app/options/options_test.go | 400 ++++++++++++- cmd/kube-scheduler/app/server_test.go | 2 +- pkg/scheduler/apis/config/latest/latest.go | 6 +- pkg/scheduler/apis/config/scheme/scheme.go | 6 +- .../apis/config/scheme/scheme_test.go | 561 ++++++++++++++++++ pkg/scheduler/apis/config/testing/config.go | 13 + .../apis/config/testing/defaults/defaults.go | 137 +++++ .../apis/config/validation/validation.go | 5 + .../apis/config/validation/validation_test.go | 309 +++++++++- pkg/scheduler/scheduler.go | 4 +- 11 files changed, 1422 insertions(+), 26 deletions(-) diff --git a/cmd/kube-scheduler/app/options/configfile.go b/cmd/kube-scheduler/app/options/configfile.go index eda49fcfed9..8727c1c62f2 100644 --- a/cmd/kube-scheduler/app/options/configfile.go +++ b/cmd/kube-scheduler/app/options/configfile.go @@ -28,6 +28,7 @@ import ( "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/apis/config/scheme" configv1beta2 "k8s.io/kubernetes/pkg/scheduler/apis/config/v1beta2" + configv1beta3 "k8s.io/kubernetes/pkg/scheduler/apis/config/v1beta3" ) func loadConfigFromFile(file string) (*config.KubeSchedulerConfiguration, error) { @@ -68,8 +69,10 @@ func encodeConfig(cfg *config.KubeSchedulerConfiguration) (*bytes.Buffer, error) switch cfg.TypeMeta.APIVersion { case configv1beta2.SchemeGroupVersion.String(): encoder = scheme.Codecs.EncoderForVersion(info.Serializer, configv1beta2.SchemeGroupVersion) + case configv1beta3.SchemeGroupVersion.String(): + encoder = scheme.Codecs.EncoderForVersion(info.Serializer, configv1beta3.SchemeGroupVersion) default: - encoder = scheme.Codecs.EncoderForVersion(info.Serializer, configv1beta2.SchemeGroupVersion) + encoder = scheme.Codecs.EncoderForVersion(info.Serializer, configv1beta3.SchemeGroupVersion) } if err := encoder.Encode(cfg, buf); err != nil { return buf, err diff --git a/cmd/kube-scheduler/app/options/options_test.go b/cmd/kube-scheduler/app/options/options_test.go index 5a8b696f3ec..3683f99c3e2 100644 --- a/cmd/kube-scheduler/app/options/options_test.go +++ b/cmd/kube-scheduler/app/options/options_test.go @@ -36,8 +36,10 @@ import ( componentbaseconfig "k8s.io/component-base/config" "k8s.io/component-base/logs" "k8s.io/kube-scheduler/config/v1beta2" + "k8s.io/kube-scheduler/config/v1beta3" kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/apis/config/latest" + configtesting "k8s.io/kubernetes/pkg/scheduler/apis/config/testing" "k8s.io/kubernetes/pkg/scheduler/apis/config/testing/defaults" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/names" ) @@ -77,7 +79,7 @@ func TestSchedulerOptions(t *testing.T) { configFile := filepath.Join(tmpDir, "scheduler.yaml") configKubeconfig := filepath.Join(tmpDir, "config.kubeconfig") if err := ioutil.WriteFile(configFile, []byte(fmt.Sprintf(` -apiVersion: kubescheduler.config.k8s.io/v1beta2 +apiVersion: kubescheduler.config.k8s.io/v1beta3 kind: KubeSchedulerConfiguration clientConnection: kubeconfig: "%s" @@ -118,6 +120,17 @@ leaderElection: t.Fatal(err) } + v1beta2VersionConfig := filepath.Join(tmpDir, "scheduler_v1beta2_api_version.yaml") + if err := ioutil.WriteFile(v1beta2VersionConfig, []byte(fmt.Sprintf(` +apiVersion: kubescheduler.config.k8s.io/v1beta2 +kind: KubeSchedulerConfiguration +clientConnection: + kubeconfig: "%s" +leaderElection: + leaderElect: true`, configKubeconfig)), os.FileMode(0600)); err != nil { + t.Fatal(err) + } + unknownVersionConfig := filepath.Join(tmpDir, "scheduler_invalid_wrong_api_version.yaml") if err := ioutil.WriteFile(unknownVersionConfig, []byte(fmt.Sprintf(` apiVersion: kubescheduler.config.k8s.io/unknown @@ -190,6 +203,37 @@ users: // plugin config pluginConfigFile := filepath.Join(tmpDir, "plugin.yaml") if err := ioutil.WriteFile(pluginConfigFile, []byte(fmt.Sprintf(` +apiVersion: kubescheduler.config.k8s.io/v1beta3 +kind: KubeSchedulerConfiguration +clientConnection: + kubeconfig: "%s" +profiles: +- plugins: + reserve: + enabled: + - name: foo + - name: bar + disabled: + - name: VolumeBinding + preBind: + enabled: + - name: foo + disabled: + - name: VolumeBinding + pluginConfig: + - name: InterPodAffinity + args: + hardPodAffinityWeight: 2 + - name: foo + args: + bar: baz +`, configKubeconfig)), os.FileMode(0600)); err != nil { + t.Fatal(err) + } + + // v1beta2 plugin config + v1beta2PluginConfigFile := filepath.Join(tmpDir, "v1beta2_plugin.yaml") + if err := ioutil.WriteFile(v1beta2PluginConfigFile, []byte(fmt.Sprintf(` apiVersion: kubescheduler.config.k8s.io/v1beta2 kind: KubeSchedulerConfiguration clientConnection: @@ -221,6 +265,33 @@ profiles: // multiple profiles config multiProfilesConfig := filepath.Join(tmpDir, "multi-profiles.yaml") if err := ioutil.WriteFile(multiProfilesConfig, []byte(fmt.Sprintf(` +apiVersion: kubescheduler.config.k8s.io/v1beta3 +kind: KubeSchedulerConfiguration +clientConnection: + kubeconfig: "%s" +profiles: +- schedulerName: "foo-profile" + plugins: + reserve: + enabled: + - name: foo + - name: VolumeBinding + disabled: + - name: VolumeBinding +- schedulerName: "bar-profile" + plugins: + preBind: + disabled: + - name: VolumeBinding + pluginConfig: + - name: foo +`, configKubeconfig)), os.FileMode(0600)); err != nil { + t.Fatal(err) + } + + // multiple profiles config + v1beta2MultiProfilesConfig := filepath.Join(tmpDir, "v1beta2_multi-profiles.yaml") + if err := ioutil.WriteFile(v1beta2MultiProfilesConfig, []byte(fmt.Sprintf(` apiVersion: kubescheduler.config.k8s.io/v1beta2 kind: KubeSchedulerConfiguration clientConnection: @@ -266,14 +337,83 @@ profiles: checkErrFn func(err error) bool }{ { - name: "v1beta2 config file", + name: "v1beta3 config file", options: &Options{ ConfigFile: configFile, ComponentConfig: func() kubeschedulerconfig.KubeSchedulerConfiguration { - cfg, err := latest.Default() - if err != nil { - t.Fatal(err) - } + cfg := configtesting.V1beta3ToInternalWithDefaults(t, v1beta3.KubeSchedulerConfiguration{}) + return *cfg + }(), + SecureServing: (&apiserveroptions.SecureServingOptions{ + ServerCert: apiserveroptions.GeneratableKeyCert{ + CertDirectory: "/a/b/c", + PairName: "kube-scheduler", + }, + HTTP2MaxStreamsPerConnection: 47, + }).WithLoopback(), + Authentication: &apiserveroptions.DelegatingAuthenticationOptions{ + CacheTTL: 10 * time.Second, + ClientCert: apiserveroptions.ClientCertAuthenticationOptions{}, + RequestHeader: apiserveroptions.RequestHeaderAuthenticationOptions{ + UsernameHeaders: []string{"x-remote-user"}, + GroupHeaders: []string{"x-remote-group"}, + ExtraHeaderPrefixes: []string{"x-remote-extra-"}, + }, + RemoteKubeConfigFileOptional: true, + }, + Authorization: &apiserveroptions.DelegatingAuthorizationOptions{ + AllowCacheTTL: 10 * time.Second, + DenyCacheTTL: 10 * time.Second, + RemoteKubeConfigFileOptional: true, + AlwaysAllowPaths: []string{"/healthz", "/readyz", "/livez"}, // note: this does not match /healthz/ or /healthz/* + AlwaysAllowGroups: []string{"system:masters"}, + }, + Logs: logs.NewOptions(), + }, + expectedUsername: "config", + expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: v1beta3.SchemeGroupVersion.String(), + }, + Parallelism: 16, + DebuggingConfiguration: componentbaseconfig.DebuggingConfiguration{ + EnableProfiling: true, + EnableContentionProfiling: true, + }, + LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ + LeaderElect: true, + LeaseDuration: metav1.Duration{Duration: 15 * time.Second}, + RenewDeadline: metav1.Duration{Duration: 10 * time.Second}, + RetryPeriod: metav1.Duration{Duration: 2 * time.Second}, + ResourceLock: "leases", + ResourceNamespace: "kube-system", + ResourceName: "kube-scheduler", + }, + ClientConnection: componentbaseconfig.ClientConnectionConfiguration{ + Kubeconfig: configKubeconfig, + QPS: 50, + Burst: 100, + ContentType: "application/vnd.kubernetes.protobuf", + }, + PercentageOfNodesToScore: defaultPercentageOfNodesToScore, + PodInitialBackoffSeconds: defaultPodInitialBackoffSeconds, + PodMaxBackoffSeconds: defaultPodMaxBackoffSeconds, + Profiles: []kubeschedulerconfig.KubeSchedulerProfile{ + { + SchedulerName: "default-scheduler", + Plugins: defaults.PluginsV1beta3, + PluginConfig: defaults.PluginConfigsV1beta3, + }, + }, + }, + }, + + { + name: "v1beta2 config file", + options: &Options{ + ConfigFile: v1beta2VersionConfig, + ComponentConfig: func() kubeschedulerconfig.KubeSchedulerConfiguration { + cfg := configtesting.V1beta2ToInternalWithDefaults(t, v1beta2.KubeSchedulerConfiguration{}) return *cfg }(), SecureServing: (&apiserveroptions.SecureServingOptions{ @@ -407,7 +547,7 @@ profiles: expectedUsername: "flag", expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ TypeMeta: metav1.TypeMeta{ - APIVersion: v1beta2.SchemeGroupVersion.String(), + APIVersion: v1beta3.SchemeGroupVersion.String(), }, Parallelism: 16, DebuggingConfiguration: componentbaseconfig.DebuggingConfiguration{ @@ -435,8 +575,8 @@ profiles: Profiles: []kubeschedulerconfig.KubeSchedulerProfile{ { SchedulerName: "default-scheduler", - Plugins: defaults.PluginsV1beta2, - PluginConfig: defaults.PluginConfigsV1beta2, + Plugins: defaults.PluginsV1beta3, + PluginConfig: defaults.PluginConfigsV1beta3, }, }, }, @@ -477,7 +617,7 @@ profiles: }, expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ TypeMeta: metav1.TypeMeta{ - APIVersion: v1beta2.SchemeGroupVersion.String(), + APIVersion: v1beta3.SchemeGroupVersion.String(), }, Parallelism: 16, DebuggingConfiguration: componentbaseconfig.DebuggingConfiguration{ @@ -505,8 +645,8 @@ profiles: Profiles: []kubeschedulerconfig.KubeSchedulerProfile{ { SchedulerName: "default-scheduler", - Plugins: defaults.PluginsV1beta2, - PluginConfig: defaults.PluginConfigsV1beta2, + Plugins: defaults.PluginsV1beta3, + PluginConfig: defaults.PluginConfigsV1beta3, }, }, }, @@ -519,6 +659,120 @@ profiles: Logs: logs.NewOptions(), }, expectedUsername: "config", + expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: v1beta3.SchemeGroupVersion.String(), + }, + Parallelism: 16, + DebuggingConfiguration: componentbaseconfig.DebuggingConfiguration{ + EnableProfiling: true, + EnableContentionProfiling: true, + }, + LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ + LeaderElect: true, + LeaseDuration: metav1.Duration{Duration: 15 * time.Second}, + RenewDeadline: metav1.Duration{Duration: 10 * time.Second}, + RetryPeriod: metav1.Duration{Duration: 2 * time.Second}, + ResourceLock: "leases", + ResourceNamespace: "kube-system", + ResourceName: "kube-scheduler", + }, + ClientConnection: componentbaseconfig.ClientConnectionConfiguration{ + Kubeconfig: configKubeconfig, + QPS: 50, + Burst: 100, + ContentType: "application/vnd.kubernetes.protobuf", + }, + PercentageOfNodesToScore: defaultPercentageOfNodesToScore, + PodInitialBackoffSeconds: defaultPodInitialBackoffSeconds, + PodMaxBackoffSeconds: defaultPodMaxBackoffSeconds, + Profiles: []kubeschedulerconfig.KubeSchedulerProfile{ + { + SchedulerName: "default-scheduler", + Plugins: &kubeschedulerconfig.Plugins{ + QueueSort: defaults.PluginsV1beta3.QueueSort, + PreFilter: defaults.PluginsV1beta3.PreFilter, + Filter: defaults.PluginsV1beta3.Filter, + PostFilter: defaults.PluginsV1beta3.PostFilter, + PreScore: defaults.PluginsV1beta3.PreScore, + Score: defaults.PluginsV1beta3.Score, + Reserve: kubeschedulerconfig.PluginSet{ + Enabled: []kubeschedulerconfig.Plugin{ + {Name: "foo"}, + {Name: "bar"}, + }, + }, + PreBind: kubeschedulerconfig.PluginSet{ + Enabled: []kubeschedulerconfig.Plugin{ + {Name: "foo"}, + }, + }, + Bind: defaults.PluginsV1beta3.Bind, + }, + PluginConfig: []kubeschedulerconfig.PluginConfig{ + { + Name: "InterPodAffinity", + Args: &kubeschedulerconfig.InterPodAffinityArgs{ + HardPodAffinityWeight: 2, + }, + }, + { + Name: "foo", + Args: &runtime.Unknown{ + Raw: []byte(`{"bar":"baz"}`), + ContentType: "application/json", + }, + }, + { + Name: "DefaultPreemption", + Args: &kubeschedulerconfig.DefaultPreemptionArgs{ + MinCandidateNodesPercentage: 10, + MinCandidateNodesAbsolute: 100, + }, + }, + { + Name: "NodeAffinity", + Args: &kubeschedulerconfig.NodeAffinityArgs{}, + }, + { + Name: "NodeResourcesBalancedAllocation", + Args: &kubeschedulerconfig.NodeResourcesBalancedAllocationArgs{ + Resources: []kubeschedulerconfig.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}}, + }, + }, + { + Name: "NodeResourcesFit", + Args: &kubeschedulerconfig.NodeResourcesFitArgs{ + ScoringStrategy: &kubeschedulerconfig.ScoringStrategy{ + Type: kubeschedulerconfig.LeastAllocated, + Resources: []kubeschedulerconfig.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}}, + }, + }, + }, + { + Name: "PodTopologySpread", + Args: &kubeschedulerconfig.PodTopologySpreadArgs{ + DefaultingType: kubeschedulerconfig.SystemDefaulting, + }, + }, + { + Name: "VolumeBinding", + Args: &kubeschedulerconfig.VolumeBindingArgs{ + BindTimeoutSeconds: 600, + }, + }, + }, + }, + }, + }, + }, + { + name: "v1beta2 plugin config", + options: &Options{ + ConfigFile: v1beta2PluginConfigFile, + Logs: logs.NewOptions(), + }, + expectedUsername: "config", expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ TypeMeta: metav1.TypeMeta{ APIVersion: v1beta2.SchemeGroupVersion.String(), @@ -633,6 +887,126 @@ profiles: Logs: logs.NewOptions(), }, expectedUsername: "config", + expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: v1beta3.SchemeGroupVersion.String(), + }, + Parallelism: 16, + DebuggingConfiguration: componentbaseconfig.DebuggingConfiguration{ + EnableProfiling: true, + EnableContentionProfiling: true, + }, + LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ + LeaderElect: true, + LeaseDuration: metav1.Duration{Duration: 15 * time.Second}, + RenewDeadline: metav1.Duration{Duration: 10 * time.Second}, + RetryPeriod: metav1.Duration{Duration: 2 * time.Second}, + ResourceLock: "leases", + ResourceNamespace: "kube-system", + ResourceName: "kube-scheduler", + }, + ClientConnection: componentbaseconfig.ClientConnectionConfiguration{ + Kubeconfig: configKubeconfig, + QPS: 50, + Burst: 100, + ContentType: "application/vnd.kubernetes.protobuf", + }, + PercentageOfNodesToScore: defaultPercentageOfNodesToScore, + PodInitialBackoffSeconds: defaultPodInitialBackoffSeconds, + PodMaxBackoffSeconds: defaultPodMaxBackoffSeconds, + Profiles: []kubeschedulerconfig.KubeSchedulerProfile{ + { + SchedulerName: "foo-profile", + Plugins: &kubeschedulerconfig.Plugins{ + QueueSort: defaults.PluginsV1beta3.QueueSort, + PreFilter: defaults.PluginsV1beta3.PreFilter, + Filter: defaults.PluginsV1beta3.Filter, + PostFilter: defaults.PluginsV1beta3.PostFilter, + PreScore: defaults.PluginsV1beta3.PreScore, + Score: defaults.PluginsV1beta3.Score, + Bind: defaults.PluginsV1beta3.Bind, + PreBind: defaults.PluginsV1beta3.PreBind, + Reserve: kubeschedulerconfig.PluginSet{ + Enabled: []kubeschedulerconfig.Plugin{ + {Name: "foo"}, + {Name: names.VolumeBinding}, + }, + }, + }, + PluginConfig: defaults.PluginConfigsV1beta3, + }, + { + SchedulerName: "bar-profile", + Plugins: &kubeschedulerconfig.Plugins{ + QueueSort: defaults.PluginsV1beta3.QueueSort, + PreFilter: defaults.PluginsV1beta3.PreFilter, + Filter: defaults.PluginsV1beta3.Filter, + PostFilter: defaults.PluginsV1beta3.PostFilter, + PreScore: defaults.PluginsV1beta3.PreScore, + Score: defaults.PluginsV1beta3.Score, + Bind: defaults.PluginsV1beta3.Bind, + Reserve: defaults.PluginsV1beta3.Reserve, + }, + PluginConfig: []kubeschedulerconfig.PluginConfig{ + { + Name: "foo", + }, + { + Name: "DefaultPreemption", + Args: &kubeschedulerconfig.DefaultPreemptionArgs{ + MinCandidateNodesPercentage: 10, + MinCandidateNodesAbsolute: 100, + }, + }, + { + Name: "InterPodAffinity", + Args: &kubeschedulerconfig.InterPodAffinityArgs{ + HardPodAffinityWeight: 1, + }, + }, + { + Name: "NodeAffinity", + Args: &kubeschedulerconfig.NodeAffinityArgs{}, + }, + { + Name: "NodeResourcesBalancedAllocation", + Args: &kubeschedulerconfig.NodeResourcesBalancedAllocationArgs{ + Resources: []kubeschedulerconfig.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}}, + }, + }, + { + Name: "NodeResourcesFit", + Args: &kubeschedulerconfig.NodeResourcesFitArgs{ + ScoringStrategy: &kubeschedulerconfig.ScoringStrategy{ + Type: kubeschedulerconfig.LeastAllocated, + Resources: []kubeschedulerconfig.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}}, + }, + }, + }, + { + Name: "PodTopologySpread", + Args: &kubeschedulerconfig.PodTopologySpreadArgs{ + DefaultingType: kubeschedulerconfig.SystemDefaulting, + }, + }, + { + Name: "VolumeBinding", + Args: &kubeschedulerconfig.VolumeBindingArgs{ + BindTimeoutSeconds: 600, + }, + }, + }, + }, + }, + }, + }, + { + name: "v1beta2 multiple profiles", + options: &Options{ + ConfigFile: v1beta2MultiProfilesConfig, + Logs: logs.NewOptions(), + }, + expectedUsername: "config", expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ TypeMeta: metav1.TypeMeta{ APIVersion: v1beta2.SchemeGroupVersion.String(), @@ -758,7 +1132,7 @@ profiles: expectedUsername: "config", expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ TypeMeta: metav1.TypeMeta{ - APIVersion: v1beta2.SchemeGroupVersion.String(), + APIVersion: v1beta3.SchemeGroupVersion.String(), }, Parallelism: 16, DebuggingConfiguration: componentbaseconfig.DebuggingConfiguration{ diff --git a/cmd/kube-scheduler/app/server_test.go b/cmd/kube-scheduler/app/server_test.go index 0b021942026..5d335561150 100644 --- a/cmd/kube-scheduler/app/server_test.go +++ b/cmd/kube-scheduler/app/server_test.go @@ -163,7 +163,7 @@ profiles: "--kubeconfig", configKubeconfig, }, wantPlugins: map[string]*config.Plugins{ - "default-scheduler": defaults.PluginsV1beta2, + "default-scheduler": defaults.PluginsV1beta3, }, }, { diff --git a/pkg/scheduler/apis/config/latest/latest.go b/pkg/scheduler/apis/config/latest/latest.go index b9fb3ff186b..1833b873fe9 100644 --- a/pkg/scheduler/apis/config/latest/latest.go +++ b/pkg/scheduler/apis/config/latest/latest.go @@ -18,7 +18,7 @@ package latest import ( "k8s.io/component-base/config/v1alpha1" - "k8s.io/kube-scheduler/config/v1beta2" + "k8s.io/kube-scheduler/config/v1beta3" "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/apis/config/scheme" ) @@ -26,7 +26,7 @@ import ( // Default creates a default configuration of the latest versioned type. // This function needs to be updated whenever we bump the scheduler's component config version. func Default() (*config.KubeSchedulerConfiguration, error) { - versionedCfg := v1beta2.KubeSchedulerConfiguration{} + versionedCfg := v1beta3.KubeSchedulerConfiguration{} versionedCfg.DebuggingConfiguration = *v1alpha1.NewRecommendedDebuggingConfiguration() scheme.Scheme.Default(&versionedCfg) @@ -38,6 +38,6 @@ func Default() (*config.KubeSchedulerConfiguration, error) { // because the field will be cleared later by API machinery during // conversion. See KubeSchedulerConfiguration internal type definition for // more details. - cfg.TypeMeta.APIVersion = v1beta2.SchemeGroupVersion.String() + cfg.TypeMeta.APIVersion = v1beta3.SchemeGroupVersion.String() return &cfg, nil } diff --git a/pkg/scheduler/apis/config/scheme/scheme.go b/pkg/scheduler/apis/config/scheme/scheme.go index 99806b7e9bf..ac9effee02d 100644 --- a/pkg/scheduler/apis/config/scheme/scheme.go +++ b/pkg/scheduler/apis/config/scheme/scheme.go @@ -23,6 +23,7 @@ import ( config "k8s.io/kubernetes/pkg/scheduler/apis/config" configv1 "k8s.io/kubernetes/pkg/scheduler/apis/config/v1" configv1beta2 "k8s.io/kubernetes/pkg/scheduler/apis/config/v1beta2" + configv1beta3 "k8s.io/kubernetes/pkg/scheduler/apis/config/v1beta3" ) var ( @@ -42,5 +43,8 @@ func AddToScheme(scheme *runtime.Scheme) { utilruntime.Must(config.AddToScheme(scheme)) utilruntime.Must(configv1.AddToScheme(scheme)) utilruntime.Must(configv1beta2.AddToScheme(scheme)) - utilruntime.Must(scheme.SetVersionPriority(configv1beta2.SchemeGroupVersion)) + utilruntime.Must(configv1beta3.AddToScheme(scheme)) + utilruntime.Must(scheme.SetVersionPriority( + configv1beta3.SchemeGroupVersion, + configv1beta2.SchemeGroupVersion)) } diff --git a/pkg/scheduler/apis/config/scheme/scheme_test.go b/pkg/scheduler/apis/config/scheme/scheme_test.go index 5a70eb6262d..fca416ede0c 100644 --- a/pkg/scheduler/apis/config/scheme/scheme_test.go +++ b/pkg/scheduler/apis/config/scheme/scheme_test.go @@ -25,6 +25,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/kube-scheduler/config/v1beta2" + "k8s.io/kube-scheduler/config/v1beta3" "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/apis/config/testing/defaults" "k8s.io/utils/pointer" @@ -381,6 +382,347 @@ profiles: }, }, }, + // v1beta3 tests + { + name: "v1beta3 all plugin args in default profile", + data: []byte(` +apiVersion: kubescheduler.config.k8s.io/v1beta3 +kind: KubeSchedulerConfiguration +profiles: +- pluginConfig: + - name: DefaultPreemption + args: + minCandidateNodesPercentage: 50 + minCandidateNodesAbsolute: 500 + - name: InterPodAffinity + args: + hardPodAffinityWeight: 5 + - name: NodeResourcesFit + args: + ignoredResources: ["foo"] + - name: PodTopologySpread + args: + defaultConstraints: + - maxSkew: 1 + topologyKey: zone + whenUnsatisfiable: ScheduleAnyway + - name: VolumeBinding + args: + bindTimeoutSeconds: 300 + - name: NodeAffinity + args: + addedAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: foo + operator: In + values: ["bar"] + - name: NodeResourcesBalancedAllocation + args: + resources: + - name: cpu # default weight(1) will be set. + - name: memory # weight 0 will be replaced by 1. + weight: 0 + - name: scalar0 + weight: 1 + - name: scalar1 # default weight(1) will be set for scalar1 + - name: scalar2 # weight 0 will be replaced by 1. + weight: 0 + - name: scalar3 + weight: 2 +`), + wantProfiles: []config.KubeSchedulerProfile{ + { + SchedulerName: "default-scheduler", + Plugins: defaults.PluginsV1beta3, + PluginConfig: []config.PluginConfig{ + { + Name: "DefaultPreemption", + Args: &config.DefaultPreemptionArgs{MinCandidateNodesPercentage: 50, MinCandidateNodesAbsolute: 500}, + }, + { + Name: "InterPodAffinity", + Args: &config.InterPodAffinityArgs{HardPodAffinityWeight: 5}, + }, + { + Name: "NodeResourcesFit", + Args: &config.NodeResourcesFitArgs{ + IgnoredResources: []string{"foo"}, + ScoringStrategy: &config.ScoringStrategy{ + Type: config.LeastAllocated, + Resources: []config.ResourceSpec{ + {Name: "cpu", Weight: 1}, + {Name: "memory", Weight: 1}, + }, + }, + }, + }, + { + Name: "PodTopologySpread", + Args: &config.PodTopologySpreadArgs{ + DefaultConstraints: []corev1.TopologySpreadConstraint{ + {MaxSkew: 1, TopologyKey: "zone", WhenUnsatisfiable: corev1.ScheduleAnyway}, + }, + DefaultingType: config.SystemDefaulting, + }, + }, + { + Name: "VolumeBinding", + Args: &config.VolumeBindingArgs{ + BindTimeoutSeconds: 300, + }, + }, + { + Name: "NodeAffinity", + Args: &config.NodeAffinityArgs{ + AddedAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "foo", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"bar"}, + }, + }, + }, + }, + }, + }, + }, + }, + { + Name: "NodeResourcesBalancedAllocation", + Args: &config.NodeResourcesBalancedAllocationArgs{ + Resources: []config.ResourceSpec{ + {Name: "cpu", Weight: 1}, + {Name: "memory", Weight: 1}, + {Name: "scalar0", Weight: 1}, + {Name: "scalar1", Weight: 1}, + {Name: "scalar2", Weight: 1}, + {Name: "scalar3", Weight: 2}}, + }, + }, + }, + }, + }, + }, + { + name: "v1beta3 plugins can include version and kind", + data: []byte(` +apiVersion: kubescheduler.config.k8s.io/v1beta3 +kind: KubeSchedulerConfiguration +profiles: +- pluginConfig: + - name: DefaultPreemption + args: + apiVersion: kubescheduler.config.k8s.io/v1beta3 + kind: DefaultPreemptionArgs + minCandidateNodesPercentage: 50 +`), + wantProfiles: []config.KubeSchedulerProfile{ + { + SchedulerName: "default-scheduler", + Plugins: defaults.PluginsV1beta3, + PluginConfig: []config.PluginConfig{ + { + Name: "DefaultPreemption", + Args: &config.DefaultPreemptionArgs{MinCandidateNodesPercentage: 50, MinCandidateNodesAbsolute: 100}, + }, + { + Name: "InterPodAffinity", + Args: &config.InterPodAffinityArgs{ + HardPodAffinityWeight: 1, + }, + }, + { + Name: "NodeAffinity", + Args: &config.NodeAffinityArgs{}, + }, + { + Name: "NodeResourcesBalancedAllocation", + Args: &config.NodeResourcesBalancedAllocationArgs{ + Resources: []config.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}}, + }, + }, + { + Name: "NodeResourcesFit", + Args: &config.NodeResourcesFitArgs{ + ScoringStrategy: &config.ScoringStrategy{ + Type: config.LeastAllocated, + Resources: []config.ResourceSpec{ + {Name: "cpu", Weight: 1}, + {Name: "memory", Weight: 1}, + }, + }, + }, + }, + { + Name: "PodTopologySpread", + Args: &config.PodTopologySpreadArgs{ + DefaultingType: config.SystemDefaulting, + }, + }, + { + Name: "VolumeBinding", + Args: &config.VolumeBindingArgs{ + BindTimeoutSeconds: 600, + }, + }, + }, + }, + }, + }, + { + name: "plugin group and kind should match the type", + data: []byte(` +apiVersion: kubescheduler.config.k8s.io/v1beta3 +kind: KubeSchedulerConfiguration +profiles: +- pluginConfig: + - name: DefaultPreemption + args: + apiVersion: kubescheduler.config.k8s.io/v1beta3 + kind: InterPodAffinityArgs +`), + wantErr: `decoding .profiles[0].pluginConfig[0]: args for plugin DefaultPreemption were not of type DefaultPreemptionArgs.kubescheduler.config.k8s.io, got InterPodAffinityArgs.kubescheduler.config.k8s.io`, + }, + { + name: "v1beta3 NodResourcesFitArgs shape encoding is strict", + data: []byte(` +apiVersion: kubescheduler.config.k8s.io/v1beta3 +kind: KubeSchedulerConfiguration +profiles: +- pluginConfig: + - name: NodeResourcesFit + args: + scoringStrategy: + requestedToCapacityRatio: + shape: + - Score: 2 + Utilization: 1 +`), + wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoder error for {"scoringStrategy":{"requestedToCapacityRatio":{"shape":[{"Score":2,"Utilization":1}]}}}: v1beta3.NodeResourcesFitArgs.ScoringStrategy: v1beta3.ScoringStrategy.RequestedToCapacityRatio: v1beta3.RequestedToCapacityRatioParam.Shape: []v1beta3.UtilizationShapePoint: v1beta3.UtilizationShapePoint.ReadObject: found unknown field: Score, error found in #10 byte of ...|:[{"Score":2,"Utiliz|..., bigger context ...|gy":{"requestedToCapacityRatio":{"shape":[{"Score":2,"Utilization":1}]}}}|...`, + }, + { + name: "v1beta3 NodeResourcesFitArgs resources encoding is strict", + data: []byte(` +apiVersion: kubescheduler.config.k8s.io/v1beta3 +kind: KubeSchedulerConfiguration +profiles: +- pluginConfig: + - name: NodeResourcesFit + args: + scoringStrategy: + resources: + - Name: cpu + Weight: 1 +`), + wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoder error for {"scoringStrategy":{"resources":[{"Name":"cpu","Weight":1}]}}: v1beta3.NodeResourcesFitArgs.ScoringStrategy: v1beta3.ScoringStrategy.Resources: []v1beta3.ResourceSpec: v1beta3.ResourceSpec.ReadObject: found unknown field: Name, error found in #10 byte of ...|":[{"Name":"cpu","We|..., bigger context ...|{"scoringStrategy":{"resources":[{"Name":"cpu","Weight":1}]}}|...`, + }, + { + name: "out-of-tree plugin args", + data: []byte(` +apiVersion: kubescheduler.config.k8s.io/v1beta3 +kind: KubeSchedulerConfiguration +profiles: +- pluginConfig: + - name: OutOfTreePlugin + args: + foo: bar +`), + wantProfiles: []config.KubeSchedulerProfile{ + { + SchedulerName: "default-scheduler", + Plugins: defaults.PluginsV1beta3, + PluginConfig: append([]config.PluginConfig{ + { + Name: "OutOfTreePlugin", + Args: &runtime.Unknown{ + ContentType: "application/json", + Raw: []byte(`{"foo":"bar"}`), + }, + }, + }, defaults.PluginConfigsV1beta3...), + }, + }, + }, + { + name: "empty and no plugin args", + data: []byte(` +apiVersion: kubescheduler.config.k8s.io/v1beta3 +kind: KubeSchedulerConfiguration +profiles: +- pluginConfig: + - name: DefaultPreemption + args: + - name: InterPodAffinity + args: + - name: NodeResourcesFit + - name: OutOfTreePlugin + args: + - name: VolumeBinding + args: + - name: PodTopologySpread + - name: NodeAffinity + - name: NodeResourcesBalancedAllocation +`), + wantProfiles: []config.KubeSchedulerProfile{ + { + SchedulerName: "default-scheduler", + Plugins: defaults.PluginsV1beta3, + PluginConfig: []config.PluginConfig{ + { + Name: "DefaultPreemption", + Args: &config.DefaultPreemptionArgs{MinCandidateNodesPercentage: 10, MinCandidateNodesAbsolute: 100}, + }, + { + Name: "InterPodAffinity", + Args: &config.InterPodAffinityArgs{ + HardPodAffinityWeight: 1, + }, + }, + { + Name: "NodeResourcesFit", + Args: &config.NodeResourcesFitArgs{ + ScoringStrategy: &config.ScoringStrategy{ + Type: config.LeastAllocated, + Resources: []config.ResourceSpec{ + {Name: "cpu", Weight: 1}, + {Name: "memory", Weight: 1}, + }, + }, + }, + }, + {Name: "OutOfTreePlugin"}, + { + Name: "VolumeBinding", + Args: &config.VolumeBindingArgs{ + BindTimeoutSeconds: 600, + }, + }, + { + Name: "PodTopologySpread", + Args: &config.PodTopologySpreadArgs{ + DefaultingType: config.SystemDefaulting, + }, + }, + { + Name: "NodeAffinity", + Args: &config.NodeAffinityArgs{}, + }, + { + Name: "NodeResourcesBalancedAllocation", + Args: &config.NodeResourcesBalancedAllocationArgs{ + Resources: []config.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}}, + }, + }, + }, + }, + }, + }, } decoder := Codecs.UniversalDecoder() for _, tt := range testCases { @@ -632,6 +974,225 @@ profiles: foo: bar name: OutOfTreePlugin schedulerName: "" +`, + }, + //v1beta3 tests + { + name: "v1beta3 in-tree and out-of-tree plugins", + version: v1beta3.SchemeGroupVersion, + obj: &v1beta3.KubeSchedulerConfiguration{ + Profiles: []v1beta3.KubeSchedulerProfile{ + { + PluginConfig: []v1beta3.PluginConfig{ + { + Name: "InterPodAffinity", + Args: runtime.RawExtension{ + Object: &v1beta3.InterPodAffinityArgs{ + HardPodAffinityWeight: pointer.Int32Ptr(5), + }, + }, + }, + { + Name: "VolumeBinding", + Args: runtime.RawExtension{ + Object: &v1beta2.VolumeBindingArgs{ + BindTimeoutSeconds: pointer.Int64Ptr(300), + Shape: []v1beta2.UtilizationShapePoint{ + { + Utilization: 0, + Score: 0, + }, + { + Utilization: 100, + Score: 10, + }, + }, + }, + }, + }, + { + Name: "NodeResourcesFit", + Args: runtime.RawExtension{ + Object: &v1beta3.NodeResourcesFitArgs{ + ScoringStrategy: &v1beta3.ScoringStrategy{ + Type: v1beta3.RequestedToCapacityRatio, + Resources: []v1beta3.ResourceSpec{{Name: "cpu", Weight: 1}}, + RequestedToCapacityRatio: &v1beta3.RequestedToCapacityRatioParam{ + Shape: []v1beta3.UtilizationShapePoint{ + {Utilization: 1, Score: 2}, + }, + }, + }, + }, + }, + }, + { + Name: "PodTopologySpread", + Args: runtime.RawExtension{ + Object: &v1beta3.PodTopologySpreadArgs{ + DefaultConstraints: []corev1.TopologySpreadConstraint{}, + }, + }, + }, + { + Name: "OutOfTreePlugin", + Args: runtime.RawExtension{ + Raw: []byte(`{"foo":"bar"}`), + }, + }, + }, + }, + }, + }, + want: `apiVersion: kubescheduler.config.k8s.io/v1beta3 +clientConnection: + acceptContentTypes: "" + burst: 0 + contentType: "" + kubeconfig: "" + qps: 0 +kind: KubeSchedulerConfiguration +leaderElection: + leaderElect: null + leaseDuration: 0s + renewDeadline: 0s + resourceLock: "" + resourceName: "" + resourceNamespace: "" + retryPeriod: 0s +profiles: +- pluginConfig: + - args: + apiVersion: kubescheduler.config.k8s.io/v1beta3 + hardPodAffinityWeight: 5 + kind: InterPodAffinityArgs + name: InterPodAffinity + - args: + apiVersion: kubescheduler.config.k8s.io/v1beta3 + bindTimeoutSeconds: 300 + kind: VolumeBindingArgs + shape: + - score: 0 + utilization: 0 + - score: 10 + utilization: 100 + name: VolumeBinding + - args: + apiVersion: kubescheduler.config.k8s.io/v1beta3 + kind: NodeResourcesFitArgs + scoringStrategy: + requestedToCapacityRatio: + shape: + - score: 2 + utilization: 1 + resources: + - name: cpu + weight: 1 + type: RequestedToCapacityRatio + name: NodeResourcesFit + - args: + apiVersion: kubescheduler.config.k8s.io/v1beta3 + kind: PodTopologySpreadArgs + name: PodTopologySpread + - args: + foo: bar + name: OutOfTreePlugin +`, + }, + { + name: "v1beta3 in-tree and out-of-tree plugins from internal", + version: v1beta3.SchemeGroupVersion, + obj: &config.KubeSchedulerConfiguration{ + Parallelism: 8, + Profiles: []config.KubeSchedulerProfile{ + { + PluginConfig: []config.PluginConfig{ + { + Name: "InterPodAffinity", + Args: &config.InterPodAffinityArgs{ + HardPodAffinityWeight: 5, + }, + }, + { + Name: "NodeResourcesFit", + Args: &config.NodeResourcesFitArgs{ + ScoringStrategy: &config.ScoringStrategy{ + Type: config.LeastAllocated, + Resources: []config.ResourceSpec{{Name: "cpu", Weight: 1}}, + }, + }, + }, + { + Name: "VolumeBinding", + Args: &config.VolumeBindingArgs{ + BindTimeoutSeconds: 300, + }, + }, + { + Name: "PodTopologySpread", + Args: &config.PodTopologySpreadArgs{}, + }, + { + Name: "OutOfTreePlugin", + Args: &runtime.Unknown{ + Raw: []byte(`{"foo":"bar"}`), + }, + }, + }, + }, + }, + }, + want: `apiVersion: kubescheduler.config.k8s.io/v1beta3 +clientConnection: + acceptContentTypes: "" + burst: 0 + contentType: "" + kubeconfig: "" + qps: 0 +enableContentionProfiling: false +enableProfiling: false +kind: KubeSchedulerConfiguration +leaderElection: + leaderElect: false + leaseDuration: 0s + renewDeadline: 0s + resourceLock: "" + resourceName: "" + resourceNamespace: "" + retryPeriod: 0s +parallelism: 8 +percentageOfNodesToScore: 0 +podInitialBackoffSeconds: 0 +podMaxBackoffSeconds: 0 +profiles: +- pluginConfig: + - args: + apiVersion: kubescheduler.config.k8s.io/v1beta3 + hardPodAffinityWeight: 5 + kind: InterPodAffinityArgs + name: InterPodAffinity + - args: + apiVersion: kubescheduler.config.k8s.io/v1beta3 + kind: NodeResourcesFitArgs + scoringStrategy: + resources: + - name: cpu + weight: 1 + type: LeastAllocated + name: NodeResourcesFit + - args: + apiVersion: kubescheduler.config.k8s.io/v1beta3 + bindTimeoutSeconds: 300 + kind: VolumeBindingArgs + name: VolumeBinding + - args: + apiVersion: kubescheduler.config.k8s.io/v1beta3 + kind: PodTopologySpreadArgs + name: PodTopologySpread + - args: + foo: bar + name: OutOfTreePlugin + schedulerName: "" `, }, } diff --git a/pkg/scheduler/apis/config/testing/config.go b/pkg/scheduler/apis/config/testing/config.go index 6b23b8a8a7c..c564fc55caf 100644 --- a/pkg/scheduler/apis/config/testing/config.go +++ b/pkg/scheduler/apis/config/testing/config.go @@ -21,6 +21,7 @@ import ( "k8s.io/component-base/config/v1alpha1" "k8s.io/kube-scheduler/config/v1beta2" + "k8s.io/kube-scheduler/config/v1beta3" "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/apis/config/scheme" ) @@ -36,3 +37,15 @@ func V1beta2ToInternalWithDefaults(t *testing.T, versionedCfg v1beta2.KubeSchedu } return &cfg } + +// V1beta3ToInternalWithDefaults creates a v1beta3 default configuration. +func V1beta3ToInternalWithDefaults(t *testing.T, versionedCfg v1beta3.KubeSchedulerConfiguration) *config.KubeSchedulerConfiguration { + versionedCfg.DebuggingConfiguration = *v1alpha1.NewRecommendedDebuggingConfiguration() + + scheme.Scheme.Default(&versionedCfg) + cfg := config.KubeSchedulerConfiguration{} + if err := scheme.Scheme.Convert(&versionedCfg, &cfg, nil); err != nil { + t.Fatal(err) + } + return &cfg +} diff --git a/pkg/scheduler/apis/config/testing/defaults/defaults.go b/pkg/scheduler/apis/config/testing/defaults/defaults.go index 8d21751d5af..d5da971e977 100644 --- a/pkg/scheduler/apis/config/testing/defaults/defaults.go +++ b/pkg/scheduler/apis/config/testing/defaults/defaults.go @@ -287,3 +287,140 @@ var PluginConfigsV1beta2 = []config.PluginConfig{ }, }, } + +// PluginsV1beta3 default set of v1beta3 plugins. +var PluginsV1beta3 = &config.Plugins{ + QueueSort: config.PluginSet{ + Enabled: []config.Plugin{ + {Name: names.PrioritySort}, + }, + }, + PreFilter: config.PluginSet{ + Enabled: []config.Plugin{ + {Name: names.NodeResourcesFit}, + {Name: names.NodePorts}, + {Name: names.VolumeRestrictions}, + {Name: names.PodTopologySpread}, + {Name: names.InterPodAffinity}, + {Name: names.VolumeBinding}, + {Name: names.NodeAffinity}, + }, + }, + Filter: config.PluginSet{ + Enabled: []config.Plugin{ + {Name: names.NodeUnschedulable}, + {Name: names.NodeName}, + {Name: names.TaintToleration}, + {Name: names.NodeAffinity}, + {Name: names.NodePorts}, + {Name: names.NodeResourcesFit}, + {Name: names.VolumeRestrictions}, + {Name: names.EBSLimits}, + {Name: names.GCEPDLimits}, + {Name: names.NodeVolumeLimits}, + {Name: names.AzureDiskLimits}, + {Name: names.VolumeBinding}, + {Name: names.VolumeZone}, + {Name: names.PodTopologySpread}, + {Name: names.InterPodAffinity}, + }, + }, + PostFilter: config.PluginSet{ + Enabled: []config.Plugin{ + {Name: names.DefaultPreemption}, + }, + }, + PreScore: config.PluginSet{ + Enabled: []config.Plugin{ + {Name: names.InterPodAffinity}, + {Name: names.PodTopologySpread}, + {Name: names.TaintToleration}, + {Name: names.NodeAffinity}, + }, + }, + Score: config.PluginSet{ + Enabled: []config.Plugin{ + {Name: names.NodeResourcesBalancedAllocation, Weight: 1}, + {Name: names.ImageLocality, Weight: 1}, + {Name: names.NodeResourcesFit, Weight: 1}, + // Weight is doubled because: + // - This is a score coming from user preference. + {Name: names.InterPodAffinity, Weight: 2}, + // Weight is doubled because: + // - This is a score coming from user preference. + {Name: names.NodeAffinity, Weight: 2}, + // Weight is doubled because: + // - This is a score coming from user preference. + // - It makes its signal comparable to NodeResourcesLeastAllocated. + {Name: names.PodTopologySpread, Weight: 2}, + // Weight is tripled because: + // - This is a score coming from user preference. + // - Usage of node tainting to group nodes in the cluster is increasing becoming a use-case + // for many user workloads + {Name: names.TaintToleration, Weight: 3}, + }, + }, + Reserve: config.PluginSet{ + Enabled: []config.Plugin{ + {Name: names.VolumeBinding}, + }, + }, + PreBind: config.PluginSet{ + Enabled: []config.Plugin{ + {Name: names.VolumeBinding}, + }, + }, + Bind: config.PluginSet{ + Enabled: []config.Plugin{ + {Name: names.DefaultBinder}, + }, + }, +} + +// PluginConfigsV1beta3 default plugin configurations. +var PluginConfigsV1beta3 = []config.PluginConfig{ + { + Name: "DefaultPreemption", + Args: &config.DefaultPreemptionArgs{ + MinCandidateNodesPercentage: 10, + MinCandidateNodesAbsolute: 100, + }, + }, + { + Name: "InterPodAffinity", + Args: &config.InterPodAffinityArgs{ + HardPodAffinityWeight: 1, + }, + }, + { + Name: "NodeAffinity", + Args: &config.NodeAffinityArgs{}, + }, + { + Name: "NodeResourcesBalancedAllocation", + Args: &config.NodeResourcesBalancedAllocationArgs{ + Resources: []config.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}}, + }, + }, + { + Name: "NodeResourcesFit", + Args: &config.NodeResourcesFitArgs{ + ScoringStrategy: &config.ScoringStrategy{ + Type: config.LeastAllocated, + Resources: []config.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}}, + }, + }, + }, + { + Name: "PodTopologySpread", + Args: &config.PodTopologySpreadArgs{ + DefaultingType: config.SystemDefaulting, + }, + }, + { + Name: "VolumeBinding", + Args: &config.VolumeBindingArgs{ + BindTimeoutSeconds: 600, + }, + }, +} diff --git a/pkg/scheduler/apis/config/validation/validation.go b/pkg/scheduler/apis/config/validation/validation.go index bb0a3b4fd0a..060d82a457d 100644 --- a/pkg/scheduler/apis/config/validation/validation.go +++ b/pkg/scheduler/apis/config/validation/validation.go @@ -34,6 +34,7 @@ import ( v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/apis/config/v1beta2" + "k8s.io/kubernetes/pkg/scheduler/apis/config/v1beta3" ) // ValidateKubeSchedulerConfiguration ensures validation of the KubeSchedulerConfiguration struct @@ -136,6 +137,10 @@ var removedPluginsByVersion = []removedPlugins{ "RequestedToCapacityRatio", }, }, + { + schemeGroupVersion: v1beta3.SchemeGroupVersion.String(), + plugins: []string{}, + }, } // isPluginRemoved checks if a given plugin was removed in the given component diff --git a/pkg/scheduler/apis/config/validation/validation_test.go b/pkg/scheduler/apis/config/validation/validation_test.go index cb5877e6222..9a6967a53e1 100644 --- a/pkg/scheduler/apis/config/validation/validation_test.go +++ b/pkg/scheduler/apis/config/validation/validation_test.go @@ -26,9 +26,10 @@ import ( componentbaseconfig "k8s.io/component-base/config" "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/apis/config/v1beta2" + "k8s.io/kubernetes/pkg/scheduler/apis/config/v1beta3" ) -func TestValidateKubeSchedulerConfiguration(t *testing.T) { +func TestValidateKubeSchedulerConfigurationV1beta2(t *testing.T) { podInitialBackoffSeconds := int64(1) podMaxBackoffSeconds := int64(1) validConfig := &config.KubeSchedulerConfiguration{ @@ -198,14 +199,312 @@ func TestValidateKubeSchedulerConfiguration(t *testing.T) { BindVerb: "bar", }) - badRemovedPlugins1 := validConfig.DeepCopy() // default v1beta2 + badRemovedPlugins1 := validConfig.DeepCopy() + badRemovedPlugins1.Profiles[0].Plugins.Score.Enabled = append(badRemovedPlugins1.Profiles[0].Plugins.Score.Enabled, config.Plugin{Name: "ServiceAffinity", Weight: 2}) + + badRemovedPlugins3 := validConfig.DeepCopy() + badRemovedPlugins3.Profiles[0].Plugins.Score.Enabled = append(badRemovedPlugins3.Profiles[0].Plugins.Score.Enabled, config.Plugin{Name: "NodeResourcesMostAllocated", Weight: 2}) + + goodRemovedPlugins2 := validConfig.DeepCopy() + goodRemovedPlugins2.Profiles[0].Plugins.Score.Enabled = append(goodRemovedPlugins2.Profiles[0].Plugins.Score.Enabled, config.Plugin{Name: "PodTopologySpread", Weight: 2}) + + deprecatedPluginsConfig := validConfig.DeepCopy() + deprecatedPluginsConfig.Profiles[0].PluginConfig = append(deprecatedPluginsConfig.Profiles[0].PluginConfig, config.PluginConfig{ + Name: "NodeResourcesLeastAllocated", + Args: &config.NodeResourcesLeastAllocatedArgs{}, + }) + + scenarios := map[string]struct { + expectedToFail bool + config *config.KubeSchedulerConfiguration + errorString string + }{ + "good": { + expectedToFail: false, + config: validConfig, + }, + "bad-parallelism-invalid-value": { + expectedToFail: true, + config: invalidParallelismValue, + }, + "bad-resource-name-not-set": { + expectedToFail: true, + config: resourceNameNotSet, + }, + "bad-resource-namespace-not-set": { + expectedToFail: true, + config: resourceNamespaceNotSet, + }, + "non-empty-metrics-bind-addr": { + expectedToFail: true, + config: metricsBindAddrInvalid, + }, + "non-empty-healthz-bind-addr": { + expectedToFail: true, + config: healthzBindAddrInvalid, + }, + "bad-percentage-of-nodes-to-score": { + expectedToFail: true, + config: percentageOfNodesToScore101, + }, + "scheduler-name-not-set": { + expectedToFail: true, + config: schedulerNameNotSet, + }, + "repeated-scheduler-name": { + expectedToFail: true, + config: repeatedSchedulerName, + }, + "different-queue-sort": { + expectedToFail: true, + config: differentQueueSort, + }, + "one-empty-queue-sort": { + expectedToFail: true, + config: oneEmptyQueueSort, + }, + "extender-negative-weight": { + expectedToFail: true, + config: extenderNegativeWeight, + }, + "extender-duplicate-managed-resources": { + expectedToFail: true, + config: extenderDuplicateManagedResource, + }, + "extender-duplicate-bind": { + expectedToFail: true, + config: extenderDuplicateBind, + }, + "invalid-node-percentage": { + expectedToFail: true, + config: invalidNodePercentage, + }, + "invalid-plugin-args": { + expectedToFail: true, + config: invalidPluginArgs, + }, + "duplicated-plugin-config": { + expectedToFail: true, + config: duplicatedPluginConfig, + }, + "mismatch-queue-sort": { + expectedToFail: true, + config: mismatchQueueSort, + }, + "bad-removed-plugins-1": { + expectedToFail: true, + config: badRemovedPlugins1, + }, + "bad-removed-plugins-3": { + expectedToFail: true, + config: badRemovedPlugins3, + }, + "good-removed-plugins-2": { + expectedToFail: false, + config: goodRemovedPlugins2, + }, + "bad-plugins-config": { + expectedToFail: true, + config: deprecatedPluginsConfig, + errorString: "profiles[0].pluginConfig[1]: Invalid value: \"NodeResourcesLeastAllocated\": was removed in version \"kubescheduler.config.k8s.io/v1beta2\" (KubeSchedulerConfiguration is version \"kubescheduler.config.k8s.io/v1beta2\")", + }, + } + + for name, scenario := range scenarios { + t.Run(name, func(t *testing.T) { + errs := ValidateKubeSchedulerConfiguration(scenario.config) + if errs == nil && scenario.expectedToFail { + t.Error("Unexpected success") + } + if errs != nil && !scenario.expectedToFail { + t.Errorf("Unexpected failure: %+v", errs) + } + fmt.Println(errs) + + if errs != nil && scenario.errorString != "" && errs.Error() != scenario.errorString { + t.Errorf("Unexpected error string\n want:\t%s\n got:\t%s", scenario.errorString, errs.Error()) + } + }) + } +} + +func TestValidateKubeSchedulerConfigurationV1beta3(t *testing.T) { + podInitialBackoffSeconds := int64(1) + podMaxBackoffSeconds := int64(1) + validConfig := &config.KubeSchedulerConfiguration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: v1beta3.SchemeGroupVersion.String(), + }, + Parallelism: 8, + ClientConnection: componentbaseconfig.ClientConnectionConfiguration{ + AcceptContentTypes: "application/json", + ContentType: "application/json", + QPS: 10, + Burst: 10, + }, + LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ + ResourceLock: "configmap", + LeaderElect: true, + LeaseDuration: metav1.Duration{Duration: 30 * time.Second}, + RenewDeadline: metav1.Duration{Duration: 15 * time.Second}, + RetryPeriod: metav1.Duration{Duration: 5 * time.Second}, + ResourceNamespace: "name", + ResourceName: "name", + }, + PodInitialBackoffSeconds: podInitialBackoffSeconds, + PodMaxBackoffSeconds: podMaxBackoffSeconds, + PercentageOfNodesToScore: 35, + Profiles: []config.KubeSchedulerProfile{ + { + SchedulerName: "me", + Plugins: &config.Plugins{ + QueueSort: config.PluginSet{ + Enabled: []config.Plugin{{Name: "CustomSort"}}, + }, + Score: config.PluginSet{ + Disabled: []config.Plugin{{Name: "*"}}, + }, + }, + PluginConfig: []config.PluginConfig{ + { + Name: "DefaultPreemption", + Args: &config.DefaultPreemptionArgs{MinCandidateNodesPercentage: 10, MinCandidateNodesAbsolute: 100}, + }, + }, + }, + { + SchedulerName: "other", + Plugins: &config.Plugins{ + QueueSort: config.PluginSet{ + Enabled: []config.Plugin{{Name: "CustomSort"}}, + }, + Bind: config.PluginSet{ + Enabled: []config.Plugin{{Name: "CustomBind"}}, + }, + }, + }, + }, + Extenders: []config.Extender{ + { + PrioritizeVerb: "prioritize", + Weight: 1, + }, + }, + } + + invalidParallelismValue := validConfig.DeepCopy() + invalidParallelismValue.Parallelism = 0 + + resourceNameNotSet := validConfig.DeepCopy() + resourceNameNotSet.LeaderElection.ResourceName = "" + + resourceNamespaceNotSet := validConfig.DeepCopy() + resourceNamespaceNotSet.LeaderElection.ResourceNamespace = "" + + enableContentProfilingSetWithoutEnableProfiling := validConfig.DeepCopy() + enableContentProfilingSetWithoutEnableProfiling.EnableProfiling = false + enableContentProfilingSetWithoutEnableProfiling.EnableContentionProfiling = true + + metricsBindAddrInvalid := validConfig.DeepCopy() + metricsBindAddrInvalid.MetricsBindAddress = "0.0.0.0:9090" + + healthzBindAddrInvalid := validConfig.DeepCopy() + healthzBindAddrInvalid.HealthzBindAddress = "0.0.0.0:9090" + + percentageOfNodesToScore101 := validConfig.DeepCopy() + percentageOfNodesToScore101.PercentageOfNodesToScore = int32(101) + + schedulerNameNotSet := validConfig.DeepCopy() + schedulerNameNotSet.Profiles[1].SchedulerName = "" + + repeatedSchedulerName := validConfig.DeepCopy() + repeatedSchedulerName.Profiles[0].SchedulerName = "other" + + differentQueueSort := validConfig.DeepCopy() + differentQueueSort.Profiles[1].Plugins.QueueSort.Enabled[0].Name = "AnotherSort" + + oneEmptyQueueSort := validConfig.DeepCopy() + oneEmptyQueueSort.Profiles[0].Plugins = nil + + extenderNegativeWeight := validConfig.DeepCopy() + extenderNegativeWeight.Extenders[0].Weight = -1 + + invalidNodePercentage := validConfig.DeepCopy() + invalidNodePercentage.Profiles[0].PluginConfig = []config.PluginConfig{ + { + Name: "DefaultPreemption", + Args: &config.DefaultPreemptionArgs{MinCandidateNodesPercentage: 200, MinCandidateNodesAbsolute: 100}, + }, + } + + invalidPluginArgs := validConfig.DeepCopy() + invalidPluginArgs.Profiles[0].PluginConfig = []config.PluginConfig{ + { + Name: "DefaultPreemption", + Args: &config.InterPodAffinityArgs{}, + }, + } + + duplicatedPluginConfig := validConfig.DeepCopy() + duplicatedPluginConfig.Profiles[0].PluginConfig = []config.PluginConfig{ + { + Name: "config", + }, + { + Name: "config", + }, + } + + mismatchQueueSort := validConfig.DeepCopy() + mismatchQueueSort.Profiles = []config.KubeSchedulerProfile{ + { + SchedulerName: "me", + Plugins: &config.Plugins{ + QueueSort: config.PluginSet{ + Enabled: []config.Plugin{{Name: "PrioritySort"}}, + }, + }, + PluginConfig: []config.PluginConfig{ + { + Name: "PrioritySort", + }, + }, + }, + { + SchedulerName: "other", + Plugins: &config.Plugins{ + QueueSort: config.PluginSet{ + Enabled: []config.Plugin{{Name: "CustomSort"}}, + }, + }, + PluginConfig: []config.PluginConfig{ + { + Name: "CustomSort", + }, + }, + }, + } + + extenderDuplicateManagedResource := validConfig.DeepCopy() + extenderDuplicateManagedResource.Extenders[0].ManagedResources = []config.ExtenderManagedResource{ + {Name: "foo", IgnoredByScheduler: false}, + {Name: "foo", IgnoredByScheduler: false}, + } + + extenderDuplicateBind := validConfig.DeepCopy() + extenderDuplicateBind.Extenders[0].BindVerb = "foo" + extenderDuplicateBind.Extenders = append(extenderDuplicateBind.Extenders, config.Extender{ + PrioritizeVerb: "prioritize", + BindVerb: "bar", + }) + + badRemovedPlugins1 := validConfig.DeepCopy() badRemovedPlugins1.Profiles[0].Plugins.Score.Enabled = append(badRemovedPlugins1.Profiles[0].Plugins.Score.Enabled, config.Plugin{Name: "ServiceAffinity", Weight: 2}) badRemovedPlugins2 := validConfig.DeepCopy() - badRemovedPlugins2.APIVersion = "kubescheduler.config.k8s.io/v1beta3" // hypothetical, v1beta3 doesn't exist badRemovedPlugins2.Profiles[0].Plugins.Score.Enabled = append(badRemovedPlugins2.Profiles[0].Plugins.Score.Enabled, config.Plugin{Name: "ServiceAffinity", Weight: 2}) - badRemovedPlugins3 := validConfig.DeepCopy() // default v1beta2 + badRemovedPlugins3 := validConfig.DeepCopy() badRemovedPlugins3.Profiles[0].Plugins.Score.Enabled = append(badRemovedPlugins3.Profiles[0].Plugins.Score.Enabled, config.Plugin{Name: "NodeResourcesMostAllocated", Weight: 2}) goodRemovedPlugins2 := validConfig.DeepCopy() @@ -313,7 +612,7 @@ func TestValidateKubeSchedulerConfiguration(t *testing.T) { "bad-plugins-config": { expectedToFail: true, config: deprecatedPluginsConfig, - errorString: "profiles[0].pluginConfig[1]: Invalid value: \"NodeResourcesLeastAllocated\": was removed in version \"kubescheduler.config.k8s.io/v1beta2\" (KubeSchedulerConfiguration is version \"kubescheduler.config.k8s.io/v1beta2\")", + errorString: "profiles[0].pluginConfig[1]: Invalid value: \"NodeResourcesLeastAllocated\": was removed in version \"kubescheduler.config.k8s.io/v1beta2\" (KubeSchedulerConfiguration is version \"kubescheduler.config.k8s.io/v1beta3\")", }, } diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 4cf15c0f3a9..9ba951b86be 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -38,7 +38,7 @@ import ( restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" - "k8s.io/kube-scheduler/config/v1beta2" + "k8s.io/kube-scheduler/config/v1beta3" podutil "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/kubernetes/pkg/apis/core/validation" "k8s.io/kubernetes/pkg/scheduler/apis/config" @@ -231,7 +231,7 @@ func New(client clientset.Interface, } if options.applyDefaultProfile { - var versionedCfg v1beta2.KubeSchedulerConfiguration + var versionedCfg v1beta3.KubeSchedulerConfiguration scheme.Scheme.Default(&versionedCfg) cfg := config.KubeSchedulerConfiguration{} if err := scheme.Scheme.Convert(&versionedCfg, &cfg, nil); err != nil {