Move scheduler plugin set and configuration defaulting to component config

This commit is contained in:
Abdullah Gharaibeh 2021-06-10 08:45:49 -04:00
parent 21ee533508
commit 265ef1741f
37 changed files with 3200 additions and 3382 deletions

View File

@ -42,10 +42,10 @@ func (o *DeprecatedOptions) AddFlags(fs *pflag.FlagSet, cfg *kubeschedulerconfig
return
}
fs.StringVar(&o.PolicyConfigFile, "policy-config-file", o.PolicyConfigFile, "DEPRECATED: file with scheduler policy configuration. This file is used if policy ConfigMap is not provided or --use-legacy-policy-config=true. Note: The scheduler will fail if this is combined with Plugin configs")
usage := fmt.Sprintf("DEPRECATED: name of the ConfigMap object that contains scheduler's policy configuration. It must exist in the system namespace before scheduler initialization if --use-legacy-policy-config=false. The config must be provided as the value of an element in 'Data' map with the key='%v'. Note: The scheduler will fail if this is combined with Plugin configs", kubeschedulerconfig.SchedulerPolicyConfigMapKey)
fs.StringVar(&o.PolicyConfigFile, "policy-config-file", o.PolicyConfigFile, "DEPRECATED: file with scheduler policy configuration. This file is used if policy ConfigMap is not provided or --use-legacy-policy-config=true. Note: The predicates/priorities defined in this file will take precedence over any profiles define in ComponentConfig.")
usage := fmt.Sprintf("DEPRECATED: name of the ConfigMap object that contains scheduler's policy configuration. It must exist in the system namespace before scheduler initialization if --use-legacy-policy-config=false. The config must be provided as the value of an element in 'Data' map with the key='%v'. Note: The predicates/priorities defined in this file will take precedence over any profiles define in ComponentConfig.", kubeschedulerconfig.SchedulerPolicyConfigMapKey)
fs.StringVar(&o.PolicyConfigMapName, "policy-configmap", o.PolicyConfigMapName, usage)
fs.StringVar(&o.PolicyConfigMapNamespace, "policy-configmap-namespace", o.PolicyConfigMapNamespace, "DEPRECATED: the namespace where policy ConfigMap is located. The kube-system namespace will be used if this is not provided or is empty. Note: The scheduler will fail if this is combined with Plugin configs")
fs.StringVar(&o.PolicyConfigMapNamespace, "policy-configmap-namespace", o.PolicyConfigMapNamespace, "DEPRECATED: the namespace where policy ConfigMap is located. The kube-system namespace will be used if this is not provided or is empty. Note: The predicates/priorities defined in this file will take precedence over any profiles define in ComponentConfig.")
fs.BoolVar(&o.UseLegacyPolicyConfig, "use-legacy-policy-config", o.UseLegacyPolicyConfig, "DEPRECATED: when set to true, scheduler will ignore policy ConfigMap and uses policy config file. Note: The scheduler will fail if this is combined with Plugin configs")
fs.BoolVar(&cfg.EnableProfiling, "profiling", cfg.EnableProfiling, "DEPRECATED: enable profiling via web interface host:port/debug/pprof/. This parameter is ignored if a config file is specified in --config.")

View File

@ -23,6 +23,7 @@ import (
"strconv"
"time"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/uuid"
apiserveroptions "k8s.io/apiserver/pkg/server/options"
@ -175,8 +176,8 @@ func (o *Options) ApplyTo(c *schedulerappconfig.Config) error {
if len(o.ConfigFile) == 0 {
c.ComponentConfig = o.ComponentConfig
// apply deprecated flags if no config file is loaded (this is the old behaviour).
o.Deprecated.ApplyTo(c)
if err := o.CombinedInsecureServing.ApplyTo(c, &c.ComponentConfig); err != nil {
return err
}
@ -194,18 +195,18 @@ func (o *Options) ApplyTo(c *schedulerappconfig.Config) error {
// apply any deprecated Policy flags, if applicable
o.Deprecated.ApplyTo(c)
// if the user has set CC profiles and is trying to use a Policy config, error out
// these configs are no longer merged and they should not be used simultaneously
if !emptySchedulerProfileConfig(c.ComponentConfig.Profiles) && c.LegacyPolicySource != nil {
return fmt.Errorf("cannot set a Plugin config and Policy config")
}
// use the loaded config file only, with the exception of --address and --port.
if err := o.CombinedInsecureServing.ApplyToFromLoadedConfig(c, &c.ComponentConfig); err != nil {
return err
}
}
// If the user is using the legacy policy config, clear the profiles, they will be set
// on scheduler instantiation based on the configurations in the policy file.
if c.LegacyPolicySource != nil {
c.ComponentConfig.Profiles = nil
}
if err := o.SecureServing.ApplyTo(&c.SecureServing, &c.LoopbackClientConfig); err != nil {
return err
}
@ -222,15 +223,6 @@ func (o *Options) ApplyTo(c *schedulerappconfig.Config) error {
return nil
}
// emptySchedulerProfileConfig returns true if the list of profiles passed to it contains only
// the "default-scheduler" profile with no plugins or pluginconfigs registered
// (this is the default empty profile initialized by defaults.go)
func emptySchedulerProfileConfig(profiles []kubeschedulerconfig.KubeSchedulerProfile) bool {
return len(profiles) == 1 &&
len(profiles[0].PluginConfig) == 0 &&
profiles[0].Plugins == nil
}
// Validate validates all the required options.
func (o *Options) Validate() []error {
var errs []error
@ -280,7 +272,11 @@ func (o *Options) Config() (*schedulerappconfig.Config, error) {
var leaderElectionConfig *leaderelection.LeaderElectionConfig
if c.ComponentConfig.LeaderElection.LeaderElect {
// Use the scheduler name in the first profile to record leader election.
coreRecorder := c.EventBroadcaster.DeprecatedNewLegacyRecorder(c.ComponentConfig.Profiles[0].SchedulerName)
schedulerName := corev1.DefaultSchedulerName
if len(c.ComponentConfig.Profiles) != 0 {
schedulerName = c.ComponentConfig.Profiles[0].SchedulerName
}
coreRecorder := c.EventBroadcaster.DeprecatedNewLegacyRecorder(schedulerName)
leaderElectionConfig, err = makeLeaderElectionConfig(c.ComponentConfig.LeaderElection, kubeConfig, coreRecorder)
if err != nil {
return nil, err

View File

@ -34,26 +34,15 @@ import (
"k8s.io/apimachinery/pkg/runtime"
apiserveroptions "k8s.io/apiserver/pkg/server/options"
componentbaseconfig "k8s.io/component-base/config"
"k8s.io/component-base/config/v1alpha1"
"k8s.io/component-base/logs"
"k8s.io/kube-scheduler/config/v1beta1"
"k8s.io/kube-scheduler/config/v1beta2"
kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config"
"k8s.io/kubernetes/pkg/scheduler/apis/config/scheme"
configtesting "k8s.io/kubernetes/pkg/scheduler/apis/config/testing"
"k8s.io/kubernetes/pkg/scheduler/apis/config/testing/defaults"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/names"
)
func newV1beta1DefaultComponentConfig() (*kubeschedulerconfig.KubeSchedulerConfiguration, error) {
versionedCfg := v1beta1.KubeSchedulerConfiguration{}
versionedCfg.DebuggingConfiguration = *v1alpha1.NewRecommendedDebuggingConfiguration()
scheme.Scheme.Default(&versionedCfg)
cfg := kubeschedulerconfig.KubeSchedulerConfiguration{}
if err := scheme.Scheme.Convert(&versionedCfg, &cfg, nil); err != nil {
return nil, err
}
return &cfg, nil
}
func TestSchedulerOptions(t *testing.T) {
// temp dir
tmpDir, err := ioutil.TempDir("", "scheduler-options")
@ -224,12 +213,12 @@ profiles:
- name: foo
- name: bar
disabled:
- name: baz
- name: VolumeBinding
preBind:
enabled:
- name: foo
disabled:
- name: baz
- name: VolumeBinding
pluginConfig:
- name: InterPodAffinity
args:
@ -255,12 +244,12 @@ profiles:
- name: foo
- name: bar
disabled:
- name: baz
- name: VolumeBinding
preBind:
enabled:
- name: foo
disabled:
- name: baz
- name: VolumeBinding
pluginConfig:
- name: ServiceAffinity
args:
@ -286,11 +275,14 @@ profiles:
reserve:
enabled:
- name: foo
- name: VolumeBinding
disabled:
- name: VolumeBinding
- schedulerName: "bar-profile"
plugins:
preBind:
disabled:
- name: baz
- name: VolumeBinding
pluginConfig:
- name: foo
`, configKubeconfig)), os.FileMode(0600)); err != nil {
@ -310,11 +302,14 @@ profiles:
reserve:
enabled:
- name: foo
- name: VolumeBinding
disabled:
- name: VolumeBinding
- schedulerName: "bar-profile"
plugins:
preBind:
disabled:
- name: baz
- name: VolumeBinding
pluginConfig:
- name: foo
`, configKubeconfig)), os.FileMode(0600)); err != nil {
@ -409,7 +404,11 @@ profiles:
PodInitialBackoffSeconds: defaultPodInitialBackoffSeconds,
PodMaxBackoffSeconds: defaultPodMaxBackoffSeconds,
Profiles: []kubeschedulerconfig.KubeSchedulerProfile{
{SchedulerName: "default-scheduler"},
{
SchedulerName: "default-scheduler",
Plugins: defaults.PluginsV1beta2,
PluginConfig: defaults.PluginConfigs,
},
},
},
},
@ -418,10 +417,7 @@ profiles:
options: &Options{
ConfigFile: v1beta1VersionConfig,
ComponentConfig: func() kubeschedulerconfig.KubeSchedulerConfiguration {
cfg, err := newV1beta1DefaultComponentConfig()
if err != nil {
t.Fatal(err)
}
cfg := configtesting.V1beta1ToInternalWithDefaults(t, v1beta1.KubeSchedulerConfiguration{})
return *cfg
}(),
SecureServing: (&apiserveroptions.SecureServingOptions{
@ -481,7 +477,11 @@ profiles:
PodInitialBackoffSeconds: defaultPodInitialBackoffSeconds,
PodMaxBackoffSeconds: defaultPodMaxBackoffSeconds,
Profiles: []kubeschedulerconfig.KubeSchedulerProfile{
{SchedulerName: "default-scheduler"},
{
SchedulerName: "default-scheduler",
Plugins: defaults.PluginsV1beta1,
PluginConfig: defaults.PluginConfigs,
},
},
},
},
@ -581,7 +581,11 @@ profiles:
PodInitialBackoffSeconds: defaultPodInitialBackoffSeconds,
PodMaxBackoffSeconds: defaultPodMaxBackoffSeconds,
Profiles: []kubeschedulerconfig.KubeSchedulerProfile{
{SchedulerName: "default-scheduler"},
{
SchedulerName: "default-scheduler",
Plugins: defaults.PluginsV1beta2,
PluginConfig: defaults.PluginConfigs,
},
},
},
},
@ -649,7 +653,11 @@ profiles:
PodInitialBackoffSeconds: defaultPodInitialBackoffSeconds,
PodMaxBackoffSeconds: defaultPodMaxBackoffSeconds,
Profiles: []kubeschedulerconfig.KubeSchedulerProfile{
{SchedulerName: "default-scheduler"},
{
SchedulerName: "default-scheduler",
Plugins: defaults.PluginsV1beta2,
PluginConfig: defaults.PluginConfigs,
},
},
},
expectedUsername: "none, http",
@ -694,23 +702,24 @@ profiles:
{
SchedulerName: "default-scheduler",
Plugins: &kubeschedulerconfig.Plugins{
QueueSort: defaults.PluginsV1beta2.QueueSort,
PreFilter: defaults.PluginsV1beta2.PreFilter,
Filter: defaults.PluginsV1beta2.Filter,
PostFilter: defaults.PluginsV1beta2.PostFilter,
PreScore: defaults.PluginsV1beta2.PreScore,
Score: defaults.PluginsV1beta2.Score,
Reserve: kubeschedulerconfig.PluginSet{
Enabled: []kubeschedulerconfig.Plugin{
{Name: "foo"},
{Name: "bar"},
},
Disabled: []kubeschedulerconfig.Plugin{
{Name: "baz"},
},
},
PreBind: kubeschedulerconfig.PluginSet{
Enabled: []kubeschedulerconfig.Plugin{
{Name: "foo"},
},
Disabled: []kubeschedulerconfig.Plugin{
{Name: "baz"},
},
},
Bind: defaults.PluginsV1beta2.Bind,
},
PluginConfig: []kubeschedulerconfig.PluginConfig{
{
@ -726,6 +735,39 @@ profiles:
ContentType: "application/json",
},
},
{
Name: "DefaultPreemption",
Args: &kubeschedulerconfig.DefaultPreemptionArgs{
MinCandidateNodesPercentage: 10,
MinCandidateNodesAbsolute: 100,
},
},
{
Name: "NodeAffinity",
Args: &kubeschedulerconfig.NodeAffinityArgs{},
},
{
Name: "NodeResourcesFit",
Args: &kubeschedulerconfig.NodeResourcesFitArgs{},
},
{
Name: "NodeResourcesLeastAllocated",
Args: &kubeschedulerconfig.NodeResourcesLeastAllocatedArgs{
Resources: []kubeschedulerconfig.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}},
},
},
{
Name: "PodTopologySpread",
Args: &kubeschedulerconfig.PodTopologySpreadArgs{
DefaultingType: kubeschedulerconfig.SystemDefaulting,
},
},
{
Name: "VolumeBinding",
Args: &kubeschedulerconfig.VolumeBindingArgs{
BindTimeoutSeconds: 600,
},
},
},
},
},
@ -771,23 +813,24 @@ profiles:
{
SchedulerName: "default-scheduler",
Plugins: &kubeschedulerconfig.Plugins{
QueueSort: defaults.PluginsV1beta1.QueueSort,
PreFilter: defaults.PluginsV1beta1.PreFilter,
Filter: defaults.PluginsV1beta1.Filter,
PostFilter: defaults.PluginsV1beta1.PostFilter,
PreScore: defaults.PluginsV1beta1.PreScore,
Score: defaults.PluginsV1beta1.Score,
Reserve: kubeschedulerconfig.PluginSet{
Enabled: []kubeschedulerconfig.Plugin{
{Name: "foo"},
{Name: "bar"},
},
Disabled: []kubeschedulerconfig.Plugin{
{Name: "baz"},
},
},
PreBind: kubeschedulerconfig.PluginSet{
Enabled: []kubeschedulerconfig.Plugin{
{Name: "foo"},
},
Disabled: []kubeschedulerconfig.Plugin{
{Name: "baz"},
},
},
Bind: defaults.PluginsV1beta1.Bind,
},
PluginConfig: []kubeschedulerconfig.PluginConfig{
{
@ -804,6 +847,45 @@ profiles:
ContentType: "application/json",
},
},
{
Name: "DefaultPreemption",
Args: &kubeschedulerconfig.DefaultPreemptionArgs{
MinCandidateNodesPercentage: 10,
MinCandidateNodesAbsolute: 100,
},
},
{
Name: "InterPodAffinity",
Args: &kubeschedulerconfig.InterPodAffinityArgs{
HardPodAffinityWeight: 1,
},
},
{
Name: "NodeAffinity",
Args: &kubeschedulerconfig.NodeAffinityArgs{},
},
{
Name: "NodeResourcesFit",
Args: &kubeschedulerconfig.NodeResourcesFitArgs{},
},
{
Name: "NodeResourcesLeastAllocated",
Args: &kubeschedulerconfig.NodeResourcesLeastAllocatedArgs{
Resources: []kubeschedulerconfig.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}},
},
},
{
Name: "PodTopologySpread",
Args: &kubeschedulerconfig.PodTopologySpreadArgs{
DefaultingType: kubeschedulerconfig.SystemDefaulting,
},
},
{
Name: "VolumeBinding",
Args: &kubeschedulerconfig.VolumeBindingArgs{
BindTimeoutSeconds: 600,
},
},
},
},
},
@ -849,26 +931,78 @@ profiles:
{
SchedulerName: "foo-profile",
Plugins: &kubeschedulerconfig.Plugins{
QueueSort: defaults.PluginsV1beta2.QueueSort,
PreFilter: defaults.PluginsV1beta2.PreFilter,
Filter: defaults.PluginsV1beta2.Filter,
PostFilter: defaults.PluginsV1beta2.PostFilter,
PreScore: defaults.PluginsV1beta2.PreScore,
Score: defaults.PluginsV1beta2.Score,
Bind: defaults.PluginsV1beta2.Bind,
PreBind: defaults.PluginsV1beta2.PreBind,
Reserve: kubeschedulerconfig.PluginSet{
Enabled: []kubeschedulerconfig.Plugin{
{Name: "foo"},
{Name: names.VolumeBinding},
},
},
},
PluginConfig: defaults.PluginConfigs,
},
{
SchedulerName: "bar-profile",
Plugins: &kubeschedulerconfig.Plugins{
PreBind: kubeschedulerconfig.PluginSet{
Disabled: []kubeschedulerconfig.Plugin{
{Name: "baz"},
},
},
QueueSort: defaults.PluginsV1beta2.QueueSort,
PreFilter: defaults.PluginsV1beta2.PreFilter,
Filter: defaults.PluginsV1beta2.Filter,
PostFilter: defaults.PluginsV1beta2.PostFilter,
PreScore: defaults.PluginsV1beta2.PreScore,
Score: defaults.PluginsV1beta2.Score,
Bind: defaults.PluginsV1beta2.Bind,
Reserve: defaults.PluginsV1beta2.Reserve,
},
PluginConfig: []kubeschedulerconfig.PluginConfig{
{
Name: "foo",
},
{
Name: "DefaultPreemption",
Args: &kubeschedulerconfig.DefaultPreemptionArgs{
MinCandidateNodesPercentage: 10,
MinCandidateNodesAbsolute: 100,
},
},
{
Name: "InterPodAffinity",
Args: &kubeschedulerconfig.InterPodAffinityArgs{
HardPodAffinityWeight: 1,
},
},
{
Name: "NodeAffinity",
Args: &kubeschedulerconfig.NodeAffinityArgs{},
},
{
Name: "NodeResourcesFit",
Args: &kubeschedulerconfig.NodeResourcesFitArgs{},
},
{
Name: "NodeResourcesLeastAllocated",
Args: &kubeschedulerconfig.NodeResourcesLeastAllocatedArgs{
Resources: []kubeschedulerconfig.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}},
},
},
{
Name: "PodTopologySpread",
Args: &kubeschedulerconfig.PodTopologySpreadArgs{
DefaultingType: kubeschedulerconfig.SystemDefaulting,
},
},
{
Name: "VolumeBinding",
Args: &kubeschedulerconfig.VolumeBindingArgs{
BindTimeoutSeconds: 600,
},
},
},
},
},
@ -914,37 +1048,82 @@ profiles:
{
SchedulerName: "foo-profile",
Plugins: &kubeschedulerconfig.Plugins{
QueueSort: defaults.PluginsV1beta1.QueueSort,
PreFilter: defaults.PluginsV1beta1.PreFilter,
Filter: defaults.PluginsV1beta1.Filter,
PostFilter: defaults.PluginsV1beta1.PostFilter,
PreScore: defaults.PluginsV1beta1.PreScore,
Score: defaults.PluginsV1beta1.Score,
Bind: defaults.PluginsV1beta1.Bind,
PreBind: defaults.PluginsV1beta1.PreBind,
Reserve: kubeschedulerconfig.PluginSet{
Enabled: []kubeschedulerconfig.Plugin{
{Name: "foo"},
{Name: names.VolumeBinding},
},
},
},
PluginConfig: defaults.PluginConfigs,
},
{
SchedulerName: "bar-profile",
Plugins: &kubeschedulerconfig.Plugins{
PreBind: kubeschedulerconfig.PluginSet{
Disabled: []kubeschedulerconfig.Plugin{
{Name: "baz"},
},
},
QueueSort: defaults.PluginsV1beta1.QueueSort,
PreFilter: defaults.PluginsV1beta1.PreFilter,
Filter: defaults.PluginsV1beta1.Filter,
PostFilter: defaults.PluginsV1beta1.PostFilter,
PreScore: defaults.PluginsV1beta1.PreScore,
Score: defaults.PluginsV1beta1.Score,
Bind: defaults.PluginsV1beta1.Bind,
Reserve: defaults.PluginsV1beta1.Reserve,
},
PluginConfig: []kubeschedulerconfig.PluginConfig{
{
Name: "foo",
},
},
},
},
{
Name: "DefaultPreemption",
Args: &kubeschedulerconfig.DefaultPreemptionArgs{
MinCandidateNodesPercentage: 10,
MinCandidateNodesAbsolute: 100,
},
},
{
name: "no config",
options: &Options{
Logs: logs.NewOptions(),
Name: "InterPodAffinity",
Args: &kubeschedulerconfig.InterPodAffinityArgs{
HardPodAffinityWeight: 1,
},
},
{
Name: "NodeAffinity",
Args: &kubeschedulerconfig.NodeAffinityArgs{},
},
{
Name: "NodeResourcesFit",
Args: &kubeschedulerconfig.NodeResourcesFitArgs{},
},
{
Name: "NodeResourcesLeastAllocated",
Args: &kubeschedulerconfig.NodeResourcesLeastAllocatedArgs{
Resources: []kubeschedulerconfig.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}},
},
},
{
Name: "PodTopologySpread",
Args: &kubeschedulerconfig.PodTopologySpreadArgs{
DefaultingType: kubeschedulerconfig.SystemDefaulting,
},
},
{
Name: "VolumeBinding",
Args: &kubeschedulerconfig.VolumeBindingArgs{
BindTimeoutSeconds: 600,
},
},
},
},
},
},
expectedError: "no configuration has been provided",
},
{
name: "Attempting to set Component Config Profiles and Policy config",
@ -953,8 +1132,46 @@ profiles:
Deprecated: &DeprecatedOptions{
PolicyConfigMapName: "bar",
},
Logs: logs.NewOptions(),
},
expectedError: "cannot set a Plugin config and Policy config",
expectedUsername: "config",
expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{
TypeMeta: metav1.TypeMeta{
APIVersion: v1beta2.SchemeGroupVersion.String(),
},
Parallelism: 16,
HealthzBindAddress: "0.0.0.0:10251",
MetricsBindAddress: "0.0.0.0:10251",
DebuggingConfiguration: componentbaseconfig.DebuggingConfiguration{
EnableProfiling: true,
EnableContentionProfiling: true,
},
LeaderElection: componentbaseconfig.LeaderElectionConfiguration{
LeaderElect: true,
LeaseDuration: metav1.Duration{Duration: 15 * time.Second},
RenewDeadline: metav1.Duration{Duration: 10 * time.Second},
RetryPeriod: metav1.Duration{Duration: 2 * time.Second},
ResourceLock: "leases",
ResourceNamespace: "kube-system",
ResourceName: "kube-scheduler",
},
ClientConnection: componentbaseconfig.ClientConnectionConfiguration{
Kubeconfig: configKubeconfig,
QPS: 50,
Burst: 100,
ContentType: "application/vnd.kubernetes.protobuf",
},
PercentageOfNodesToScore: defaultPercentageOfNodesToScore,
PodInitialBackoffSeconds: defaultPodInitialBackoffSeconds,
PodMaxBackoffSeconds: defaultPodMaxBackoffSeconds,
},
},
{
name: "no config",
options: &Options{
Logs: logs.NewOptions(),
},
expectedError: "no configuration has been provided",
},
{
name: "unknown field",

View File

@ -30,7 +30,8 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/spf13/pflag"
"k8s.io/kubernetes/cmd/kube-scheduler/app/options"
kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config"
"k8s.io/kubernetes/pkg/scheduler/apis/config"
"k8s.io/kubernetes/pkg/scheduler/apis/config/testing/defaults"
)
func TestSetup(t *testing.T) {
@ -151,116 +152,58 @@ profiles:
t.Fatal(err)
}
defaultPlugins := map[string][]kubeschedulerconfig.Plugin{
"QueueSortPlugin": {
{Name: "PrioritySort"},
},
"PreFilterPlugin": {
{Name: "NodeResourcesFit"},
{Name: "NodePorts"},
{Name: "PodTopologySpread"},
{Name: "InterPodAffinity"},
{Name: "VolumeBinding"},
{Name: "NodeAffinity"},
},
"FilterPlugin": {
{Name: "NodeUnschedulable"},
{Name: "NodeName"},
{Name: "TaintToleration"},
{Name: "NodeAffinity"},
{Name: "NodePorts"},
{Name: "NodeResourcesFit"},
{Name: "VolumeRestrictions"},
{Name: "EBSLimits"},
{Name: "GCEPDLimits"},
{Name: "NodeVolumeLimits"},
{Name: "AzureDiskLimits"},
{Name: "VolumeBinding"},
{Name: "VolumeZone"},
{Name: "PodTopologySpread"},
{Name: "InterPodAffinity"},
},
"PostFilterPlugin": {
{Name: "DefaultPreemption"},
},
"PreScorePlugin": {
{Name: "InterPodAffinity"},
{Name: "PodTopologySpread"},
{Name: "TaintToleration"},
{Name: "NodeAffinity"},
},
"ScorePlugin": {
{Name: "NodeResourcesBalancedAllocation", Weight: 1},
{Name: "ImageLocality", Weight: 1},
{Name: "InterPodAffinity", Weight: 1},
{Name: "NodeResourcesLeastAllocated", Weight: 1},
{Name: "NodeAffinity", Weight: 1},
{Name: "NodePreferAvoidPods", Weight: 10000},
{Name: "PodTopologySpread", Weight: 2},
{Name: "TaintToleration", Weight: 1},
},
"BindPlugin": {{Name: "DefaultBinder"}},
"ReservePlugin": {{Name: "VolumeBinding"}},
"PreBindPlugin": {{Name: "VolumeBinding"}},
}
testcases := []struct {
name string
flags []string
wantPlugins map[string]map[string][]kubeschedulerconfig.Plugin
wantPlugins map[string]*config.Plugins
}{
{
name: "default config",
flags: []string{
"--kubeconfig", configKubeconfig,
},
wantPlugins: map[string]map[string][]kubeschedulerconfig.Plugin{
"default-scheduler": defaultPlugins,
wantPlugins: map[string]*config.Plugins{
"default-scheduler": defaults.PluginsV1beta2,
},
},
{
name: "plugin config with single profile",
name: "component configuration",
flags: []string{
"--config", pluginConfigFile,
"--kubeconfig", configKubeconfig,
},
wantPlugins: map[string]map[string][]kubeschedulerconfig.Plugin{
wantPlugins: map[string]*config.Plugins{
"default-scheduler": {
"BindPlugin": {{Name: "DefaultBinder"}},
"FilterPlugin": {{Name: "NodeResourcesFit"}, {Name: "NodePorts"}},
"PreFilterPlugin": {{Name: "NodeResourcesFit"}, {Name: "NodePorts"}},
"PostFilterPlugin": {{Name: "DefaultPreemption"}},
"PreScorePlugin": {{Name: "InterPodAffinity"}, {Name: "TaintToleration"}},
"QueueSortPlugin": {{Name: "PrioritySort"}},
"ScorePlugin": {{Name: "InterPodAffinity", Weight: 1}, {Name: "TaintToleration", Weight: 1}},
"ReservePlugin": {{Name: "VolumeBinding"}},
"PreBindPlugin": {{Name: "VolumeBinding"}},
Bind: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultBinder"}}},
Filter: config.PluginSet{
Enabled: []config.Plugin{
{Name: "NodeResourcesFit"},
{Name: "NodePorts"},
},
},
},
{
name: "plugin config with multiple profiles",
flags: []string{
"--config", multiProfilesConfig,
"--kubeconfig", configKubeconfig,
},
wantPlugins: map[string]map[string][]kubeschedulerconfig.Plugin{
"profile-default-plugins": defaultPlugins,
"profile-disable-all-filter-and-score-plugins": {
"BindPlugin": {{Name: "DefaultBinder"}},
"QueueSortPlugin": {{Name: "PrioritySort"}},
"ReservePlugin": {{Name: "VolumeBinding"}},
"PreBindPlugin": {{Name: "VolumeBinding"}},
PreFilter: config.PluginSet{
Enabled: []config.Plugin{
{Name: "NodeResourcesFit"},
{Name: "NodePorts"},
},
},
PostFilter: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultPreemption"}}},
PreScore: config.PluginSet{
Enabled: []config.Plugin{
{Name: "InterPodAffinity"},
{Name: "TaintToleration"},
},
{
name: "default algorithm provider",
flags: []string{
"--kubeconfig", configKubeconfig,
},
wantPlugins: map[string]map[string][]kubeschedulerconfig.Plugin{
"default-scheduler": defaultPlugins,
QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: "PrioritySort"}}},
Score: config.PluginSet{
Enabled: []config.Plugin{
{Name: "InterPodAffinity", Weight: 1},
{Name: "TaintToleration", Weight: 1},
},
},
Reserve: config.PluginSet{Enabled: []config.Plugin{{Name: "VolumeBinding"}}},
PreBind: config.PluginSet{Enabled: []config.Plugin{{Name: "VolumeBinding"}}},
},
},
},
{
@ -269,25 +212,21 @@ profiles:
"--kubeconfig", configKubeconfig,
"--policy-config-file", policyConfigFile,
},
wantPlugins: map[string]map[string][]kubeschedulerconfig.Plugin{
wantPlugins: map[string]*config.Plugins{
"default-scheduler": {
"QueueSortPlugin": {{Name: "PrioritySort"}},
"PreFilterPlugin": {
{Name: "InterPodAffinity"},
},
"FilterPlugin": {
QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: "PrioritySort"}}},
PreFilter: config.PluginSet{Enabled: []config.Plugin{{Name: "InterPodAffinity"}}},
Filter: config.PluginSet{
Enabled: []config.Plugin{
{Name: "NodeUnschedulable"},
{Name: "TaintToleration"},
{Name: "InterPodAffinity"},
},
"PostFilterPlugin": {{Name: "DefaultPreemption"}},
"PreScorePlugin": {
{Name: "InterPodAffinity"},
},
"ScorePlugin": {
{Name: "InterPodAffinity", Weight: 2},
},
"BindPlugin": {{Name: "DefaultBinder"}},
PostFilter: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultPreemption"}}},
PreScore: config.PluginSet{Enabled: []config.Plugin{{Name: "InterPodAffinity"}}},
Score: config.PluginSet{Enabled: []config.Plugin{{Name: "InterPodAffinity", Weight: 2}}},
Bind: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultBinder"}}},
},
},
},
@ -332,7 +271,7 @@ profiles:
t.Fatal(err)
}
gotPlugins := make(map[string]map[string][]kubeschedulerconfig.Plugin)
gotPlugins := make(map[string]*config.Plugins)
for n, p := range sched.Profiles {
gotPlugins[n] = p.ListPlugins()
}

View File

@ -1,144 +0,0 @@
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package algorithmprovider
import (
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/features"
schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultpreemption"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/imagelocality"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/interpodaffinity"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeaffinity"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodename"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeports"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodepreferavoidpods"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeunschedulable"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodevolumelimits"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/podtopologyspread"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/queuesort"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/selectorspread"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/tainttoleration"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumebinding"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumerestrictions"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumezone"
)
func GetDefaultConfig() *schedulerapi.Plugins {
plugins := &schedulerapi.Plugins{
QueueSort: schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{
{Name: queuesort.Name},
},
},
PreFilter: schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{
{Name: noderesources.FitName},
{Name: nodeports.Name},
{Name: podtopologyspread.Name},
{Name: interpodaffinity.Name},
{Name: volumebinding.Name},
{Name: nodeaffinity.Name},
},
},
Filter: schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{
{Name: nodeunschedulable.Name},
{Name: nodename.Name},
{Name: tainttoleration.Name},
{Name: nodeaffinity.Name},
{Name: nodeports.Name},
{Name: noderesources.FitName},
{Name: volumerestrictions.Name},
{Name: nodevolumelimits.EBSName},
{Name: nodevolumelimits.GCEPDName},
{Name: nodevolumelimits.CSIName},
{Name: nodevolumelimits.AzureDiskName},
{Name: volumebinding.Name},
{Name: volumezone.Name},
{Name: podtopologyspread.Name},
{Name: interpodaffinity.Name},
},
},
PostFilter: schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{
{Name: defaultpreemption.Name},
},
},
PreScore: schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{
{Name: interpodaffinity.Name},
{Name: podtopologyspread.Name},
{Name: tainttoleration.Name},
{Name: nodeaffinity.Name},
},
},
Score: schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{
{Name: noderesources.BalancedAllocationName, Weight: 1},
{Name: imagelocality.Name, Weight: 1},
{Name: interpodaffinity.Name, Weight: 1},
{Name: noderesources.LeastAllocatedName, Weight: 1},
{Name: nodeaffinity.Name, Weight: 1},
{Name: nodepreferavoidpods.Name, Weight: 10000},
// Weight is doubled because:
// - This is a score coming from user preference.
// - It makes its signal comparable to NodeResourcesLeastAllocated.
{Name: podtopologyspread.Name, Weight: 2},
{Name: tainttoleration.Name, Weight: 1},
},
},
Reserve: schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{
{Name: volumebinding.Name},
},
},
PreBind: schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{
{Name: volumebinding.Name},
},
},
Bind: schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{
{Name: defaultbinder.Name},
},
},
}
applyFeatureGates(plugins)
return plugins
}
func applyFeatureGates(config *schedulerapi.Plugins) {
if utilfeature.DefaultFeatureGate.Enabled(features.VolumeCapacityPriority) {
config.Score.Enabled = append(config.Score.Enabled, schedulerapi.Plugin{Name: volumebinding.Name, Weight: 1})
}
if !utilfeature.DefaultFeatureGate.Enabled(features.DefaultPodTopologySpread) {
// When feature is enabled, the default spreading is done by
// PodTopologySpread plugin, which is enabled by default.
klog.Infof("Registering SelectorSpread plugin")
s := schedulerapi.Plugin{Name: selectorspread.Name}
config.PreScore.Enabled = append(config.PreScore.Enabled, s)
s.Weight = 1
config.Score.Enabled = append(config.Score.Enabled, s)
}
}

View File

@ -1,232 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package algorithmprovider
import (
"testing"
"github.com/google/go-cmp/cmp"
"k8s.io/apiserver/pkg/util/feature"
"k8s.io/component-base/featuregate"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/kubernetes/pkg/features"
schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultpreemption"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/imagelocality"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/interpodaffinity"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeaffinity"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodename"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeports"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodepreferavoidpods"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeunschedulable"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodevolumelimits"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/podtopologyspread"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/queuesort"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/selectorspread"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/tainttoleration"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumebinding"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumerestrictions"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumezone"
)
func TestApplyFeatureGates(t *testing.T) {
tests := []struct {
name string
features map[featuregate.Feature]bool
wantConfig *schedulerapi.Plugins
}{
{
name: "Feature gates disabled",
wantConfig: &schedulerapi.Plugins{
QueueSort: schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{
{Name: queuesort.Name},
},
},
PreFilter: schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{
{Name: noderesources.FitName},
{Name: nodeports.Name},
{Name: podtopologyspread.Name},
{Name: interpodaffinity.Name},
{Name: volumebinding.Name},
{Name: nodeaffinity.Name},
},
},
Filter: schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{
{Name: nodeunschedulable.Name},
{Name: nodename.Name},
{Name: tainttoleration.Name},
{Name: nodeaffinity.Name},
{Name: nodeports.Name},
{Name: noderesources.FitName},
{Name: volumerestrictions.Name},
{Name: nodevolumelimits.EBSName},
{Name: nodevolumelimits.GCEPDName},
{Name: nodevolumelimits.CSIName},
{Name: nodevolumelimits.AzureDiskName},
{Name: volumebinding.Name},
{Name: volumezone.Name},
{Name: podtopologyspread.Name},
{Name: interpodaffinity.Name},
},
},
PostFilter: schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{
{Name: defaultpreemption.Name},
},
},
PreScore: schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{
{Name: interpodaffinity.Name},
{Name: podtopologyspread.Name},
{Name: tainttoleration.Name},
{Name: nodeaffinity.Name},
},
},
Score: schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{
{Name: noderesources.BalancedAllocationName, Weight: 1},
{Name: imagelocality.Name, Weight: 1},
{Name: interpodaffinity.Name, Weight: 1},
{Name: noderesources.LeastAllocatedName, Weight: 1},
{Name: nodeaffinity.Name, Weight: 1},
{Name: nodepreferavoidpods.Name, Weight: 10000},
{Name: podtopologyspread.Name, Weight: 2},
{Name: tainttoleration.Name, Weight: 1},
},
},
Reserve: schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{
{Name: volumebinding.Name},
},
},
PreBind: schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{
{Name: volumebinding.Name},
},
},
Bind: schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{
{Name: defaultbinder.Name},
},
},
},
},
{
name: "DefaultPodTopologySpread disabled",
features: map[featuregate.Feature]bool{
features.DefaultPodTopologySpread: false,
},
wantConfig: &schedulerapi.Plugins{
QueueSort: schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{
{Name: queuesort.Name},
},
},
PreFilter: schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{
{Name: noderesources.FitName},
{Name: nodeports.Name},
{Name: podtopologyspread.Name},
{Name: interpodaffinity.Name},
{Name: volumebinding.Name},
{Name: nodeaffinity.Name},
},
},
Filter: schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{
{Name: nodeunschedulable.Name},
{Name: nodename.Name},
{Name: tainttoleration.Name},
{Name: nodeaffinity.Name},
{Name: nodeports.Name},
{Name: noderesources.FitName},
{Name: volumerestrictions.Name},
{Name: nodevolumelimits.EBSName},
{Name: nodevolumelimits.GCEPDName},
{Name: nodevolumelimits.CSIName},
{Name: nodevolumelimits.AzureDiskName},
{Name: volumebinding.Name},
{Name: volumezone.Name},
{Name: podtopologyspread.Name},
{Name: interpodaffinity.Name},
},
},
PostFilter: schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{
{Name: defaultpreemption.Name},
},
},
PreScore: schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{
{Name: interpodaffinity.Name},
{Name: podtopologyspread.Name},
{Name: tainttoleration.Name},
{Name: nodeaffinity.Name},
{Name: selectorspread.Name},
},
},
Score: schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{
{Name: noderesources.BalancedAllocationName, Weight: 1},
{Name: imagelocality.Name, Weight: 1},
{Name: interpodaffinity.Name, Weight: 1},
{Name: noderesources.LeastAllocatedName, Weight: 1},
{Name: nodeaffinity.Name, Weight: 1},
{Name: nodepreferavoidpods.Name, Weight: 10000},
{Name: podtopologyspread.Name, Weight: 2},
{Name: tainttoleration.Name, Weight: 1},
{Name: selectorspread.Name, Weight: 1},
},
},
Reserve: schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{
{Name: volumebinding.Name},
},
},
PreBind: schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{
{Name: volumebinding.Name},
},
},
Bind: schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{
{Name: defaultbinder.Name},
},
},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
for k, v := range test.features {
defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, k, v)()
}
gotConfig := GetDefaultConfig()
if diff := cmp.Diff(test.wantConfig, gotConfig); diff != "" {
t.Errorf("unexpected config diff (-want, +got): %s", diff)
}
})
}
}

View File

@ -27,6 +27,7 @@ import (
"k8s.io/kube-scheduler/config/v1beta1"
"k8s.io/kube-scheduler/config/v1beta2"
"k8s.io/kubernetes/pkg/scheduler/apis/config"
"k8s.io/kubernetes/pkg/scheduler/apis/config/testing/defaults"
"k8s.io/utils/pointer"
"sigs.k8s.io/yaml"
)
@ -102,6 +103,7 @@ profiles:
wantProfiles: []config.KubeSchedulerProfile{
{
SchedulerName: "default-scheduler",
Plugins: defaults.PluginsV1beta1,
PluginConfig: []config.PluginConfig{
{
Name: "DefaultPreemption",
@ -199,12 +201,13 @@ profiles:
wantProfiles: []config.KubeSchedulerProfile{
{
SchedulerName: "default-scheduler",
PluginConfig: []config.PluginConfig{
Plugins: defaults.PluginsV1beta1,
PluginConfig: append([]config.PluginConfig{
{
Name: "NodeLabel",
Args: &config.NodeLabelArgs{PresentLabels: []string{"bars"}},
},
},
}, defaults.PluginConfigs...),
},
},
},
@ -269,7 +272,8 @@ profiles:
wantProfiles: []config.KubeSchedulerProfile{
{
SchedulerName: "default-scheduler",
PluginConfig: []config.PluginConfig{
Plugins: defaults.PluginsV1beta1,
PluginConfig: append([]config.PluginConfig{
{
Name: "OutOfTreePlugin",
Args: &runtime.Unknown{
@ -277,7 +281,7 @@ profiles:
Raw: []byte(`{"foo":"bar"}`),
},
},
},
}, defaults.PluginConfigs...),
},
},
},
@ -307,6 +311,7 @@ profiles:
wantProfiles: []config.KubeSchedulerProfile{
{
SchedulerName: "default-scheduler",
Plugins: defaults.PluginsV1beta1,
PluginConfig: []config.PluginConfig{
{
Name: "DefaultPreemption",
@ -411,6 +416,7 @@ profiles:
wantProfiles: []config.KubeSchedulerProfile{
{
SchedulerName: "default-scheduler",
Plugins: defaults.PluginsV1beta2,
PluginConfig: []config.PluginConfig{
{
Name: "DefaultPreemption",
@ -498,11 +504,44 @@ profiles:
wantProfiles: []config.KubeSchedulerProfile{
{
SchedulerName: "default-scheduler",
Plugins: defaults.PluginsV1beta2,
PluginConfig: []config.PluginConfig{
{
Name: "DefaultPreemption",
Args: &config.DefaultPreemptionArgs{MinCandidateNodesPercentage: 50, MinCandidateNodesAbsolute: 100},
},
{
Name: "InterPodAffinity",
Args: &config.InterPodAffinityArgs{
HardPodAffinityWeight: 1,
},
},
{
Name: "NodeAffinity",
Args: &config.NodeAffinityArgs{},
},
{
Name: "NodeResourcesFit",
Args: &config.NodeResourcesFitArgs{},
},
{
Name: "NodeResourcesLeastAllocated",
Args: &config.NodeResourcesLeastAllocatedArgs{
Resources: []config.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}},
},
},
{
Name: "PodTopologySpread",
Args: &config.PodTopologySpreadArgs{
DefaultingType: config.SystemDefaulting,
},
},
{
Name: "VolumeBinding",
Args: &config.VolumeBindingArgs{
BindTimeoutSeconds: 600,
},
},
},
},
},
@ -568,7 +607,8 @@ profiles:
wantProfiles: []config.KubeSchedulerProfile{
{
SchedulerName: "default-scheduler",
PluginConfig: []config.PluginConfig{
Plugins: defaults.PluginsV1beta2,
PluginConfig: append([]config.PluginConfig{
{
Name: "OutOfTreePlugin",
Args: &runtime.Unknown{
@ -576,7 +616,7 @@ profiles:
Raw: []byte(`{"foo":"bar"}`),
},
},
},
}, defaults.PluginConfigs...),
},
},
},
@ -606,6 +646,7 @@ profiles:
wantProfiles: []config.KubeSchedulerProfile{
{
SchedulerName: "default-scheduler",
Plugins: defaults.PluginsV1beta2,
PluginConfig: []config.PluginConfig{
{
Name: "DefaultPreemption",

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,51 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package testing
import (
"testing"
"k8s.io/component-base/config/v1alpha1"
"k8s.io/kube-scheduler/config/v1beta1"
"k8s.io/kube-scheduler/config/v1beta2"
"k8s.io/kubernetes/pkg/scheduler/apis/config"
"k8s.io/kubernetes/pkg/scheduler/apis/config/scheme"
)
// V1beta1ToInternalWithDefaults creates a v1beta1 default configuration.
func V1beta1ToInternalWithDefaults(t *testing.T, versionedCfg v1beta1.KubeSchedulerConfiguration) *config.KubeSchedulerConfiguration {
versionedCfg.DebuggingConfiguration = *v1alpha1.NewRecommendedDebuggingConfiguration()
scheme.Scheme.Default(&versionedCfg)
cfg := config.KubeSchedulerConfiguration{}
if err := scheme.Scheme.Convert(&versionedCfg, &cfg, nil); err != nil {
t.Fatal(err)
}
return &cfg
}
// V1beta2ToInternalWithDefaults creates a v1beta2 default configuration.
func V1beta2ToInternalWithDefaults(t *testing.T, versionedCfg v1beta2.KubeSchedulerConfiguration) *config.KubeSchedulerConfiguration {
versionedCfg.DebuggingConfiguration = *v1alpha1.NewRecommendedDebuggingConfiguration()
scheme.Scheme.Default(&versionedCfg)
cfg := config.KubeSchedulerConfiguration{}
if err := scheme.Scheme.Convert(&versionedCfg, &cfg, nil); err != nil {
t.Fatal(err)
}
return &cfg
}

View File

@ -0,0 +1,227 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package defaults
import (
"k8s.io/kubernetes/pkg/scheduler/apis/config"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/names"
)
// PluginsV1beta1 default set of v1beta1 plugins.
var PluginsV1beta1 = &config.Plugins{
QueueSort: config.PluginSet{
Enabled: []config.Plugin{
{Name: names.PrioritySort},
},
},
PreFilter: config.PluginSet{
Enabled: []config.Plugin{
{Name: names.NodeResourcesFit},
{Name: names.NodePorts},
{Name: names.PodTopologySpread},
{Name: names.InterPodAffinity},
{Name: names.VolumeBinding},
{Name: names.NodeAffinity},
},
},
Filter: config.PluginSet{
Enabled: []config.Plugin{
{Name: names.NodeUnschedulable},
{Name: names.NodeName},
{Name: names.TaintToleration},
{Name: names.NodeAffinity},
{Name: names.NodePorts},
{Name: names.NodeResourcesFit},
{Name: names.VolumeRestrictions},
{Name: names.EBSLimits},
{Name: names.GCEPDLimits},
{Name: names.NodeVolumeLimits},
{Name: names.AzureDiskLimits},
{Name: names.VolumeBinding},
{Name: names.VolumeZone},
{Name: names.PodTopologySpread},
{Name: names.InterPodAffinity},
},
},
PostFilter: config.PluginSet{
Enabled: []config.Plugin{
{Name: names.DefaultPreemption},
},
},
PreScore: config.PluginSet{
Enabled: []config.Plugin{
{Name: names.InterPodAffinity},
{Name: names.PodTopologySpread},
{Name: names.TaintToleration},
{Name: names.NodeAffinity},
},
},
Score: config.PluginSet{
Enabled: []config.Plugin{
{Name: names.NodeResourcesBalancedAllocation, Weight: 1},
{Name: names.ImageLocality, Weight: 1},
{Name: names.InterPodAffinity, Weight: 1},
{Name: names.NodeResourcesLeastAllocated, Weight: 1},
{Name: names.NodeAffinity, Weight: 1},
{Name: names.NodePreferAvoidPods, Weight: 10000},
// Weight is doubled because:
// - This is a score coming from user preference.
// - It makes its signal comparable to NodeResourcesLeastAllocated.
{Name: names.PodTopologySpread, Weight: 2},
{Name: names.TaintToleration, Weight: 1},
},
},
Reserve: config.PluginSet{
Enabled: []config.Plugin{
{Name: names.VolumeBinding},
},
},
PreBind: config.PluginSet{
Enabled: []config.Plugin{
{Name: names.VolumeBinding},
},
},
Bind: config.PluginSet{
Enabled: []config.Plugin{
{Name: names.DefaultBinder},
},
},
}
// PluginsV1beta2 default set of v1beta2 plugins.
var PluginsV1beta2 = &config.Plugins{
QueueSort: config.PluginSet{
Enabled: []config.Plugin{
{Name: names.PrioritySort},
},
},
PreFilter: config.PluginSet{
Enabled: []config.Plugin{
{Name: names.NodeResourcesFit},
{Name: names.NodePorts},
{Name: names.PodTopologySpread},
{Name: names.InterPodAffinity},
{Name: names.VolumeBinding},
{Name: names.NodeAffinity},
},
},
Filter: config.PluginSet{
Enabled: []config.Plugin{
{Name: names.NodeUnschedulable},
{Name: names.NodeName},
{Name: names.TaintToleration},
{Name: names.NodeAffinity},
{Name: names.NodePorts},
{Name: names.NodeResourcesFit},
{Name: names.VolumeRestrictions},
{Name: names.EBSLimits},
{Name: names.GCEPDLimits},
{Name: names.NodeVolumeLimits},
{Name: names.AzureDiskLimits},
{Name: names.VolumeBinding},
{Name: names.VolumeZone},
{Name: names.PodTopologySpread},
{Name: names.InterPodAffinity},
},
},
PostFilter: config.PluginSet{
Enabled: []config.Plugin{
{Name: names.DefaultPreemption},
},
},
PreScore: config.PluginSet{
Enabled: []config.Plugin{
{Name: names.InterPodAffinity},
{Name: names.PodTopologySpread},
{Name: names.TaintToleration},
{Name: names.NodeAffinity},
},
},
Score: config.PluginSet{
Enabled: []config.Plugin{
{Name: names.NodeResourcesBalancedAllocation, Weight: 1},
{Name: names.ImageLocality, Weight: 1},
{Name: names.InterPodAffinity, Weight: 1},
{Name: names.NodeResourcesLeastAllocated, Weight: 1},
{Name: names.NodeAffinity, Weight: 1},
// Weight is doubled because:
// - This is a score coming from user preference.
// - It makes its signal comparable to NodeResourcesLeastAllocated.
{Name: names.PodTopologySpread, Weight: 2},
{Name: names.TaintToleration, Weight: 1},
},
},
Reserve: config.PluginSet{
Enabled: []config.Plugin{
{Name: names.VolumeBinding},
},
},
PreBind: config.PluginSet{
Enabled: []config.Plugin{
{Name: names.VolumeBinding},
},
},
Bind: config.PluginSet{
Enabled: []config.Plugin{
{Name: names.DefaultBinder},
},
},
}
// PluginConfigs default plugin configurations. This could get versioned, but since
// all available versions produce the same defaults, we just have one for now.
var PluginConfigs = []config.PluginConfig{
{
Name: "DefaultPreemption",
Args: &config.DefaultPreemptionArgs{
MinCandidateNodesPercentage: 10,
MinCandidateNodesAbsolute: 100,
},
},
{
Name: "InterPodAffinity",
Args: &config.InterPodAffinityArgs{
HardPodAffinityWeight: 1,
},
},
{
Name: "NodeAffinity",
Args: &config.NodeAffinityArgs{},
},
{
Name: "NodeResourcesFit",
Args: &config.NodeResourcesFitArgs{},
},
{
Name: "NodeResourcesLeastAllocated",
Args: &config.NodeResourcesLeastAllocatedArgs{
Resources: []config.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}},
},
},
{
Name: "PodTopologySpread",
Args: &config.PodTopologySpreadArgs{
DefaultingType: config.SystemDefaulting,
},
},
{
Name: "VolumeBinding",
Args: &config.VolumeBindingArgs{
BindTimeoutSeconds: 600,
},
},
}

View File

@ -243,69 +243,31 @@ const (
MaxWeight = MaxTotalScore / MaxCustomPriorityScore
)
func appendPluginSet(dst PluginSet, src PluginSet) PluginSet {
dst.Enabled = append(dst.Enabled, src.Enabled...)
dst.Disabled = append(dst.Disabled, src.Disabled...)
return dst
}
// Append appends src Plugins to current Plugins. If a PluginSet is nil, it will
// be created.
func (p *Plugins) Append(src *Plugins) {
if p == nil || src == nil {
return
// Names returns the list of enabled plugin names.
func (p *Plugins) Names() []string {
if p == nil {
return nil
}
p.QueueSort = appendPluginSet(p.QueueSort, src.QueueSort)
p.PreFilter = appendPluginSet(p.PreFilter, src.PreFilter)
p.Filter = appendPluginSet(p.Filter, src.Filter)
p.PostFilter = appendPluginSet(p.PostFilter, src.PostFilter)
p.PreScore = appendPluginSet(p.PreScore, src.PreScore)
p.Score = appendPluginSet(p.Score, src.Score)
p.Reserve = appendPluginSet(p.Reserve, src.Reserve)
p.Permit = appendPluginSet(p.Permit, src.Permit)
p.PreBind = appendPluginSet(p.PreBind, src.PreBind)
p.Bind = appendPluginSet(p.Bind, src.Bind)
p.PostBind = appendPluginSet(p.PostBind, src.PostBind)
}
// Apply merges the plugin configuration from custom plugins, handling disabled sets.
func (p *Plugins) Apply(customPlugins *Plugins) {
if customPlugins == nil {
return
extensions := []PluginSet{
p.PreFilter,
p.Filter,
p.PostFilter,
p.Reserve,
p.PreScore,
p.Score,
p.PreBind,
p.Bind,
p.PostBind,
p.Permit,
p.QueueSort,
}
p.QueueSort = mergePluginSets(p.QueueSort, customPlugins.QueueSort)
p.PreFilter = mergePluginSets(p.PreFilter, customPlugins.PreFilter)
p.Filter = mergePluginSets(p.Filter, customPlugins.Filter)
p.PostFilter = mergePluginSets(p.PostFilter, customPlugins.PostFilter)
p.PreScore = mergePluginSets(p.PreScore, customPlugins.PreScore)
p.Score = mergePluginSets(p.Score, customPlugins.Score)
p.Reserve = mergePluginSets(p.Reserve, customPlugins.Reserve)
p.Permit = mergePluginSets(p.Permit, customPlugins.Permit)
p.PreBind = mergePluginSets(p.PreBind, customPlugins.PreBind)
p.Bind = mergePluginSets(p.Bind, customPlugins.Bind)
p.PostBind = mergePluginSets(p.PostBind, customPlugins.PostBind)
}
func mergePluginSets(defaultPluginSet, customPluginSet PluginSet) PluginSet {
disabledPlugins := sets.NewString()
for _, disabledPlugin := range customPluginSet.Disabled {
disabledPlugins.Insert(disabledPlugin.Name)
}
var enabledPlugins []Plugin
if !disabledPlugins.Has("*") {
for _, defaultEnabledPlugin := range defaultPluginSet.Enabled {
if disabledPlugins.Has(defaultEnabledPlugin.Name) {
continue
}
enabledPlugins = append(enabledPlugins, defaultEnabledPlugin)
n := sets.NewString()
for _, e := range extensions {
for _, pg := range e.Enabled {
n.Insert(pg.Name)
}
}
enabledPlugins = append(enabledPlugins, customPluginSet.Enabled...)
return PluginSet{Enabled: enabledPlugins}
return n.List()
}
// Extender holds the parameters used to communicate with the extender. If a verb is unspecified/empty,

View File

@ -22,210 +22,40 @@ import (
"github.com/google/go-cmp/cmp"
)
func TestPluginsAppend(t *testing.T) {
func TestPluginsNames(t *testing.T) {
tests := []struct {
name string
customPlugins *Plugins
defaultPlugins *Plugins
expectedPlugins *Plugins
plugins *Plugins
want []string
}{
{
name: "AppendPlugin",
customPlugins: &Plugins{
Filter: PluginSet{
Enabled: []Plugin{
{Name: "CustomPlugin"},
},
},
},
defaultPlugins: &Plugins{
Filter: PluginSet{
Enabled: []Plugin{
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
},
},
},
expectedPlugins: &Plugins{
Filter: PluginSet{
Enabled: []Plugin{
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
{Name: "CustomPlugin"},
},
},
},
name: "empty",
},
{
name: "AppendNilPlugin",
customPlugins: nil,
defaultPlugins: &Plugins{},
expectedPlugins: &Plugins{},
name: "with duplicates",
plugins: &Plugins{
Filter: PluginSet{
Enabled: []Plugin{
{Name: "CustomFilter"},
},
},
PreFilter: PluginSet{
Enabled: []Plugin{
{Name: "CustomFilter"},
},
},
Score: PluginSet{
Enabled: []Plugin{
{Name: "CustomScore"},
},
},
},
want: []string{"CustomFilter", "CustomScore"},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
test.defaultPlugins.Append(test.customPlugins)
if d := cmp.Diff(test.expectedPlugins, test.defaultPlugins); d != "" {
t.Fatalf("plugins mismatch (-want +got):\n%s", d)
}
})
}
}
func TestPluginsApply(t *testing.T) {
tests := []struct {
name string
customPlugins *Plugins
defaultPlugins *Plugins
expectedPlugins *Plugins
}{
{
name: "AppendCustomPlugin",
customPlugins: &Plugins{
Filter: PluginSet{
Enabled: []Plugin{
{Name: "CustomPlugin"},
},
},
},
defaultPlugins: &Plugins{
Filter: PluginSet{
Enabled: []Plugin{
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
},
},
},
expectedPlugins: &Plugins{
Filter: PluginSet{
Enabled: []Plugin{
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
{Name: "CustomPlugin"},
},
},
},
},
{
name: "InsertAfterDefaultPlugins2",
customPlugins: &Plugins{
Filter: PluginSet{
Enabled: []Plugin{
{Name: "CustomPlugin"},
{Name: "DefaultPlugin2"},
},
Disabled: []Plugin{
{Name: "DefaultPlugin2"},
},
},
},
defaultPlugins: &Plugins{
Filter: PluginSet{
Enabled: []Plugin{
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
},
},
},
expectedPlugins: &Plugins{
Filter: PluginSet{
Enabled: []Plugin{
{Name: "DefaultPlugin1"},
{Name: "CustomPlugin"},
{Name: "DefaultPlugin2"},
},
},
},
},
{
name: "InsertBeforeAllPlugins",
customPlugins: &Plugins{
Filter: PluginSet{
Enabled: []Plugin{
{Name: "CustomPlugin"},
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
},
Disabled: []Plugin{
{Name: "*"},
},
},
},
defaultPlugins: &Plugins{
Filter: PluginSet{
Enabled: []Plugin{
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
},
},
},
expectedPlugins: &Plugins{
Filter: PluginSet{
Enabled: []Plugin{
{Name: "CustomPlugin"},
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
},
},
},
},
{
name: "ReorderDefaultPlugins",
customPlugins: &Plugins{
Filter: PluginSet{
Enabled: []Plugin{
{Name: "DefaultPlugin2"},
{Name: "DefaultPlugin1"},
},
Disabled: []Plugin{
{Name: "*"},
},
},
},
defaultPlugins: &Plugins{
Filter: PluginSet{
Enabled: []Plugin{
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
},
},
},
expectedPlugins: &Plugins{
Filter: PluginSet{
Enabled: []Plugin{
{Name: "DefaultPlugin2"},
{Name: "DefaultPlugin1"},
},
},
},
},
{
name: "ApplyNilCustomPlugin",
customPlugins: nil,
defaultPlugins: &Plugins{
Filter: PluginSet{
Enabled: []Plugin{
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
},
},
},
expectedPlugins: &Plugins{
Filter: PluginSet{
Enabled: []Plugin{
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
},
},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
test.defaultPlugins.Apply(test.customPlugins)
if d := cmp.Diff(test.expectedPlugins, test.defaultPlugins); d != "" {
if d := cmp.Diff(test.want, test.plugins.Names()); d != "" {
t.Fatalf("plugins mismatch (-want +got):\n%s", d)
}
})

View File

@ -149,20 +149,20 @@ func Convert_v1beta1_KubeSchedulerConfiguration_To_config_KubeSchedulerConfigura
func convertToInternalPluginConfigArgs(out *config.KubeSchedulerConfiguration) error {
scheme := getPluginArgConversionScheme()
for i := range out.Profiles {
for j := range out.Profiles[i].PluginConfig {
args := out.Profiles[i].PluginConfig[j].Args
prof := &out.Profiles[i]
for j := range prof.PluginConfig {
args := prof.PluginConfig[j].Args
if args == nil {
continue
}
if _, isUnknown := args.(*runtime.Unknown); isUnknown {
continue
}
scheme.Default(args)
internalArgs, err := scheme.ConvertToVersion(args, config.SchemeGroupVersion)
if err != nil {
return fmt.Errorf("converting .Profiles[%d].PluginConfig[%d].Args into internal type: %w", i, j, err)
}
out.Profiles[i].PluginConfig[j].Args = internalArgs
prof.PluginConfig[j].Args = internalArgs
}
}
return nil

View File

@ -0,0 +1,179 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package v1beta1
import (
"k8s.io/apimachinery/pkg/util/sets"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/klog/v2"
"k8s.io/kube-scheduler/config/v1beta1"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/names"
"k8s.io/utils/pointer"
)
// getDefaultPlugins returns the default set of plugins.
func getDefaultPlugins() *v1beta1.Plugins {
plugins := &v1beta1.Plugins{
QueueSort: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.PrioritySort},
},
},
PreFilter: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.NodeResourcesFit},
{Name: names.NodePorts},
{Name: names.PodTopologySpread},
{Name: names.InterPodAffinity},
{Name: names.VolumeBinding},
{Name: names.NodeAffinity},
},
},
Filter: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.NodeUnschedulable},
{Name: names.NodeName},
{Name: names.TaintToleration},
{Name: names.NodeAffinity},
{Name: names.NodePorts},
{Name: names.NodeResourcesFit},
{Name: names.VolumeRestrictions},
{Name: names.EBSLimits},
{Name: names.GCEPDLimits},
{Name: names.NodeVolumeLimits},
{Name: names.AzureDiskLimits},
{Name: names.VolumeBinding},
{Name: names.VolumeZone},
{Name: names.PodTopologySpread},
{Name: names.InterPodAffinity},
},
},
PostFilter: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.DefaultPreemption},
},
},
PreScore: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.InterPodAffinity},
{Name: names.PodTopologySpread},
{Name: names.TaintToleration},
{Name: names.NodeAffinity},
},
},
Score: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.NodeResourcesBalancedAllocation, Weight: pointer.Int32Ptr(1)},
{Name: names.ImageLocality, Weight: pointer.Int32Ptr(1)},
{Name: names.InterPodAffinity, Weight: pointer.Int32Ptr(1)},
{Name: names.NodeResourcesLeastAllocated, Weight: pointer.Int32Ptr(1)},
{Name: names.NodeAffinity, Weight: pointer.Int32Ptr(1)},
{Name: names.NodePreferAvoidPods, Weight: pointer.Int32Ptr(10000)},
// Weight is doubled because:
// - This is a score coming from user preference.
// - It makes its signal comparable to NodeResourcesLeastAllocated.
{Name: names.PodTopologySpread, Weight: pointer.Int32Ptr(2)},
{Name: names.TaintToleration, Weight: pointer.Int32Ptr(1)},
},
},
Reserve: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.VolumeBinding},
},
},
PreBind: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.VolumeBinding},
},
},
Bind: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.DefaultBinder},
},
},
}
applyFeatureGates(plugins)
return plugins
}
func applyFeatureGates(config *v1beta1.Plugins) {
if utilfeature.DefaultFeatureGate.Enabled(features.VolumeCapacityPriority) {
config.Score.Enabled = append(config.Score.Enabled, v1beta1.Plugin{Name: names.VolumeBinding, Weight: pointer.Int32Ptr(1)})
}
if !utilfeature.DefaultFeatureGate.Enabled(features.DefaultPodTopologySpread) {
// When feature is enabled, the default spreading is done by
// PodTopologySpread plugin, which is enabled by default.
klog.InfoS("Registering SelectorSpread plugin")
s := v1beta1.Plugin{Name: names.SelectorSpread}
config.PreScore.Enabled = append(config.PreScore.Enabled, s)
s.Weight = pointer.Int32Ptr(1)
config.Score.Enabled = append(config.Score.Enabled, s)
}
}
// mergePlugins merges the custom set into the given default one, handling disabled sets.
func mergePlugins(defaultPlugins, customPlugins *v1beta1.Plugins) *v1beta1.Plugins {
if customPlugins == nil {
return defaultPlugins
}
defaultPlugins.QueueSort = mergePluginSet(defaultPlugins.QueueSort, customPlugins.QueueSort)
defaultPlugins.PreFilter = mergePluginSet(defaultPlugins.PreFilter, customPlugins.PreFilter)
defaultPlugins.Filter = mergePluginSet(defaultPlugins.Filter, customPlugins.Filter)
defaultPlugins.PostFilter = mergePluginSet(defaultPlugins.PostFilter, customPlugins.PostFilter)
defaultPlugins.PreScore = mergePluginSet(defaultPlugins.PreScore, customPlugins.PreScore)
defaultPlugins.Score = mergePluginSet(defaultPlugins.Score, customPlugins.Score)
defaultPlugins.Reserve = mergePluginSet(defaultPlugins.Reserve, customPlugins.Reserve)
defaultPlugins.Permit = mergePluginSet(defaultPlugins.Permit, customPlugins.Permit)
defaultPlugins.PreBind = mergePluginSet(defaultPlugins.PreBind, customPlugins.PreBind)
defaultPlugins.Bind = mergePluginSet(defaultPlugins.Bind, customPlugins.Bind)
defaultPlugins.PostBind = mergePluginSet(defaultPlugins.PostBind, customPlugins.PostBind)
return defaultPlugins
}
func mergePluginSet(defaultPluginSet, customPluginSet *v1beta1.PluginSet) *v1beta1.PluginSet {
disabledPlugins := sets.NewString()
if customPluginSet != nil {
for _, disabledPlugin := range customPluginSet.Disabled {
disabledPlugins.Insert(disabledPlugin.Name)
}
}
var enabledPlugins []v1beta1.Plugin
if defaultPluginSet != nil && !disabledPlugins.Has("*") {
for _, defaultEnabledPlugin := range defaultPluginSet.Enabled {
if disabledPlugins.Has(defaultEnabledPlugin.Name) {
continue
}
enabledPlugins = append(enabledPlugins, defaultEnabledPlugin)
}
}
if customPluginSet != nil {
enabledPlugins = append(enabledPlugins, customPluginSet.Enabled...)
}
if len(enabledPlugins) == 0 {
return nil
}
return &v1beta1.PluginSet{Enabled: enabledPlugins}
}

View File

@ -0,0 +1,374 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package v1beta1
import (
"testing"
"github.com/google/go-cmp/cmp"
"k8s.io/apiserver/pkg/util/feature"
"k8s.io/component-base/featuregate"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/kube-scheduler/config/v1beta1"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/names"
"k8s.io/utils/pointer"
)
func TestApplyFeatureGates(t *testing.T) {
tests := []struct {
name string
features map[featuregate.Feature]bool
wantConfig *v1beta1.Plugins
}{
{
name: "Feature gates disabled",
wantConfig: &v1beta1.Plugins{
QueueSort: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.PrioritySort},
},
},
PreFilter: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.NodeResourcesFit},
{Name: names.NodePorts},
{Name: names.PodTopologySpread},
{Name: names.InterPodAffinity},
{Name: names.VolumeBinding},
{Name: names.NodeAffinity},
},
},
Filter: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.NodeUnschedulable},
{Name: names.NodeName},
{Name: names.TaintToleration},
{Name: names.NodeAffinity},
{Name: names.NodePorts},
{Name: names.NodeResourcesFit},
{Name: names.VolumeRestrictions},
{Name: names.EBSLimits},
{Name: names.GCEPDLimits},
{Name: names.NodeVolumeLimits},
{Name: names.AzureDiskLimits},
{Name: names.VolumeBinding},
{Name: names.VolumeZone},
{Name: names.PodTopologySpread},
{Name: names.InterPodAffinity},
},
},
PostFilter: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.DefaultPreemption},
},
},
PreScore: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.InterPodAffinity},
{Name: names.PodTopologySpread},
{Name: names.TaintToleration},
{Name: names.NodeAffinity},
},
},
Score: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.NodeResourcesBalancedAllocation, Weight: pointer.Int32Ptr(1)},
{Name: names.ImageLocality, Weight: pointer.Int32Ptr(1)},
{Name: names.InterPodAffinity, Weight: pointer.Int32Ptr(1)},
{Name: names.NodeResourcesLeastAllocated, Weight: pointer.Int32Ptr(1)},
{Name: names.NodeAffinity, Weight: pointer.Int32Ptr(1)},
{Name: names.NodePreferAvoidPods, Weight: pointer.Int32Ptr(10000)},
{Name: names.PodTopologySpread, Weight: pointer.Int32Ptr(2)},
{Name: names.TaintToleration, Weight: pointer.Int32Ptr(1)},
},
},
Reserve: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.VolumeBinding},
},
},
PreBind: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.VolumeBinding},
},
},
Bind: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.DefaultBinder},
},
},
},
},
{
name: "DefaultPodTopologySpread disabled",
features: map[featuregate.Feature]bool{
features.DefaultPodTopologySpread: false,
},
wantConfig: &v1beta1.Plugins{
QueueSort: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.PrioritySort},
},
},
PreFilter: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.NodeResourcesFit},
{Name: names.NodePorts},
{Name: names.PodTopologySpread},
{Name: names.InterPodAffinity},
{Name: names.VolumeBinding},
{Name: names.NodeAffinity},
},
},
Filter: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.NodeUnschedulable},
{Name: names.NodeName},
{Name: names.TaintToleration},
{Name: names.NodeAffinity},
{Name: names.NodePorts},
{Name: names.NodeResourcesFit},
{Name: names.VolumeRestrictions},
{Name: names.EBSLimits},
{Name: names.GCEPDLimits},
{Name: names.NodeVolumeLimits},
{Name: names.AzureDiskLimits},
{Name: names.VolumeBinding},
{Name: names.VolumeZone},
{Name: names.PodTopologySpread},
{Name: names.InterPodAffinity},
},
},
PostFilter: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.DefaultPreemption},
},
},
PreScore: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.InterPodAffinity},
{Name: names.PodTopologySpread},
{Name: names.TaintToleration},
{Name: names.NodeAffinity},
{Name: names.SelectorSpread},
},
},
Score: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.NodeResourcesBalancedAllocation, Weight: pointer.Int32Ptr(1)},
{Name: names.ImageLocality, Weight: pointer.Int32Ptr(1)},
{Name: names.InterPodAffinity, Weight: pointer.Int32Ptr(1)},
{Name: names.NodeResourcesLeastAllocated, Weight: pointer.Int32Ptr(1)},
{Name: names.NodeAffinity, Weight: pointer.Int32Ptr(1)},
{Name: names.NodePreferAvoidPods, Weight: pointer.Int32Ptr(10000)},
{Name: names.PodTopologySpread, Weight: pointer.Int32Ptr(2)},
{Name: names.TaintToleration, Weight: pointer.Int32Ptr(1)},
{Name: names.SelectorSpread, Weight: pointer.Int32Ptr(1)},
},
},
Reserve: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.VolumeBinding},
},
},
PreBind: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.VolumeBinding},
},
},
Bind: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.DefaultBinder},
},
},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
for k, v := range test.features {
defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, k, v)()
}
gotConfig := getDefaultPlugins()
if diff := cmp.Diff(test.wantConfig, gotConfig); diff != "" {
t.Errorf("unexpected config diff (-want, +got): %s", diff)
}
})
}
}
func TestMergePlugins(t *testing.T) {
tests := []struct {
name string
customPlugins *v1beta1.Plugins
defaultPlugins *v1beta1.Plugins
expectedPlugins *v1beta1.Plugins
}{
{
name: "AppendCustomPlugin",
customPlugins: &v1beta1.Plugins{
Filter: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: "CustomPlugin"},
},
},
},
defaultPlugins: &v1beta1.Plugins{
Filter: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
},
},
},
expectedPlugins: &v1beta1.Plugins{
Filter: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
{Name: "CustomPlugin"},
},
},
},
},
{
name: "InsertAfterDefaultPlugins2",
customPlugins: &v1beta1.Plugins{
Filter: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: "CustomPlugin"},
{Name: "DefaultPlugin2"},
},
Disabled: []v1beta1.Plugin{
{Name: "DefaultPlugin2"},
},
},
},
defaultPlugins: &v1beta1.Plugins{
Filter: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
},
},
},
expectedPlugins: &v1beta1.Plugins{
Filter: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: "DefaultPlugin1"},
{Name: "CustomPlugin"},
{Name: "DefaultPlugin2"},
},
},
},
},
{
name: "InsertBeforeAllPlugins",
customPlugins: &v1beta1.Plugins{
Filter: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: "CustomPlugin"},
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
},
Disabled: []v1beta1.Plugin{
{Name: "*"},
},
},
},
defaultPlugins: &v1beta1.Plugins{
Filter: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
},
},
},
expectedPlugins: &v1beta1.Plugins{
Filter: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: "CustomPlugin"},
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
},
},
},
},
{
name: "ReorderDefaultPlugins",
customPlugins: &v1beta1.Plugins{
Filter: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: "DefaultPlugin2"},
{Name: "DefaultPlugin1"},
},
Disabled: []v1beta1.Plugin{
{Name: "*"},
},
},
},
defaultPlugins: &v1beta1.Plugins{
Filter: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
},
},
},
expectedPlugins: &v1beta1.Plugins{
Filter: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: "DefaultPlugin2"},
{Name: "DefaultPlugin1"},
},
},
},
},
{
name: "ApplyNilCustomPlugin",
customPlugins: nil,
defaultPlugins: &v1beta1.Plugins{
Filter: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
},
},
},
expectedPlugins: &v1beta1.Plugins{
Filter: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
},
},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
test.defaultPlugins = mergePlugins(test.defaultPlugins, test.customPlugins)
if d := cmp.Diff(test.expectedPlugins, test.defaultPlugins); d != "" {
t.Fatalf("plugins mismatch (-want +got):\n%s", d)
}
})
}
}

View File

@ -22,6 +22,7 @@ import (
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apiserver/pkg/util/feature"
componentbaseconfigv1alpha1 "k8s.io/component-base/config/v1alpha1"
"k8s.io/kube-scheduler/config/v1beta1"
@ -39,9 +40,72 @@ func addDefaultingFuncs(scheme *runtime.Scheme) error {
return RegisterDefaults(scheme)
}
func pluginsNames(p *v1beta1.Plugins) []string {
if p == nil {
return nil
}
extensions := []*v1beta1.PluginSet{
p.PreFilter,
p.Filter,
p.PostFilter,
p.Reserve,
p.PreScore,
p.Score,
p.PreBind,
p.Bind,
p.PostBind,
p.Permit,
p.QueueSort,
}
n := sets.NewString()
for _, e := range extensions {
if e != nil {
for _, pg := range e.Enabled {
n.Insert(pg.Name)
}
}
}
return n.List()
}
func setDefaults_KubeSchedulerProfile(prof *v1beta1.KubeSchedulerProfile) {
// Set default plugins.
prof.Plugins = mergePlugins(getDefaultPlugins(), prof.Plugins)
// Set default plugin configs.
scheme := getPluginArgConversionScheme()
existingConfigs := sets.NewString()
for j := range prof.PluginConfig {
existingConfigs.Insert(prof.PluginConfig[j].Name)
args := prof.PluginConfig[j].Args.Object
if _, isUnknown := args.(*runtime.Unknown); isUnknown {
continue
}
scheme.Default(args)
}
// Append default configs for plugins that didn't have one explicitly set.
for _, name := range pluginsNames(prof.Plugins) {
if existingConfigs.Has(name) {
continue
}
gvk := v1beta1.SchemeGroupVersion.WithKind(name + "Args")
args, err := scheme.New(gvk)
if err != nil {
// This plugin is out-of-tree or doesn't require configuration.
continue
}
scheme.Default(args)
args.GetObjectKind().SetGroupVersionKind(gvk)
prof.PluginConfig = append(prof.PluginConfig, v1beta1.PluginConfig{
Name: name,
Args: runtime.RawExtension{Object: args},
})
}
}
// SetDefaults_KubeSchedulerConfiguration sets additional defaults
func SetDefaults_KubeSchedulerConfiguration(obj *v1beta1.KubeSchedulerConfiguration) {
if obj.Parallelism == nil {
obj.Parallelism = pointer.Int32Ptr(16)
}
@ -55,6 +119,12 @@ func SetDefaults_KubeSchedulerConfiguration(obj *v1beta1.KubeSchedulerConfigurat
obj.Profiles[0].SchedulerName = pointer.StringPtr(corev1.DefaultSchedulerName)
}
// Add the default set of plugins and apply the configuration.
for i := range obj.Profiles {
prof := &obj.Profiles[i]
setDefaults_KubeSchedulerProfile(prof)
}
// For Healthz and Metrics bind addresses, we want to check:
// 1. If the value is nil, default to 0.0.0.0 and default scheduler port
// 2. If there is a value set, attempt to split it. If it's just a port (ie, ":1234"), default to 0.0.0.0 with that port

View File

@ -31,9 +31,84 @@ import (
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/kube-scheduler/config/v1beta1"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/names"
"k8s.io/utils/pointer"
)
var pluginConfigs = []v1beta1.PluginConfig{
{
Name: "DefaultPreemption",
Args: runtime.RawExtension{
Object: &v1beta1.DefaultPreemptionArgs{
TypeMeta: metav1.TypeMeta{
Kind: "DefaultPreemptionArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta1",
},
MinCandidateNodesPercentage: pointer.Int32Ptr(10),
MinCandidateNodesAbsolute: pointer.Int32Ptr(100),
}},
},
{
Name: "InterPodAffinity",
Args: runtime.RawExtension{
Object: &v1beta1.InterPodAffinityArgs{
TypeMeta: metav1.TypeMeta{
Kind: "InterPodAffinityArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta1",
},
HardPodAffinityWeight: pointer.Int32Ptr(1),
}},
},
{
Name: "NodeAffinity",
Args: runtime.RawExtension{Object: &v1beta1.NodeAffinityArgs{
TypeMeta: metav1.TypeMeta{
Kind: "NodeAffinityArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta1",
},
}},
},
{
Name: "NodeResourcesFit",
Args: runtime.RawExtension{Object: &v1beta1.NodeResourcesFitArgs{
TypeMeta: metav1.TypeMeta{
Kind: "NodeResourcesFitArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta1",
},
}},
},
{
Name: "NodeResourcesLeastAllocated",
Args: runtime.RawExtension{Object: &v1beta1.NodeResourcesLeastAllocatedArgs{
TypeMeta: metav1.TypeMeta{
Kind: "NodeResourcesLeastAllocatedArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta1",
},
Resources: []v1beta1.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}},
}},
},
{
Name: "PodTopologySpread",
Args: runtime.RawExtension{Object: &v1beta1.PodTopologySpreadArgs{
TypeMeta: metav1.TypeMeta{
Kind: "PodTopologySpreadArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta1",
},
DefaultingType: v1beta1.SystemDefaulting,
}},
},
{
Name: "VolumeBinding",
Args: runtime.RawExtension{Object: &v1beta1.VolumeBindingArgs{
TypeMeta: metav1.TypeMeta{
Kind: "VolumeBindingArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta1",
},
BindTimeoutSeconds: pointer.Int64Ptr(600),
}},
},
}
func TestSchedulerDefaults(t *testing.T) {
enable := true
tests := []struct {
@ -70,20 +145,18 @@ func TestSchedulerDefaults(t *testing.T) {
PodInitialBackoffSeconds: pointer.Int64Ptr(1),
PodMaxBackoffSeconds: pointer.Int64Ptr(10),
Profiles: []v1beta1.KubeSchedulerProfile{
{SchedulerName: pointer.StringPtr("default-scheduler")},
{
SchedulerName: pointer.StringPtr("default-scheduler"),
Plugins: getDefaultPlugins(),
PluginConfig: pluginConfigs,
},
},
},
},
{
name: "no scheduler name",
config: &v1beta1.KubeSchedulerConfiguration{
Profiles: []v1beta1.KubeSchedulerProfile{
{
PluginConfig: []v1beta1.PluginConfig{
{Name: "FooPlugin"},
},
},
},
Profiles: []v1beta1.KubeSchedulerProfile{{}},
},
expected: &v1beta1.KubeSchedulerConfiguration{
Parallelism: pointer.Int32Ptr(16),
@ -113,9 +186,8 @@ func TestSchedulerDefaults(t *testing.T) {
Profiles: []v1beta1.KubeSchedulerProfile{
{
SchedulerName: pointer.StringPtr("default-scheduler"),
PluginConfig: []v1beta1.PluginConfig{
{Name: "FooPlugin"},
},
Plugins: getDefaultPlugins(),
PluginConfig: pluginConfigs,
},
},
},
@ -169,19 +241,162 @@ func TestSchedulerDefaults(t *testing.T) {
PodMaxBackoffSeconds: pointer.Int64Ptr(10),
Profiles: []v1beta1.KubeSchedulerProfile{
{
Plugins: getDefaultPlugins(),
PluginConfig: []v1beta1.PluginConfig{
{Name: "FooPlugin"},
{
Name: "DefaultPreemption",
Args: runtime.RawExtension{
Object: &v1beta1.DefaultPreemptionArgs{
TypeMeta: metav1.TypeMeta{
Kind: "DefaultPreemptionArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta1",
},
MinCandidateNodesPercentage: pointer.Int32Ptr(10),
MinCandidateNodesAbsolute: pointer.Int32Ptr(100),
}},
},
{
Name: "InterPodAffinity",
Args: runtime.RawExtension{
Object: &v1beta1.InterPodAffinityArgs{
TypeMeta: metav1.TypeMeta{
Kind: "InterPodAffinityArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta1",
},
HardPodAffinityWeight: pointer.Int32Ptr(1),
}},
},
{
Name: "NodeAffinity",
Args: runtime.RawExtension{Object: &v1beta1.NodeAffinityArgs{
TypeMeta: metav1.TypeMeta{
Kind: "NodeAffinityArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta1",
},
}},
},
{
Name: "NodeResourcesFit",
Args: runtime.RawExtension{Object: &v1beta1.NodeResourcesFitArgs{
TypeMeta: metav1.TypeMeta{
Kind: "NodeResourcesFitArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta1",
},
}},
},
{
Name: "NodeResourcesLeastAllocated",
Args: runtime.RawExtension{Object: &v1beta1.NodeResourcesLeastAllocatedArgs{
TypeMeta: metav1.TypeMeta{
Kind: "NodeResourcesLeastAllocatedArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta1",
},
Resources: []v1beta1.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}},
}},
},
{
Name: "PodTopologySpread",
Args: runtime.RawExtension{Object: &v1beta1.PodTopologySpreadArgs{
TypeMeta: metav1.TypeMeta{
Kind: "PodTopologySpreadArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta1",
},
DefaultingType: v1beta1.SystemDefaulting,
}},
},
{
Name: "VolumeBinding",
Args: runtime.RawExtension{Object: &v1beta1.VolumeBindingArgs{
TypeMeta: metav1.TypeMeta{
Kind: "VolumeBindingArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta1",
},
BindTimeoutSeconds: pointer.Int64Ptr(600),
}},
},
},
},
{
SchedulerName: pointer.StringPtr("custom-scheduler"),
Plugins: &v1beta1.Plugins{
QueueSort: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.PrioritySort},
},
},
PreFilter: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.NodeResourcesFit},
{Name: names.NodePorts},
{Name: names.PodTopologySpread},
{Name: names.InterPodAffinity},
{Name: names.VolumeBinding},
{Name: names.NodeAffinity},
},
},
Filter: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.NodeUnschedulable},
{Name: names.NodeName},
{Name: names.TaintToleration},
{Name: names.NodeAffinity},
{Name: names.NodePorts},
{Name: names.NodeResourcesFit},
{Name: names.VolumeRestrictions},
{Name: names.EBSLimits},
{Name: names.GCEPDLimits},
{Name: names.NodeVolumeLimits},
{Name: names.AzureDiskLimits},
{Name: names.VolumeBinding},
{Name: names.VolumeZone},
{Name: names.PodTopologySpread},
{Name: names.InterPodAffinity},
},
},
PostFilter: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.DefaultPreemption},
},
},
PreScore: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.InterPodAffinity},
{Name: names.PodTopologySpread},
{Name: names.TaintToleration},
{Name: names.NodeAffinity},
},
},
Score: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.NodeResourcesBalancedAllocation, Weight: pointer.Int32Ptr(1)},
{Name: names.ImageLocality, Weight: pointer.Int32Ptr(1)},
{Name: names.InterPodAffinity, Weight: pointer.Int32Ptr(1)},
{Name: names.NodeResourcesLeastAllocated, Weight: pointer.Int32Ptr(1)},
{Name: names.NodeAffinity, Weight: pointer.Int32Ptr(1)},
{Name: names.NodePreferAvoidPods, Weight: pointer.Int32Ptr(10000)},
{Name: names.PodTopologySpread, Weight: pointer.Int32Ptr(2)},
{Name: names.TaintToleration, Weight: pointer.Int32Ptr(1)},
},
},
Reserve: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.VolumeBinding},
},
},
PreBind: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.VolumeBinding},
},
},
Bind: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: names.DefaultBinder},
{Name: "BarPlugin"},
},
},
},
PluginConfig: pluginConfigs,
},
},
},
@ -219,7 +434,11 @@ func TestSchedulerDefaults(t *testing.T) {
PodInitialBackoffSeconds: pointer.Int64Ptr(1),
PodMaxBackoffSeconds: pointer.Int64Ptr(10),
Profiles: []v1beta1.KubeSchedulerProfile{
{SchedulerName: pointer.StringPtr("default-scheduler")},
{
SchedulerName: pointer.StringPtr("default-scheduler"),
Plugins: getDefaultPlugins(),
PluginConfig: pluginConfigs,
},
},
},
},
@ -255,7 +474,11 @@ func TestSchedulerDefaults(t *testing.T) {
PodInitialBackoffSeconds: pointer.Int64Ptr(1),
PodMaxBackoffSeconds: pointer.Int64Ptr(10),
Profiles: []v1beta1.KubeSchedulerProfile{
{SchedulerName: pointer.StringPtr("default-scheduler")},
{
SchedulerName: pointer.StringPtr("default-scheduler"),
Plugins: getDefaultPlugins(),
PluginConfig: pluginConfigs,
},
},
},
},
@ -290,7 +513,11 @@ func TestSchedulerDefaults(t *testing.T) {
PodInitialBackoffSeconds: pointer.Int64Ptr(1),
PodMaxBackoffSeconds: pointer.Int64Ptr(10),
Profiles: []v1beta1.KubeSchedulerProfile{
{SchedulerName: pointer.StringPtr("default-scheduler")},
{
SchedulerName: pointer.StringPtr("default-scheduler"),
Plugins: getDefaultPlugins(),
PluginConfig: pluginConfigs,
},
},
},
},

View File

@ -35,7 +35,7 @@ var (
initPluginArgConversionScheme sync.Once
)
func getPluginArgConversionScheme() *runtime.Scheme {
func GetPluginArgConversionScheme() *runtime.Scheme {
initPluginArgConversionScheme.Do(func() {
// set up the scheme used for plugin arg conversion
pluginArgConversionScheme = runtime.NewScheme()
@ -55,22 +55,22 @@ func Convert_v1beta2_KubeSchedulerConfiguration_To_config_KubeSchedulerConfigura
// convertToInternalPluginConfigArgs converts PluginConfig#Args into internal
// types using a scheme, after applying defaults.
func convertToInternalPluginConfigArgs(out *config.KubeSchedulerConfiguration) error {
scheme := getPluginArgConversionScheme()
scheme := GetPluginArgConversionScheme()
for i := range out.Profiles {
for j := range out.Profiles[i].PluginConfig {
args := out.Profiles[i].PluginConfig[j].Args
prof := &out.Profiles[i]
for j := range prof.PluginConfig {
args := prof.PluginConfig[j].Args
if args == nil {
continue
}
if _, isUnknown := args.(*runtime.Unknown); isUnknown {
continue
}
scheme.Default(args)
internalArgs, err := scheme.ConvertToVersion(args, config.SchemeGroupVersion)
if err != nil {
return fmt.Errorf("converting .Profiles[%d].PluginConfig[%d].Args into internal type: %w", i, j, err)
}
out.Profiles[i].PluginConfig[j].Args = internalArgs
prof.PluginConfig[j].Args = internalArgs
}
}
return nil
@ -86,7 +86,7 @@ func Convert_config_KubeSchedulerConfiguration_To_v1beta2_KubeSchedulerConfigura
// convertToExternalPluginConfigArgs converts PluginConfig#Args into
// external (versioned) types using a scheme.
func convertToExternalPluginConfigArgs(out *v1beta2.KubeSchedulerConfiguration) error {
scheme := getPluginArgConversionScheme()
scheme := GetPluginArgConversionScheme()
for i := range out.Profiles {
for j := range out.Profiles[i].PluginConfig {
args := out.Profiles[i].PluginConfig[j].Args

View File

@ -0,0 +1,169 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package v1beta2
import (
"k8s.io/apimachinery/pkg/util/sets"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/klog/v2"
"k8s.io/kube-scheduler/config/v1beta2"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/names"
"k8s.io/utils/pointer"
)
// getDefaultPlugins returns the default set of plugins.
func getDefaultPlugins() *v1beta2.Plugins {
plugins := &v1beta2.Plugins{
QueueSort: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.PrioritySort},
},
},
PreFilter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.NodeResourcesFit},
{Name: names.NodePorts},
{Name: names.PodTopologySpread},
{Name: names.InterPodAffinity},
{Name: names.VolumeBinding},
{Name: names.NodeAffinity},
},
},
Filter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.NodeUnschedulable},
{Name: names.NodeName},
{Name: names.TaintToleration},
{Name: names.NodeAffinity},
{Name: names.NodePorts},
{Name: names.NodeResourcesFit},
{Name: names.VolumeRestrictions},
{Name: names.EBSLimits},
{Name: names.GCEPDLimits},
{Name: names.NodeVolumeLimits},
{Name: names.AzureDiskLimits},
{Name: names.VolumeBinding},
{Name: names.VolumeZone},
{Name: names.PodTopologySpread},
{Name: names.InterPodAffinity},
},
},
PostFilter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.DefaultPreemption},
},
},
PreScore: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.InterPodAffinity},
{Name: names.PodTopologySpread},
{Name: names.TaintToleration},
{Name: names.NodeAffinity},
},
},
Score: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.NodeResourcesBalancedAllocation, Weight: pointer.Int32Ptr(1)},
{Name: names.ImageLocality, Weight: pointer.Int32Ptr(1)},
{Name: names.InterPodAffinity, Weight: pointer.Int32Ptr(1)},
{Name: names.NodeResourcesLeastAllocated, Weight: pointer.Int32Ptr(1)},
{Name: names.NodeAffinity, Weight: pointer.Int32Ptr(1)},
// Weight is doubled because:
// - This is a score coming from user preference.
// - It makes its signal comparable to NodeResourcesLeastAllocated.
{Name: names.PodTopologySpread, Weight: pointer.Int32Ptr(2)},
{Name: names.TaintToleration, Weight: pointer.Int32Ptr(1)},
},
},
Reserve: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.VolumeBinding},
},
},
PreBind: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.VolumeBinding},
},
},
Bind: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.DefaultBinder},
},
},
}
applyFeatureGates(plugins)
return plugins
}
func applyFeatureGates(config *v1beta2.Plugins) {
if utilfeature.DefaultFeatureGate.Enabled(features.VolumeCapacityPriority) {
config.Score.Enabled = append(config.Score.Enabled, v1beta2.Plugin{Name: names.VolumeBinding, Weight: pointer.Int32Ptr(1)})
}
if !utilfeature.DefaultFeatureGate.Enabled(features.DefaultPodTopologySpread) {
// When feature is enabled, the default spreading is done by
// PodTopologySpread plugin, which is enabled by default.
klog.InfoS("Registering SelectorSpread plugin")
s := v1beta2.Plugin{Name: names.SelectorSpread}
config.PreScore.Enabled = append(config.PreScore.Enabled, s)
s.Weight = pointer.Int32Ptr(1)
config.Score.Enabled = append(config.Score.Enabled, s)
}
}
// mergePlugins merges the custom set into the given default one, handling disabled sets.
func mergePlugins(defaultPlugins, customPlugins *v1beta2.Plugins) *v1beta2.Plugins {
if customPlugins == nil {
return defaultPlugins
}
defaultPlugins.QueueSort = mergePluginSet(defaultPlugins.QueueSort, customPlugins.QueueSort)
defaultPlugins.PreFilter = mergePluginSet(defaultPlugins.PreFilter, customPlugins.PreFilter)
defaultPlugins.Filter = mergePluginSet(defaultPlugins.Filter, customPlugins.Filter)
defaultPlugins.PostFilter = mergePluginSet(defaultPlugins.PostFilter, customPlugins.PostFilter)
defaultPlugins.PreScore = mergePluginSet(defaultPlugins.PreScore, customPlugins.PreScore)
defaultPlugins.Score = mergePluginSet(defaultPlugins.Score, customPlugins.Score)
defaultPlugins.Reserve = mergePluginSet(defaultPlugins.Reserve, customPlugins.Reserve)
defaultPlugins.Permit = mergePluginSet(defaultPlugins.Permit, customPlugins.Permit)
defaultPlugins.PreBind = mergePluginSet(defaultPlugins.PreBind, customPlugins.PreBind)
defaultPlugins.Bind = mergePluginSet(defaultPlugins.Bind, customPlugins.Bind)
defaultPlugins.PostBind = mergePluginSet(defaultPlugins.PostBind, customPlugins.PostBind)
return defaultPlugins
}
func mergePluginSet(defaultPluginSet, customPluginSet v1beta2.PluginSet) v1beta2.PluginSet {
disabledPlugins := sets.NewString()
for _, disabledPlugin := range customPluginSet.Disabled {
disabledPlugins.Insert(disabledPlugin.Name)
}
var enabledPlugins []v1beta2.Plugin
if !disabledPlugins.Has("*") {
for _, defaultEnabledPlugin := range defaultPluginSet.Enabled {
if disabledPlugins.Has(defaultEnabledPlugin.Name) {
continue
}
enabledPlugins = append(enabledPlugins, defaultEnabledPlugin)
}
}
enabledPlugins = append(enabledPlugins, customPluginSet.Enabled...)
return v1beta2.PluginSet{Enabled: enabledPlugins}
}

View File

@ -0,0 +1,372 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package v1beta2
import (
"testing"
"github.com/google/go-cmp/cmp"
"k8s.io/apiserver/pkg/util/feature"
"k8s.io/component-base/featuregate"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/kube-scheduler/config/v1beta2"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/names"
"k8s.io/utils/pointer"
)
func TestApplyFeatureGates(t *testing.T) {
tests := []struct {
name string
features map[featuregate.Feature]bool
wantConfig *v1beta2.Plugins
}{
{
name: "Feature gates disabled",
wantConfig: &v1beta2.Plugins{
QueueSort: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.PrioritySort},
},
},
PreFilter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.NodeResourcesFit},
{Name: names.NodePorts},
{Name: names.PodTopologySpread},
{Name: names.InterPodAffinity},
{Name: names.VolumeBinding},
{Name: names.NodeAffinity},
},
},
Filter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.NodeUnschedulable},
{Name: names.NodeName},
{Name: names.TaintToleration},
{Name: names.NodeAffinity},
{Name: names.NodePorts},
{Name: names.NodeResourcesFit},
{Name: names.VolumeRestrictions},
{Name: names.EBSLimits},
{Name: names.GCEPDLimits},
{Name: names.NodeVolumeLimits},
{Name: names.AzureDiskLimits},
{Name: names.VolumeBinding},
{Name: names.VolumeZone},
{Name: names.PodTopologySpread},
{Name: names.InterPodAffinity},
},
},
PostFilter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.DefaultPreemption},
},
},
PreScore: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.InterPodAffinity},
{Name: names.PodTopologySpread},
{Name: names.TaintToleration},
{Name: names.NodeAffinity},
},
},
Score: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.NodeResourcesBalancedAllocation, Weight: pointer.Int32Ptr(1)},
{Name: names.ImageLocality, Weight: pointer.Int32Ptr(1)},
{Name: names.InterPodAffinity, Weight: pointer.Int32Ptr(1)},
{Name: names.NodeResourcesLeastAllocated, Weight: pointer.Int32Ptr(1)},
{Name: names.NodeAffinity, Weight: pointer.Int32Ptr(1)},
{Name: names.PodTopologySpread, Weight: pointer.Int32Ptr(2)},
{Name: names.TaintToleration, Weight: pointer.Int32Ptr(1)},
},
},
Reserve: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.VolumeBinding},
},
},
PreBind: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.VolumeBinding},
},
},
Bind: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.DefaultBinder},
},
},
},
},
{
name: "DefaultPodTopologySpread disabled",
features: map[featuregate.Feature]bool{
features.DefaultPodTopologySpread: false,
},
wantConfig: &v1beta2.Plugins{
QueueSort: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.PrioritySort},
},
},
PreFilter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.NodeResourcesFit},
{Name: names.NodePorts},
{Name: names.PodTopologySpread},
{Name: names.InterPodAffinity},
{Name: names.VolumeBinding},
{Name: names.NodeAffinity},
},
},
Filter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.NodeUnschedulable},
{Name: names.NodeName},
{Name: names.TaintToleration},
{Name: names.NodeAffinity},
{Name: names.NodePorts},
{Name: names.NodeResourcesFit},
{Name: names.VolumeRestrictions},
{Name: names.EBSLimits},
{Name: names.GCEPDLimits},
{Name: names.NodeVolumeLimits},
{Name: names.AzureDiskLimits},
{Name: names.VolumeBinding},
{Name: names.VolumeZone},
{Name: names.PodTopologySpread},
{Name: names.InterPodAffinity},
},
},
PostFilter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.DefaultPreemption},
},
},
PreScore: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.InterPodAffinity},
{Name: names.PodTopologySpread},
{Name: names.TaintToleration},
{Name: names.NodeAffinity},
{Name: names.SelectorSpread},
},
},
Score: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.NodeResourcesBalancedAllocation, Weight: pointer.Int32Ptr(1)},
{Name: names.ImageLocality, Weight: pointer.Int32Ptr(1)},
{Name: names.InterPodAffinity, Weight: pointer.Int32Ptr(1)},
{Name: names.NodeResourcesLeastAllocated, Weight: pointer.Int32Ptr(1)},
{Name: names.NodeAffinity, Weight: pointer.Int32Ptr(1)},
{Name: names.PodTopologySpread, Weight: pointer.Int32Ptr(2)},
{Name: names.TaintToleration, Weight: pointer.Int32Ptr(1)},
{Name: names.SelectorSpread, Weight: pointer.Int32Ptr(1)},
},
},
Reserve: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.VolumeBinding},
},
},
PreBind: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.VolumeBinding},
},
},
Bind: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.DefaultBinder},
},
},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
for k, v := range test.features {
defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, k, v)()
}
gotConfig := getDefaultPlugins()
if diff := cmp.Diff(test.wantConfig, gotConfig); diff != "" {
t.Errorf("unexpected config diff (-want, +got): %s", diff)
}
})
}
}
func TestMergePlugins(t *testing.T) {
tests := []struct {
name string
customPlugins *v1beta2.Plugins
defaultPlugins *v1beta2.Plugins
expectedPlugins *v1beta2.Plugins
}{
{
name: "AppendCustomPlugin",
customPlugins: &v1beta2.Plugins{
Filter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: "CustomPlugin"},
},
},
},
defaultPlugins: &v1beta2.Plugins{
Filter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
},
},
},
expectedPlugins: &v1beta2.Plugins{
Filter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
{Name: "CustomPlugin"},
},
},
},
},
{
name: "InsertAfterDefaultPlugins2",
customPlugins: &v1beta2.Plugins{
Filter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: "CustomPlugin"},
{Name: "DefaultPlugin2"},
},
Disabled: []v1beta2.Plugin{
{Name: "DefaultPlugin2"},
},
},
},
defaultPlugins: &v1beta2.Plugins{
Filter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
},
},
},
expectedPlugins: &v1beta2.Plugins{
Filter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: "DefaultPlugin1"},
{Name: "CustomPlugin"},
{Name: "DefaultPlugin2"},
},
},
},
},
{
name: "InsertBeforeAllPlugins",
customPlugins: &v1beta2.Plugins{
Filter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: "CustomPlugin"},
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
},
Disabled: []v1beta2.Plugin{
{Name: "*"},
},
},
},
defaultPlugins: &v1beta2.Plugins{
Filter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
},
},
},
expectedPlugins: &v1beta2.Plugins{
Filter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: "CustomPlugin"},
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
},
},
},
},
{
name: "ReorderDefaultPlugins",
customPlugins: &v1beta2.Plugins{
Filter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: "DefaultPlugin2"},
{Name: "DefaultPlugin1"},
},
Disabled: []v1beta2.Plugin{
{Name: "*"},
},
},
},
defaultPlugins: &v1beta2.Plugins{
Filter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
},
},
},
expectedPlugins: &v1beta2.Plugins{
Filter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: "DefaultPlugin2"},
{Name: "DefaultPlugin1"},
},
},
},
},
{
name: "ApplyNilCustomPlugin",
customPlugins: nil,
defaultPlugins: &v1beta2.Plugins{
Filter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
},
},
},
expectedPlugins: &v1beta2.Plugins{
Filter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
},
},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
test.defaultPlugins = mergePlugins(test.defaultPlugins, test.customPlugins)
if d := cmp.Diff(test.expectedPlugins, test.defaultPlugins); d != "" {
t.Fatalf("plugins mismatch (-want +got):\n%s", d)
}
})
}
}

View File

@ -23,6 +23,7 @@ import (
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apiserver/pkg/util/feature"
componentbaseconfigv1alpha1 "k8s.io/component-base/config/v1alpha1"
"k8s.io/kube-scheduler/config/v1beta2"
@ -40,9 +41,70 @@ func addDefaultingFuncs(scheme *runtime.Scheme) error {
return RegisterDefaults(scheme)
}
func pluginsNames(p *v1beta2.Plugins) []string {
if p == nil {
return nil
}
extensions := []v1beta2.PluginSet{
p.PreFilter,
p.Filter,
p.PostFilter,
p.Reserve,
p.PreScore,
p.Score,
p.PreBind,
p.Bind,
p.PostBind,
p.Permit,
p.QueueSort,
}
n := sets.NewString()
for _, e := range extensions {
for _, pg := range e.Enabled {
n.Insert(pg.Name)
}
}
return n.List()
}
func setDefaults_KubeSchedulerProfile(prof *v1beta2.KubeSchedulerProfile) {
// Set default plugins.
prof.Plugins = mergePlugins(getDefaultPlugins(), prof.Plugins)
// Set default plugin configs.
scheme := GetPluginArgConversionScheme()
existingConfigs := sets.NewString()
for j := range prof.PluginConfig {
existingConfigs.Insert(prof.PluginConfig[j].Name)
args := prof.PluginConfig[j].Args.Object
if _, isUnknown := args.(*runtime.Unknown); isUnknown {
continue
}
scheme.Default(args)
}
// Append default configs for plugins that didn't have one explicitly set.
for _, name := range pluginsNames(prof.Plugins) {
if existingConfigs.Has(name) {
continue
}
gvk := v1beta2.SchemeGroupVersion.WithKind(name + "Args")
args, err := scheme.New(gvk)
if err != nil {
// This plugin is out-of-tree or doesn't require configuration.
continue
}
scheme.Default(args)
args.GetObjectKind().SetGroupVersionKind(gvk)
prof.PluginConfig = append(prof.PluginConfig, v1beta2.PluginConfig{
Name: name,
Args: runtime.RawExtension{Object: args},
})
}
}
// SetDefaults_KubeSchedulerConfiguration sets additional defaults
func SetDefaults_KubeSchedulerConfiguration(obj *v1beta2.KubeSchedulerConfiguration) {
if obj.Parallelism == nil {
obj.Parallelism = pointer.Int32Ptr(16)
}
@ -56,6 +118,12 @@ func SetDefaults_KubeSchedulerConfiguration(obj *v1beta2.KubeSchedulerConfigurat
obj.Profiles[0].SchedulerName = pointer.StringPtr(v1.DefaultSchedulerName)
}
// Add the default set of plugins and apply the configuration.
for i := range obj.Profiles {
prof := &obj.Profiles[i]
setDefaults_KubeSchedulerProfile(prof)
}
// For Healthz and Metrics bind addresses, we want to check:
// 1. If the value is nil, default to 0.0.0.0 and default scheduler port
// 2. If there is a value set, attempt to split it. If it's just a port (ie, ":1234"), default to 0.0.0.0 with that port

View File

@ -21,6 +21,7 @@ import (
"time"
"github.com/google/go-cmp/cmp"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
@ -31,9 +32,84 @@ import (
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/kube-scheduler/config/v1beta2"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/names"
"k8s.io/utils/pointer"
)
var pluginConfigs = []v1beta2.PluginConfig{
{
Name: "DefaultPreemption",
Args: runtime.RawExtension{
Object: &v1beta2.DefaultPreemptionArgs{
TypeMeta: metav1.TypeMeta{
Kind: "DefaultPreemptionArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta2",
},
MinCandidateNodesPercentage: pointer.Int32Ptr(10),
MinCandidateNodesAbsolute: pointer.Int32Ptr(100),
}},
},
{
Name: "InterPodAffinity",
Args: runtime.RawExtension{
Object: &v1beta2.InterPodAffinityArgs{
TypeMeta: metav1.TypeMeta{
Kind: "InterPodAffinityArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta2",
},
HardPodAffinityWeight: pointer.Int32Ptr(1),
}},
},
{
Name: "NodeAffinity",
Args: runtime.RawExtension{Object: &v1beta2.NodeAffinityArgs{
TypeMeta: metav1.TypeMeta{
Kind: "NodeAffinityArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta2",
},
}},
},
{
Name: "NodeResourcesFit",
Args: runtime.RawExtension{Object: &v1beta2.NodeResourcesFitArgs{
TypeMeta: metav1.TypeMeta{
Kind: "NodeResourcesFitArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta2",
},
}},
},
{
Name: "NodeResourcesLeastAllocated",
Args: runtime.RawExtension{Object: &v1beta2.NodeResourcesLeastAllocatedArgs{
TypeMeta: metav1.TypeMeta{
Kind: "NodeResourcesLeastAllocatedArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta2",
},
Resources: []v1beta2.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}},
}},
},
{
Name: "PodTopologySpread",
Args: runtime.RawExtension{Object: &v1beta2.PodTopologySpreadArgs{
TypeMeta: metav1.TypeMeta{
Kind: "PodTopologySpreadArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta2",
},
DefaultingType: v1beta2.SystemDefaulting,
}},
},
{
Name: "VolumeBinding",
Args: runtime.RawExtension{Object: &v1beta2.VolumeBindingArgs{
TypeMeta: metav1.TypeMeta{
Kind: "VolumeBindingArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta2",
},
BindTimeoutSeconds: pointer.Int64Ptr(600),
}},
},
}
func TestSchedulerDefaults(t *testing.T) {
enable := true
tests := []struct {
@ -70,20 +146,18 @@ func TestSchedulerDefaults(t *testing.T) {
PodInitialBackoffSeconds: pointer.Int64Ptr(1),
PodMaxBackoffSeconds: pointer.Int64Ptr(10),
Profiles: []v1beta2.KubeSchedulerProfile{
{SchedulerName: pointer.StringPtr("default-scheduler")},
{
Plugins: getDefaultPlugins(),
PluginConfig: pluginConfigs,
SchedulerName: pointer.StringPtr("default-scheduler"),
},
},
},
},
{
name: "no scheduler name",
config: &v1beta2.KubeSchedulerConfiguration{
Profiles: []v1beta2.KubeSchedulerProfile{
{
PluginConfig: []v1beta2.PluginConfig{
{Name: "FooPlugin"},
},
},
},
Profiles: []v1beta2.KubeSchedulerProfile{{}},
},
expected: &v1beta2.KubeSchedulerConfiguration{
Parallelism: pointer.Int32Ptr(16),
@ -113,10 +187,8 @@ func TestSchedulerDefaults(t *testing.T) {
Profiles: []v1beta2.KubeSchedulerProfile{
{
SchedulerName: pointer.StringPtr("default-scheduler"),
PluginConfig: []v1beta2.PluginConfig{
{Name: "FooPlugin"},
},
},
Plugins: getDefaultPlugins(),
PluginConfig: pluginConfigs},
},
},
},
@ -137,6 +209,9 @@ func TestSchedulerDefaults(t *testing.T) {
Enabled: []v1beta2.Plugin{
{Name: "BarPlugin"},
},
Disabled: []v1beta2.Plugin{
{Name: names.DefaultBinder},
},
},
},
},
@ -169,19 +244,160 @@ func TestSchedulerDefaults(t *testing.T) {
PodMaxBackoffSeconds: pointer.Int64Ptr(10),
Profiles: []v1beta2.KubeSchedulerProfile{
{
Plugins: getDefaultPlugins(),
PluginConfig: []v1beta2.PluginConfig{
{Name: "FooPlugin"},
{
Name: "DefaultPreemption",
Args: runtime.RawExtension{
Object: &v1beta2.DefaultPreemptionArgs{
TypeMeta: metav1.TypeMeta{
Kind: "DefaultPreemptionArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta2",
},
MinCandidateNodesPercentage: pointer.Int32Ptr(10),
MinCandidateNodesAbsolute: pointer.Int32Ptr(100),
}},
},
{
Name: "InterPodAffinity",
Args: runtime.RawExtension{
Object: &v1beta2.InterPodAffinityArgs{
TypeMeta: metav1.TypeMeta{
Kind: "InterPodAffinityArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta2",
},
HardPodAffinityWeight: pointer.Int32Ptr(1),
}},
},
{
Name: "NodeAffinity",
Args: runtime.RawExtension{Object: &v1beta2.NodeAffinityArgs{
TypeMeta: metav1.TypeMeta{
Kind: "NodeAffinityArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta2",
},
}},
},
{
Name: "NodeResourcesFit",
Args: runtime.RawExtension{Object: &v1beta2.NodeResourcesFitArgs{
TypeMeta: metav1.TypeMeta{
Kind: "NodeResourcesFitArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta2",
},
}},
},
{
Name: "NodeResourcesLeastAllocated",
Args: runtime.RawExtension{Object: &v1beta2.NodeResourcesLeastAllocatedArgs{
TypeMeta: metav1.TypeMeta{
Kind: "NodeResourcesLeastAllocatedArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta2",
},
Resources: []v1beta2.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}},
}},
},
{
Name: "PodTopologySpread",
Args: runtime.RawExtension{Object: &v1beta2.PodTopologySpreadArgs{
TypeMeta: metav1.TypeMeta{
Kind: "PodTopologySpreadArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta2",
},
DefaultingType: v1beta2.SystemDefaulting,
}},
},
{
Name: "VolumeBinding",
Args: runtime.RawExtension{Object: &v1beta2.VolumeBindingArgs{
TypeMeta: metav1.TypeMeta{
Kind: "VolumeBindingArgs",
APIVersion: "kubescheduler.config.k8s.io/v1beta2",
},
BindTimeoutSeconds: pointer.Int64Ptr(600),
}},
},
},
},
{
SchedulerName: pointer.StringPtr("custom-scheduler"),
Plugins: &v1beta2.Plugins{
QueueSort: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.PrioritySort},
},
},
PreFilter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.NodeResourcesFit},
{Name: names.NodePorts},
{Name: names.PodTopologySpread},
{Name: names.InterPodAffinity},
{Name: names.VolumeBinding},
{Name: names.NodeAffinity},
},
},
Filter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.NodeUnschedulable},
{Name: names.NodeName},
{Name: names.TaintToleration},
{Name: names.NodeAffinity},
{Name: names.NodePorts},
{Name: names.NodeResourcesFit},
{Name: names.VolumeRestrictions},
{Name: names.EBSLimits},
{Name: names.GCEPDLimits},
{Name: names.NodeVolumeLimits},
{Name: names.AzureDiskLimits},
{Name: names.VolumeBinding},
{Name: names.VolumeZone},
{Name: names.PodTopologySpread},
{Name: names.InterPodAffinity},
},
},
PostFilter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.DefaultPreemption},
},
},
PreScore: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.InterPodAffinity},
{Name: names.PodTopologySpread},
{Name: names.TaintToleration},
{Name: names.NodeAffinity},
},
},
Score: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.NodeResourcesBalancedAllocation, Weight: pointer.Int32Ptr(1)},
{Name: names.ImageLocality, Weight: pointer.Int32Ptr(1)},
{Name: names.InterPodAffinity, Weight: pointer.Int32Ptr(1)},
{Name: names.NodeResourcesLeastAllocated, Weight: pointer.Int32Ptr(1)},
{Name: names.NodeAffinity, Weight: pointer.Int32Ptr(1)},
{Name: names.PodTopologySpread, Weight: pointer.Int32Ptr(2)},
{Name: names.TaintToleration, Weight: pointer.Int32Ptr(1)},
},
},
Reserve: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.VolumeBinding},
},
},
PreBind: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: names.VolumeBinding},
},
},
Bind: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: "BarPlugin"},
},
},
},
PluginConfig: pluginConfigs,
},
},
},
@ -219,7 +435,11 @@ func TestSchedulerDefaults(t *testing.T) {
PodInitialBackoffSeconds: pointer.Int64Ptr(1),
PodMaxBackoffSeconds: pointer.Int64Ptr(10),
Profiles: []v1beta2.KubeSchedulerProfile{
{SchedulerName: pointer.StringPtr("default-scheduler")},
{
Plugins: getDefaultPlugins(),
PluginConfig: pluginConfigs,
SchedulerName: pointer.StringPtr("default-scheduler"),
},
},
},
},
@ -255,7 +475,11 @@ func TestSchedulerDefaults(t *testing.T) {
PodInitialBackoffSeconds: pointer.Int64Ptr(1),
PodMaxBackoffSeconds: pointer.Int64Ptr(10),
Profiles: []v1beta2.KubeSchedulerProfile{
{SchedulerName: pointer.StringPtr("default-scheduler")},
{
Plugins: getDefaultPlugins(),
PluginConfig: pluginConfigs,
SchedulerName: pointer.StringPtr("default-scheduler"),
},
},
},
},
@ -291,7 +515,11 @@ func TestSchedulerDefaults(t *testing.T) {
PodInitialBackoffSeconds: pointer.Int64Ptr(1),
PodMaxBackoffSeconds: pointer.Int64Ptr(10),
Profiles: []v1beta2.KubeSchedulerProfile{
{SchedulerName: pointer.StringPtr("default-scheduler")},
{
Plugins: getDefaultPlugins(),
PluginConfig: pluginConfigs,
SchedulerName: pointer.StringPtr("default-scheduler"),
},
},
},
},
@ -326,7 +554,11 @@ func TestSchedulerDefaults(t *testing.T) {
PodInitialBackoffSeconds: pointer.Int64Ptr(1),
PodMaxBackoffSeconds: pointer.Int64Ptr(10),
Profiles: []v1beta2.KubeSchedulerProfile{
{SchedulerName: pointer.StringPtr("default-scheduler")},
{
Plugins: getDefaultPlugins(),
PluginConfig: pluginConfigs,
SchedulerName: pointer.StringPtr("default-scheduler"),
},
},
},
},

View File

@ -33,8 +33,8 @@ import (
corelisters "k8s.io/client-go/listers/core/v1"
restclient "k8s.io/client-go/rest"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/scheduler/algorithmprovider"
schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config"
"k8s.io/kubernetes/pkg/scheduler/apis/config/v1beta2"
"k8s.io/kubernetes/pkg/scheduler/apis/config/validation"
"k8s.io/kubernetes/pkg/scheduler/core"
"k8s.io/kubernetes/pkg/scheduler/framework"
@ -120,20 +120,30 @@ func (c *Configurator) create() (*Scheduler, error) {
}
// If there are any extended resources found from the Extenders, append them to the pluginConfig for each profile.
// This should only have an effect on ComponentConfig v1beta1, where it is possible to configure Extenders and
// This should only have an effect on ComponentConfig, where it is possible to configure Extenders and
// plugin args (and in which case the extender ignored resources take precedence).
// For earlier versions, using both policy and custom plugin config is disallowed, so this should be the only
// plugin config for this plugin.
if len(ignoredExtendedResources) > 0 {
for i := range c.profiles {
prof := &c.profiles[i]
pc := schedulerapi.PluginConfig{
Name: noderesources.FitName,
Args: &schedulerapi.NodeResourcesFitArgs{
IgnoredResources: ignoredExtendedResources,
},
var found = false
for k := range prof.PluginConfig {
if prof.PluginConfig[k].Name == noderesources.FitName {
// Update the existing args
pc := &prof.PluginConfig[k]
args, ok := pc.Args.(*schedulerapi.NodeResourcesFitArgs)
if !ok {
return nil, fmt.Errorf("want args to be of type NodeResourcesFitArgs, got %T", pc.Args)
}
args.IgnoredResources = ignoredExtendedResources
found = true
break
}
}
if !found {
return nil, fmt.Errorf("can't find NodeResourcesFitArgs in plugin config")
}
prof.PluginConfig = append(prof.PluginConfig, pc)
}
}
@ -195,21 +205,7 @@ func (c *Configurator) create() (*Scheduler, error) {
}, nil
}
// createFromConfig creates a scheduler from ComonentConfig profiles.
func (c *Configurator) createFromConfig() (*Scheduler, error) {
defaultPlugins := algorithmprovider.GetDefaultConfig()
for i := range c.profiles {
prof := &c.profiles[i]
plugins := &schedulerapi.Plugins{}
plugins.Append(defaultPlugins)
plugins.Apply(prof.Plugins)
prof.Plugins = plugins
}
return c.create()
}
// createFromPolicy creates a scheduler from the legacy policy file
// createFromPolicy creates a scheduler from the legacy policy file.
func (c *Configurator) createFromPolicy(policy schedulerapi.Policy) (*Scheduler, error) {
lr := frameworkplugins.NewLegacyRegistry()
args := &frameworkplugins.ConfigProducerArgs{}
@ -221,6 +217,12 @@ func (c *Configurator) createFromPolicy(policy schedulerapi.Policy) (*Scheduler,
return nil, err
}
// If profiles is already set, it means the user is using both CC and policy config, error out
// since these configs are no longer merged and they should not be used simultaneously.
if c.profiles != nil {
return nil, fmt.Errorf("profiles and policy config both set, this should not happen")
}
predicateKeys := sets.NewString()
if policy.Predicates == nil {
predicateKeys = lr.DefaultPredicates
@ -297,17 +299,59 @@ func (c *Configurator) createFromPolicy(policy schedulerapi.Policy) (*Scheduler,
if pluginConfig, err = dedupPluginConfigs(pluginConfig); err != nil {
return nil, err
}
for i := range c.profiles {
prof := &c.profiles[i]
// Plugins and PluginConfig are empty when using Policy; overriding.
prof.Plugins = &schedulerapi.Plugins{}
prof.Plugins.Append(&plugins)
prof.PluginConfig = pluginConfig
c.profiles = []schedulerapi.KubeSchedulerProfile{
{
SchedulerName: v1.DefaultSchedulerName,
Plugins: &plugins,
PluginConfig: pluginConfig,
},
}
if err := defaultPluginConfigArgs(&c.profiles[0]); err != nil {
return nil, err
}
return c.create()
}
func defaultPluginConfigArgs(prof *schedulerapi.KubeSchedulerProfile) error {
scheme := v1beta2.GetPluginArgConversionScheme()
existingConfigs := sets.NewString()
for j := range prof.PluginConfig {
existingConfigs.Insert(prof.PluginConfig[j].Name)
// For existing plugin configs, we don't apply any defaulting, the assumption
// is that the legacy registry does it already.
}
// Append default configs for plugins that didn't have one explicitly set.
for _, name := range prof.Plugins.Names() {
if existingConfigs.Has(name) {
continue
}
gvk := v1beta2.SchemeGroupVersion.WithKind(name + "Args")
args, err := scheme.New(gvk)
if err != nil {
if runtime.IsNotRegisteredError(err) {
// This plugin is out-of-tree or doesn't require configuration.
continue
}
return err
}
scheme.Default(args)
internalArgs, err := scheme.ConvertToVersion(args, schedulerapi.SchemeGroupVersion)
if err != nil {
return fmt.Errorf("converting %q into internal type: %w", gvk.Kind, err)
}
prof.PluginConfig = append(prof.PluginConfig, schedulerapi.PluginConfig{
Name: name,
Args: internalArgs,
})
}
return nil
}
// dedupPluginConfigs removes duplicates from pluginConfig, ensuring that,
// if a plugin name is repeated, the arguments are the same.
func dedupPluginConfigs(pc []schedulerapi.PluginConfig) ([]schedulerapi.PluginConfig, error) {

View File

@ -64,7 +64,7 @@ func TestCreate(t *testing.T) {
stopCh := make(chan struct{})
defer close(stopCh)
factory := newConfigFactory(client, stopCh)
if _, err := factory.createFromConfig(); err != nil {
if _, err := factory.create(); err != nil {
t.Error(err)
}
}
@ -402,6 +402,7 @@ func TestCreateFromConfig(t *testing.T) {
informerFactory,
recorderFactory,
make(chan struct{}),
WithProfiles([]schedulerapi.KubeSchedulerProfile(nil)...),
WithLegacyPolicySource(createPolicySource(tc.configData, client)),
WithBuildFrameworkCapturer(func(p schedulerapi.KubeSchedulerProfile) {
if p.SchedulerName != v1.DefaultSchedulerName {
@ -641,7 +642,13 @@ func newConfigFactoryWithFrameworkRegistry(
StopEverything: stopCh,
registry: registry,
profiles: []schedulerapi.KubeSchedulerProfile{
{SchedulerName: testSchedulerName},
{
SchedulerName: testSchedulerName,
Plugins: &schedulerapi.Plugins{
QueueSort: schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "PrioritySort"}}},
Bind: schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "DefaultBinder"}}},
},
},
},
recorderFactory: recorderFactory,
nodeInfoSnapshot: snapshot,

View File

@ -525,7 +525,7 @@ type Framework interface {
HasScorePlugins() bool
// ListPlugins returns a map of extension point name to list of configured Plugins.
ListPlugins() map[string][]config.Plugin
ListPlugins() *config.Plugins
// ProfileName returns the profile name associated to this framework.
ProfileName() string

View File

@ -25,7 +25,6 @@ import (
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/client-go/informers"
@ -34,10 +33,7 @@ import (
"k8s.io/client-go/tools/events"
"k8s.io/component-helpers/scheduling/corev1"
"k8s.io/klog/v2"
"k8s.io/kube-scheduler/config/v1beta1"
"k8s.io/kube-scheduler/config/v1beta2"
"k8s.io/kubernetes/pkg/scheduler/apis/config"
"k8s.io/kubernetes/pkg/scheduler/apis/config/scheme"
"k8s.io/kubernetes/pkg/scheduler/framework"
"k8s.io/kubernetes/pkg/scheduler/internal/parallelize"
"k8s.io/kubernetes/pkg/scheduler/metrics"
@ -72,8 +68,6 @@ var allClusterEvents = []framework.ClusterEvent{
{Resource: framework.StorageClass, ActionType: framework.All},
}
var configDecoder = scheme.Codecs.UniversalDecoder()
// frameworkImpl is the component responsible for initializing and running scheduler
// plugins.
type frameworkImpl struct {
@ -116,7 +110,7 @@ type frameworkImpl struct {
// frameworkImpl.
type extensionPoint struct {
// the set of plugins to be configured at this extension point.
plugins config.PluginSet
plugins *config.PluginSet
// a pointer to the slice storing plugins implementations that will run at this
// extension point.
slicePtr interface{}
@ -124,17 +118,17 @@ type extensionPoint struct {
func (f *frameworkImpl) getExtensionPoints(plugins *config.Plugins) []extensionPoint {
return []extensionPoint{
{plugins.PreFilter, &f.preFilterPlugins},
{plugins.Filter, &f.filterPlugins},
{plugins.PostFilter, &f.postFilterPlugins},
{plugins.Reserve, &f.reservePlugins},
{plugins.PreScore, &f.preScorePlugins},
{plugins.Score, &f.scorePlugins},
{plugins.PreBind, &f.preBindPlugins},
{plugins.Bind, &f.bindPlugins},
{plugins.PostBind, &f.postBindPlugins},
{plugins.Permit, &f.permitPlugins},
{plugins.QueueSort, &f.queueSortPlugins},
{&plugins.PreFilter, &f.preFilterPlugins},
{&plugins.Filter, &f.filterPlugins},
{&plugins.PostFilter, &f.postFilterPlugins},
{&plugins.Reserve, &f.reservePlugins},
{&plugins.PreScore, &f.preScorePlugins},
{&plugins.Score, &f.scorePlugins},
{&plugins.PreBind, &f.preBindPlugins},
{&plugins.Bind, &f.bindPlugins},
{&plugins.PostBind, &f.postBindPlugins},
{&plugins.Permit, &f.permitPlugins},
{&plugins.QueueSort, &f.queueSortPlugins},
}
}
@ -335,10 +329,7 @@ func NewFramework(r Registry, profile *config.KubeSchedulerProfile, opts ...Opti
continue
}
args, err := getPluginArgsOrDefault(options.componentConfigVersion, pluginConfig, name)
if err != nil {
return nil, fmt.Errorf("getting args for Plugin %q: %w", name, err)
}
args := pluginConfig[name]
if args != nil {
outputProfile.PluginConfig = append(outputProfile.PluginConfig, config.PluginConfig{
Name: name,
@ -356,7 +347,7 @@ func NewFramework(r Registry, profile *config.KubeSchedulerProfile, opts ...Opti
}
for _, e := range f.getExtensionPoints(profile.Plugins) {
if err := updatePluginList(e.slicePtr, e.plugins, pluginsMap); err != nil {
if err := updatePluginList(e.slicePtr, *e.plugins, pluginsMap); err != nil {
return nil, err
}
}
@ -424,34 +415,6 @@ func registerClusterEvents(name string, eventToPlugins map[framework.ClusterEven
}
}
// getPluginArgsOrDefault returns a configuration provided by the user or builds
// a default from the scheme. Returns `nil, nil` if the plugin does not have a
// defined arg types, such as in-tree plugins that don't require configuration
// or out-of-tree plugins.
func getPluginArgsOrDefault(componentConfigVersion string, pluginConfig map[string]runtime.Object, name string) (runtime.Object, error) {
res, ok := pluginConfig[name]
if ok {
return res, nil
}
// Use defaults from latest config API version.
var gvk schema.GroupVersionKind
switch componentConfigVersion {
case v1beta1.SchemeGroupVersion.String():
gvk = v1beta1.SchemeGroupVersion.WithKind(name + "Args")
case v1beta2.SchemeGroupVersion.String():
gvk = v1beta2.SchemeGroupVersion.WithKind(name + "Args")
default:
// default to v1beta2 (latest API)
gvk = v1beta2.SchemeGroupVersion.WithKind(name + "Args")
}
obj, _, err := configDecoder.Decode(nil, &gvk, nil)
if runtime.IsNotRegisteredError(err) {
// This plugin is out-of-tree or doesn't require configuration.
return nil, nil
}
return obj, err
}
func updatePluginList(pluginList interface{}, pluginSet config.PluginSet, pluginsMap map[string]framework.Plugin) error {
plugins := reflect.ValueOf(pluginList).Elem()
pluginType := plugins.Type().Elem()
@ -1153,10 +1116,10 @@ func (f *frameworkImpl) HasScorePlugins() bool {
// ListPlugins returns a map of extension point name to plugin names configured at each extension
// point. Returns nil if no plugins where configured.
func (f *frameworkImpl) ListPlugins() map[string][]config.Plugin {
m := make(map[string][]config.Plugin)
func (f *frameworkImpl) ListPlugins() *config.Plugins {
m := config.Plugins{}
for _, e := range f.getExtensionPoints(&config.Plugins{}) {
for _, e := range f.getExtensionPoints(&m) {
plugins := reflect.ValueOf(e.slicePtr).Elem()
extName := plugins.Type().Elem().Name()
var cfgs []config.Plugin
@ -1170,13 +1133,10 @@ func (f *frameworkImpl) ListPlugins() map[string][]config.Plugin {
cfgs = append(cfgs, p)
}
if len(cfgs) > 0 {
m[extName] = cfgs
e.plugins.Enabled = cfgs
}
}
if len(m) > 0 {
return m
}
return nil
return &m
}
// ClientSet returns a kubernetes clientset.
@ -1206,7 +1166,7 @@ func (f *frameworkImpl) pluginsNeeded(plugins *config.Plugins) map[string]config
return pgMap
}
find := func(pgs config.PluginSet) {
find := func(pgs *config.PluginSet) {
for _, pg := range pgs.Enabled {
pgMap[pg.Name] = pg
}

View File

@ -378,23 +378,13 @@ func newFrameworkWithQueueSortAndBind(r Registry, profile config.KubeSchedulerPr
if _, ok := r[bindPlugin]; !ok {
r[bindPlugin] = newBindPlugin
}
plugins := &config.Plugins{}
plugins.Append(profile.Plugins)
if len(plugins.QueueSort.Enabled) == 0 {
plugins.Append(&config.Plugins{
QueueSort: config.PluginSet{
Enabled: []config.Plugin{{Name: queueSortPlugin}},
},
})
if len(profile.Plugins.QueueSort.Enabled) == 0 {
profile.Plugins.QueueSort.Enabled = append(profile.Plugins.QueueSort.Enabled, config.Plugin{Name: queueSortPlugin})
}
if len(plugins.Bind.Enabled) == 0 {
plugins.Append(&config.Plugins{
Bind: config.PluginSet{
Enabled: []config.Plugin{{Name: bindPlugin}},
},
})
if len(profile.Plugins.Bind.Enabled) == 0 {
profile.Plugins.Bind.Enabled = append(profile.Plugins.Bind.Enabled, config.Plugin{Name: bindPlugin})
}
profile.Plugins = plugins
return NewFramework(r, &profile, opts...)
}
@ -500,160 +490,6 @@ func TestNewFrameworkErrors(t *testing.T) {
}
}
func recordingPluginFactory(name string, result map[string]runtime.Object) PluginFactory {
return func(args runtime.Object, f framework.Handle) (framework.Plugin, error) {
result[name] = args
return &TestPlugin{
name: name,
}, nil
}
}
func TestNewFrameworkPluginDefaults(t *testing.T) {
// In-tree plugins that use args.
pluginsWithArgs := []string{
"InterPodAffinity",
"NodeLabel",
"NodeResourcesFit",
"NodeResourcesLeastAllocated",
"NodeResourcesMostAllocated",
"PodTopologySpread",
"RequestedToCapacityRatio",
"VolumeBinding",
}
plugins := config.Plugins{}
// Use all plugins in Filter.
// NOTE: This does not mean those plugins implemented `Filter` interfaces.
// `TestPlugin` is created in this test to fake the behavior for test purpose.
for _, name := range pluginsWithArgs {
plugins.Filter.Enabled = append(plugins.Filter.Enabled, config.Plugin{Name: name})
}
// Set required extension points.
onePlugin := config.PluginSet{
Enabled: []config.Plugin{{Name: pluginsWithArgs[0]}},
}
plugins.QueueSort = onePlugin
plugins.Bind = onePlugin
tests := []struct {
name string
pluginCfg []config.PluginConfig
wantCfg map[string]runtime.Object
}{
{
name: "empty plugin config",
wantCfg: map[string]runtime.Object{
"InterPodAffinity": &config.InterPodAffinityArgs{
HardPodAffinityWeight: 1,
},
"NodeLabel": nil,
"NodeResourcesFit": &config.NodeResourcesFitArgs{},
"NodeResourcesLeastAllocated": &config.NodeResourcesLeastAllocatedArgs{
Resources: []config.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}},
},
"NodeResourcesMostAllocated": &config.NodeResourcesMostAllocatedArgs{
Resources: []config.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}},
},
"RequestedToCapacityRatio": &config.RequestedToCapacityRatioArgs{
Resources: []config.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}},
},
"PodTopologySpread": &config.PodTopologySpreadArgs{
DefaultingType: config.SystemDefaulting,
},
"VolumeBinding": &config.VolumeBindingArgs{
BindTimeoutSeconds: 600,
},
},
},
{
name: "some overridden plugin config",
pluginCfg: []config.PluginConfig{
{
Name: "InterPodAffinity",
Args: &config.InterPodAffinityArgs{
HardPodAffinityWeight: 3,
},
},
{
Name: "NodeResourcesFit",
Args: &config.NodeResourcesFitArgs{
IgnoredResources: []string{"example.com/foo"},
},
},
{
Name: "NodeResourcesLeastAllocated",
Args: &config.NodeResourcesLeastAllocatedArgs{
Resources: []config.ResourceSpec{{Name: "resource", Weight: 4}},
},
},
{
Name: "NodeResourcesMostAllocated",
Args: &config.NodeResourcesMostAllocatedArgs{
Resources: []config.ResourceSpec{{Name: "resource", Weight: 3}},
},
},
{
Name: "RequestedToCapacityRatio",
Args: &config.RequestedToCapacityRatioArgs{
Resources: []config.ResourceSpec{{Name: "resource", Weight: 2}},
},
},
{
Name: "VolumeBinding",
Args: &config.VolumeBindingArgs{
BindTimeoutSeconds: 300,
},
},
},
wantCfg: map[string]runtime.Object{
"InterPodAffinity": &config.InterPodAffinityArgs{
HardPodAffinityWeight: 3,
},
"NodeLabel": nil,
"NodeResourcesFit": &config.NodeResourcesFitArgs{
IgnoredResources: []string{"example.com/foo"},
},
"NodeResourcesLeastAllocated": &config.NodeResourcesLeastAllocatedArgs{
Resources: []config.ResourceSpec{{Name: "resource", Weight: 4}},
},
"NodeResourcesMostAllocated": &config.NodeResourcesMostAllocatedArgs{
Resources: []config.ResourceSpec{{Name: "resource", Weight: 3}},
},
"PodTopologySpread": &config.PodTopologySpreadArgs{
DefaultingType: config.SystemDefaulting,
},
"RequestedToCapacityRatio": &config.RequestedToCapacityRatioArgs{
Resources: []config.ResourceSpec{{Name: "resource", Weight: 2}},
},
"VolumeBinding": &config.VolumeBindingArgs{
BindTimeoutSeconds: 300,
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// result will hold plugin args passed to factories.
result := make(map[string]runtime.Object)
registry := make(Registry, len(pluginsWithArgs))
for _, name := range pluginsWithArgs {
registry[name] = recordingPluginFactory(name, result)
}
profile := &config.KubeSchedulerProfile{
Plugins: &plugins,
PluginConfig: tt.pluginCfg,
}
_, err := NewFramework(registry, profile)
if err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(tt.wantCfg, result); diff != "" {
t.Errorf("unexpected plugin args (-want,+got):\n%s", diff)
}
})
}
}
// fakeNoopPlugin doesn't implement interface framework.EnqueueExtensions.
type fakeNoopPlugin struct{}
@ -2335,54 +2171,25 @@ func TestListPlugins(t *testing.T) {
tests := []struct {
name string
plugins *config.Plugins
// pluginSetCount include queue sort plugin and bind plugin.
pluginSetCount int
want *config.Plugins
}{
{
name: "Add empty plugin",
plugins: &config.Plugins{},
pluginSetCount: 2,
want: &config.Plugins{
QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: queueSortPlugin}}},
Bind: config.PluginSet{Enabled: []config.Plugin{{Name: bindPlugin}}},
},
},
{
name: "Add multiple plugins",
plugins: &config.Plugins{
Score: config.PluginSet{Enabled: []config.Plugin{{Name: scorePlugin1}, {Name: scoreWithNormalizePlugin1}}},
Score: config.PluginSet{Enabled: []config.Plugin{{Name: scorePlugin1, Weight: 3}, {Name: scoreWithNormalizePlugin1}}},
},
pluginSetCount: 3,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
profile := config.KubeSchedulerProfile{Plugins: tt.plugins}
f, err := newFrameworkWithQueueSortAndBind(registry, profile)
if err != nil {
t.Fatalf("Failed to create framework for testing: %v", err)
}
plugins := f.ListPlugins()
if len(plugins) != tt.pluginSetCount {
t.Fatalf("Unexpected pluginSet count: %v", len(plugins))
}
})
}
}
func TestNewFrameworkPluginWeights(t *testing.T) {
tests := []struct {
name string
plugins *config.Plugins
}{
{
name: "Extend multiple extension points by same plugin",
plugins: &config.Plugins{
Score: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin, Weight: 3}}},
PostBind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin, Weight: 6}}},
},
},
{
name: "Add multiple score plugins",
plugins: &config.Plugins{
Score: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin, Weight: 3}, {Name: scorePlugin1, Weight: 6}}},
want: &config.Plugins{
QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: queueSortPlugin}}},
Bind: config.PluginSet{Enabled: []config.Plugin{{Name: bindPlugin}}},
Score: config.PluginSet{Enabled: []config.Plugin{{Name: scorePlugin1, Weight: 3}, {Name: scoreWithNormalizePlugin1, Weight: 1}}},
},
},
}
@ -2394,14 +2201,9 @@ func TestNewFrameworkPluginWeights(t *testing.T) {
if err != nil {
t.Fatalf("Failed to create framework for testing: %v", err)
}
plugins := f.ListPlugins()
if len(plugins["ScorePlugin"]) != len(tt.plugins.Score.Enabled) {
t.Fatalf("Expect %d ScorePlugin, got %d from: %v", len(tt.plugins.Score.Enabled), len(plugins["ScorePlugin"]), plugins["ScorePlugin"])
}
if diff := cmp.Diff(tt.plugins.Score.Enabled, plugins["ScorePlugin"]); diff != "" {
t.Errorf("unexpected plugin weights (-want,+got):\n%s", diff)
got := f.ListPlugins()
if diff := cmp.Diff(tt.want, got); diff != "" {
t.Errorf("unexpected plugins (-want,+got):\n%s", diff)
}
})
}

View File

@ -39,8 +39,10 @@ import (
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/cache"
"k8s.io/klog/v2"
"k8s.io/kube-scheduler/config/v1beta2"
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
"k8s.io/kubernetes/pkg/apis/core/validation"
"k8s.io/kubernetes/pkg/scheduler/apis/config"
schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config"
"k8s.io/kubernetes/pkg/scheduler/apis/config/scheme"
"k8s.io/kubernetes/pkg/scheduler/core"
@ -108,6 +110,7 @@ type schedulerOptions struct {
extenders []schedulerapi.Extender
frameworkCapturer FrameworkCapturer
parallelism int32
applyDefaultProfile bool
}
// Option configures a Scheduler
@ -135,6 +138,7 @@ func WithKubeConfig(cfg *restclient.Config) Option {
func WithProfiles(p ...schedulerapi.KubeSchedulerProfile) Option {
return func(o *schedulerOptions) {
o.profiles = p
o.applyDefaultProfile = false
}
}
@ -145,7 +149,7 @@ func WithParallelism(threads int32) Option {
}
}
// WithPolicySource sets legacy policy config file source.
// WithLegacyPolicySource sets legacy policy config file source.
func WithLegacyPolicySource(source *schedulerapi.SchedulerPolicySource) Option {
return func(o *schedulerOptions) {
o.legacyPolicySource = source
@ -199,14 +203,15 @@ func WithBuildFrameworkCapturer(fc FrameworkCapturer) Option {
}
var defaultSchedulerOptions = schedulerOptions{
profiles: []schedulerapi.KubeSchedulerProfile{
// Profiles' default plugins are set from the algorithm provider.
{SchedulerName: v1.DefaultSchedulerName},
},
percentageOfNodesToScore: schedulerapi.DefaultPercentageOfNodesToScore,
podInitialBackoffSeconds: int64(internalqueue.DefaultPodInitialBackoffDuration.Seconds()),
podMaxBackoffSeconds: int64(internalqueue.DefaultPodMaxBackoffDuration.Seconds()),
parallelism: int32(parallelize.DefaultParallelism),
// Ideally we would statically set the default profile here, but we can't because
// creating the default profile may require testing feature gates, which may get
// set dynamically in tests. Therefore, we delay creating it until New is actually
// invoked.
applyDefaultProfile: true,
}
// New returns a Scheduler
@ -226,6 +231,15 @@ func New(client clientset.Interface,
opt(&options)
}
if options.applyDefaultProfile {
var versionedCfg v1beta2.KubeSchedulerConfiguration
scheme.Scheme.Default(&versionedCfg)
cfg := config.KubeSchedulerConfiguration{}
if err := scheme.Scheme.Convert(&versionedCfg, &cfg, nil); err != nil {
return nil, err
}
options.profiles = cfg.Profiles
}
schedulerCache := internalcache.New(30*time.Second, stopEverything)
registry := frameworkplugins.NewInTreeRegistry()
@ -260,12 +274,12 @@ func New(client clientset.Interface,
var sched *Scheduler
if options.legacyPolicySource == nil {
sc, err := configurator.createFromConfig()
// Create the config from component config
sc, err := configurator.create()
if err != nil {
return nil, fmt.Errorf("couldn't create scheduler: %v", err)
}
sched = sc
} else {
// Create the config from a user specified policy source.
policy := &schedulerapi.Policy{}

View File

@ -131,35 +131,83 @@ func TestSchedulerCreation(t *testing.T) {
wantErr string
wantProfiles []string
}{
{
name: "default scheduler",
wantProfiles: []string{"default-scheduler"},
},
{
name: "valid out-of-tree registry",
opts: []Option{WithFrameworkOutOfTreeRegistry(validRegistry)},
opts: []Option{
WithFrameworkOutOfTreeRegistry(validRegistry),
WithProfiles(
schedulerapi.KubeSchedulerProfile{
SchedulerName: "default-scheduler",
Plugins: &schedulerapi.Plugins{
QueueSort: schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "PrioritySort"}}},
Bind: schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "DefaultBinder"}}},
},
},
)},
wantProfiles: []string{"default-scheduler"},
},
{
name: "repeated plugin name in out-of-tree plugin",
opts: []Option{WithFrameworkOutOfTreeRegistry(invalidRegistry)},
opts: []Option{
WithFrameworkOutOfTreeRegistry(invalidRegistry),
WithProfiles(
schedulerapi.KubeSchedulerProfile{
SchedulerName: "default-scheduler",
Plugins: &schedulerapi.Plugins{
QueueSort: schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "PrioritySort"}}},
Bind: schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "DefaultBinder"}}},
},
},
)},
wantProfiles: []string{"default-scheduler"},
wantErr: "a plugin named DefaultBinder already exists",
},
{
name: "multiple profiles",
opts: []Option{WithProfiles(
schedulerapi.KubeSchedulerProfile{SchedulerName: "foo"},
schedulerapi.KubeSchedulerProfile{SchedulerName: "bar"},
opts: []Option{
WithProfiles(
schedulerapi.KubeSchedulerProfile{
SchedulerName: "foo",
Plugins: &schedulerapi.Plugins{
QueueSort: schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "PrioritySort"}}},
Bind: schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "DefaultBinder"}}},
},
},
schedulerapi.KubeSchedulerProfile{
SchedulerName: "bar",
Plugins: &schedulerapi.Plugins{
QueueSort: schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "PrioritySort"}}},
Bind: schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "DefaultBinder"}}},
},
},
)},
wantProfiles: []string{"bar", "foo"},
},
{
name: "Repeated profiles",
opts: []Option{WithProfiles(
schedulerapi.KubeSchedulerProfile{SchedulerName: "foo"},
schedulerapi.KubeSchedulerProfile{SchedulerName: "bar"},
schedulerapi.KubeSchedulerProfile{SchedulerName: "foo"},
opts: []Option{
WithProfiles(
schedulerapi.KubeSchedulerProfile{
SchedulerName: "foo",
Plugins: &schedulerapi.Plugins{
QueueSort: schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "PrioritySort"}}},
Bind: schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "DefaultBinder"}}},
},
},
schedulerapi.KubeSchedulerProfile{
SchedulerName: "bar",
Plugins: &schedulerapi.Plugins{
QueueSort: schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "PrioritySort"}}},
Bind: schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "DefaultBinder"}}},
},
},
schedulerapi.KubeSchedulerProfile{
SchedulerName: "foo",
Plugins: &schedulerapi.Plugins{
QueueSort: schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "PrioritySort"}}},
Bind: schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "DefaultBinder"}}},
},
},
)},
wantErr: "duplicate profile with scheduler name \"foo\"",
},
@ -463,12 +511,13 @@ func TestSchedulerMultipleProfilesScheduling(t *testing.T) {
WithProfiles(
schedulerapi.KubeSchedulerProfile{SchedulerName: "match-machine2",
Plugins: &schedulerapi.Plugins{
Filter: schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{{Name: "FakeNodeSelector"}},
Disabled: []schedulerapi.Plugin{{Name: "*"}},
}},
Filter: schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "FakeNodeSelector"}}},
QueueSort: schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "PrioritySort"}}},
Bind: schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "DefaultBinder"}}},
},
PluginConfig: []schedulerapi.PluginConfig{
{Name: "FakeNodeSelector",
{
Name: "FakeNodeSelector",
Args: &runtime.Unknown{Raw: []byte(`{"nodeName":"machine2"}`)},
},
},
@ -476,12 +525,13 @@ func TestSchedulerMultipleProfilesScheduling(t *testing.T) {
schedulerapi.KubeSchedulerProfile{
SchedulerName: "match-machine3",
Plugins: &schedulerapi.Plugins{
Filter: schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{{Name: "FakeNodeSelector"}},
Disabled: []schedulerapi.Plugin{{Name: "*"}},
}},
Filter: schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "FakeNodeSelector"}}},
QueueSort: schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "PrioritySort"}}},
Bind: schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "DefaultBinder"}}},
},
PluginConfig: []schedulerapi.PluginConfig{
{Name: "FakeNodeSelector",
{
Name: "FakeNodeSelector",
Args: &runtime.Unknown{Raw: []byte(`{"nodeName":"machine3"}`)},
},
},

View File

@ -17,28 +17,32 @@ limitations under the License.
package testing
import (
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/kube-scheduler/config/v1beta2"
schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config"
"k8s.io/kubernetes/pkg/scheduler/apis/config/scheme"
"k8s.io/kubernetes/pkg/scheduler/framework"
"k8s.io/kubernetes/pkg/scheduler/framework/runtime"
)
var configDecoder = scheme.Codecs.UniversalDecoder()
// NewFramework creates a Framework from the register functions and options.
func NewFramework(fns []RegisterPluginFunc, profileName string, opts ...runtime.Option) (framework.Framework, error) {
registry := runtime.Registry{}
plugins := &schedulerapi.Plugins{}
for _, f := range fns {
f(&registry, plugins)
}
profile := &schedulerapi.KubeSchedulerProfile{
SchedulerName: profileName,
Plugins: plugins,
Plugins: &schedulerapi.Plugins{},
}
for _, f := range fns {
f(&registry, profile)
}
return runtime.NewFramework(registry, profile, opts...)
}
// RegisterPluginFunc is a function signature used in method RegisterFilterPlugin()
// to register a Filter Plugin to a given registry.
type RegisterPluginFunc func(reg *runtime.Registry, plugins *schedulerapi.Plugins)
type RegisterPluginFunc func(reg *runtime.Registry, profile *schedulerapi.KubeSchedulerProfile)
// RegisterQueueSortPlugin returns a function to register a QueueSort Plugin to a given registry.
func RegisterQueueSortPlugin(pluginName string, pluginNewFunc runtime.PluginFactory) RegisterPluginFunc {
@ -92,15 +96,24 @@ func RegisterPluginAsExtensions(pluginName string, pluginNewFunc runtime.PluginF
// RegisterPluginAsExtensionsWithWeight returns a function to register a Plugin as given extensionPoints with weight to a given registry.
func RegisterPluginAsExtensionsWithWeight(pluginName string, weight int32, pluginNewFunc runtime.PluginFactory, extensions ...string) RegisterPluginFunc {
return func(reg *runtime.Registry, plugins *schedulerapi.Plugins) {
return func(reg *runtime.Registry, profile *schedulerapi.KubeSchedulerProfile) {
reg.Register(pluginName, pluginNewFunc)
for _, extension := range extensions {
ps := getPluginSetByExtension(plugins, extension)
ps := getPluginSetByExtension(profile.Plugins, extension)
if ps == nil {
continue
}
ps.Enabled = append(ps.Enabled, schedulerapi.Plugin{Name: pluginName, Weight: weight})
}
// Use defaults from latest config API version.
var gvk schema.GroupVersionKind
gvk = v1beta2.SchemeGroupVersion.WithKind(pluginName + "Args")
if args, _, err := configDecoder.Decode(nil, &gvk, nil); err == nil {
profile.PluginConfig = append(profile.PluginConfig, schedulerapi.PluginConfig{
Name: pluginName,
Args: args,
})
}
}
}

View File

@ -34,6 +34,7 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
extenderv1 "k8s.io/kube-scheduler/extender/v1"
"k8s.io/kubernetes/pkg/scheduler"
schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config"
testutils "k8s.io/kubernetes/test/integration/util"
imageutils "k8s.io/kubernetes/test/utils/image"
@ -350,7 +351,8 @@ func TestSchedulerExtender(t *testing.T) {
}
policy.APIVersion = "v1"
testCtx = testutils.InitTestScheduler(t, testCtx, &policy)
testCtx = testutils.InitTestSchedulerWithOptions(t, testCtx, &policy,
scheduler.WithProfiles([]schedulerapi.KubeSchedulerProfile(nil)...))
testutils.SyncInformerFactory(testCtx)
go testCtx.Scheduler.Run(testCtx.Ctx)
defer testutils.CleanupTest(t, testCtx)

View File

@ -30,13 +30,16 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kube-scheduler/config/v1beta2"
"k8s.io/kubernetes/pkg/scheduler"
schedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config"
configtesting "k8s.io/kubernetes/pkg/scheduler/apis/config/testing"
"k8s.io/kubernetes/pkg/scheduler/framework"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder"
frameworkruntime "k8s.io/kubernetes/pkg/scheduler/framework/runtime"
st "k8s.io/kubernetes/pkg/scheduler/testing"
testutils "k8s.io/kubernetes/test/integration/util"
"k8s.io/utils/pointer"
)
type PreFilterPlugin struct {
@ -510,20 +513,22 @@ func TestPreFilterPlugin(t *testing.T) {
registry := frameworkruntime.Registry{prefilterPluginName: newPlugin(preFilterPlugin)}
// Setup initial prefilter plugin for testing.
prof := schedulerconfig.KubeSchedulerProfile{
SchedulerName: v1.DefaultSchedulerName,
Plugins: &schedulerconfig.Plugins{
PreFilter: schedulerconfig.PluginSet{
Enabled: []schedulerconfig.Plugin{
cfg := configtesting.V1beta2ToInternalWithDefaults(t, v1beta2.KubeSchedulerConfiguration{
Profiles: []v1beta2.KubeSchedulerProfile{{
SchedulerName: pointer.StringPtr(v1.DefaultSchedulerName),
Plugins: &v1beta2.Plugins{
PreFilter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: prefilterPluginName},
},
},
},
}
}},
})
// Create the master and the scheduler with the test plugin set.
testCtx := initTestSchedulerForFrameworkTest(t, testutils.InitTestMaster(t, "prefilter-plugin", nil), 2,
scheduler.WithProfiles(prof),
scheduler.WithProfiles(cfg.Profiles...),
scheduler.WithFrameworkOutOfTreeRegistry(registry))
defer testutils.CleanupTest(t, testCtx)
@ -653,43 +658,44 @@ func TestPostFilterPlugin(t *testing.T) {
}
// Setup plugins for testing.
prof := schedulerconfig.KubeSchedulerProfile{
SchedulerName: v1.DefaultSchedulerName,
Plugins: &schedulerconfig.Plugins{
Filter: schedulerconfig.PluginSet{
Enabled: []schedulerconfig.Plugin{
cfg := configtesting.V1beta2ToInternalWithDefaults(t, v1beta2.KubeSchedulerConfiguration{
Profiles: []v1beta2.KubeSchedulerProfile{{
SchedulerName: pointer.StringPtr(v1.DefaultSchedulerName),
Plugins: &v1beta2.Plugins{
Filter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: filterPluginName},
},
},
Score: schedulerconfig.PluginSet{
Enabled: []schedulerconfig.Plugin{
Score: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: scorePluginName},
},
// disable default in-tree Score plugins
// to make it easy to control configured ScorePlugins failure
Disabled: []schedulerconfig.Plugin{
Disabled: []v1beta2.Plugin{
{Name: "*"},
},
},
PostFilter: schedulerconfig.PluginSet{
Enabled: []schedulerconfig.Plugin{
PostFilter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: postfilterPluginName},
},
// Need to disable default in-tree PostFilter plugins, as they will
// call RunFilterPlugins and hence impact the "numFilterCalled".
Disabled: []schedulerconfig.Plugin{
Disabled: []v1beta2.Plugin{
{Name: "*"},
},
},
},
}
}}})
// Create the master and the scheduler with the test plugin set.
testCtx := initTestSchedulerForFrameworkTest(
t,
testutils.InitTestMaster(t, fmt.Sprintf("postfilter%v-", i), nil),
int(tt.numNodes),
scheduler.WithProfiles(prof),
scheduler.WithProfiles(cfg.Profiles...),
scheduler.WithFrameworkOutOfTreeRegistry(registry),
)
defer testutils.CleanupTest(t, testCtx)
@ -740,19 +746,21 @@ func TestScorePlugin(t *testing.T) {
scorePluginName: newPlugin(scorePlugin),
}
prof := schedulerconfig.KubeSchedulerProfile{
SchedulerName: v1.DefaultSchedulerName,
Plugins: &schedulerconfig.Plugins{
Score: schedulerconfig.PluginSet{
Enabled: []schedulerconfig.Plugin{
cfg := configtesting.V1beta2ToInternalWithDefaults(t, v1beta2.KubeSchedulerConfiguration{
Profiles: []v1beta2.KubeSchedulerProfile{{
SchedulerName: pointer.StringPtr(v1.DefaultSchedulerName),
Plugins: &v1beta2.Plugins{
Score: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: scorePluginName},
},
},
},
}
}},
})
testCtx := initTestSchedulerForFrameworkTest(t, testutils.InitTestMaster(t, "score-plugin", nil), 10,
scheduler.WithProfiles(prof),
scheduler.WithProfiles(cfg.Profiles...),
scheduler.WithFrameworkOutOfTreeRegistry(registry))
defer testutils.CleanupTest(t, testCtx)
@ -816,18 +824,21 @@ func TestNormalizeScorePlugin(t *testing.T) {
}
// Setup initial score plugin for testing.
prof := schedulerconfig.KubeSchedulerProfile{
SchedulerName: v1.DefaultSchedulerName,
Plugins: &schedulerconfig.Plugins{
Score: schedulerconfig.PluginSet{
Enabled: []schedulerconfig.Plugin{
cfg := configtesting.V1beta2ToInternalWithDefaults(t, v1beta2.KubeSchedulerConfiguration{
Profiles: []v1beta2.KubeSchedulerProfile{{
SchedulerName: pointer.StringPtr(v1.DefaultSchedulerName),
Plugins: &v1beta2.Plugins{
Score: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: scoreWithNormalizePluginName},
},
},
},
}
}},
})
testCtx := initTestSchedulerForFrameworkTest(t, testutils.InitTestMaster(t, "score-plugin", nil), 10,
scheduler.WithProfiles(prof),
scheduler.WithProfiles(cfg.Profiles...),
scheduler.WithFrameworkOutOfTreeRegistry(registry))
defer testutils.CleanupTest(t, testCtx)
@ -860,22 +871,22 @@ func TestReservePluginReserve(t *testing.T) {
registry := frameworkruntime.Registry{reservePluginName: newPlugin(reservePlugin)}
// Setup initial reserve plugin for testing.
prof := schedulerconfig.KubeSchedulerProfile{
SchedulerName: v1.DefaultSchedulerName,
Plugins: &schedulerconfig.Plugins{
Reserve: schedulerconfig.PluginSet{
Enabled: []schedulerconfig.Plugin{
{
Name: reservePluginName,
cfg := configtesting.V1beta2ToInternalWithDefaults(t, v1beta2.KubeSchedulerConfiguration{
Profiles: []v1beta2.KubeSchedulerProfile{{
SchedulerName: pointer.StringPtr(v1.DefaultSchedulerName),
Plugins: &v1beta2.Plugins{
Reserve: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: reservePluginName},
},
},
},
},
}
}},
})
// Create the master and the scheduler with the test plugin set.
testCtx := initTestSchedulerForFrameworkTest(t, testutils.InitTestMaster(t, "reserve-plugin-reserve", nil), 2,
scheduler.WithProfiles(prof),
scheduler.WithProfiles(cfg.Profiles...),
scheduler.WithFrameworkOutOfTreeRegistry(registry))
defer testutils.CleanupTest(t, testCtx)
@ -931,22 +942,22 @@ func TestPrebindPlugin(t *testing.T) {
registry := frameworkruntime.Registry{preBindPluginName: newPlugin(preBindPlugin)}
// Setup initial prebind plugin for testing.
prof := schedulerconfig.KubeSchedulerProfile{
SchedulerName: v1.DefaultSchedulerName,
Plugins: &schedulerconfig.Plugins{
PreBind: schedulerconfig.PluginSet{
Enabled: []schedulerconfig.Plugin{
{
Name: preBindPluginName,
cfg := configtesting.V1beta2ToInternalWithDefaults(t, v1beta2.KubeSchedulerConfiguration{
Profiles: []v1beta2.KubeSchedulerProfile{{
SchedulerName: pointer.StringPtr(v1.DefaultSchedulerName),
Plugins: &v1beta2.Plugins{
PreBind: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: preBindPluginName},
},
},
},
},
}
}},
})
// Create the master and the scheduler with the test plugin set.
testCtx := initTestSchedulerForFrameworkTest(t, testutils.InitTestMaster(t, "prebind-plugin", nil), 2,
scheduler.WithProfiles(prof),
scheduler.WithProfiles(cfg.Profiles...),
scheduler.WithFrameworkOutOfTreeRegistry(registry))
defer testutils.CleanupTest(t, testCtx)
@ -1006,124 +1017,124 @@ func TestPrebindPlugin(t *testing.T) {
}
}
// TestUnreserveReservePlugin tests invocation of the Unreserve operation in
// reserve plugins through failures in execution points such as pre-bind. Also
// tests that the order of invocation of Unreserve operation is executed in the
// reverse order of invocation of the Reserve operation.
func TestReservePluginUnreserve(t *testing.T) {
tests := []struct {
name string
failReserve bool
failReserveIndex int
failPreBind bool
}{
{
name: "fail reserve",
failReserve: true,
failReserveIndex: 1,
},
{
name: "fail preBind",
failPreBind: true,
},
{
name: "pass everything",
},
}
// // TestUnreserveReservePlugin tests invocation of the Unreserve operation in
// // reserve plugins through failures in execution points such as pre-bind. Also
// // tests that the order of invocation of Unreserve operation is executed in the
// // reverse order of invocation of the Reserve operation.
// func TestReservePluginUnreserve(t *testing.T) {
// tests := []struct {
// name string
// failReserve bool
// failReserveIndex int
// failPreBind bool
// }{
// {
// name: "fail reserve",
// failReserve: true,
// failReserveIndex: 1,
// },
// {
// name: "fail preBind",
// failPreBind: true,
// },
// {
// name: "pass everything",
// },
// }
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
numReservePlugins := 3
pluginInvokeEventChan := make(chan pluginInvokeEvent, numReservePlugins)
// for _, test := range tests {
// t.Run(test.name, func(t *testing.T) {
// numReservePlugins := 3
// pluginInvokeEventChan := make(chan pluginInvokeEvent, numReservePlugins)
preBindPlugin := &PreBindPlugin{
failPreBind: true,
}
var reservePlugins []*ReservePlugin
for i := 0; i < numReservePlugins; i++ {
reservePlugins = append(reservePlugins, &ReservePlugin{
name: fmt.Sprintf("%s-%d", reservePluginName, i),
pluginInvokeEventChan: pluginInvokeEventChan,
})
}
// preBindPlugin := &PreBindPlugin{
// failPreBind: true,
// }
// var reservePlugins []*ReservePlugin
// for i := 0; i < numReservePlugins; i++ {
// reservePlugins = append(reservePlugins, &ReservePlugin{
// name: fmt.Sprintf("%s-%d", reservePluginName, i),
// pluginInvokeEventChan: pluginInvokeEventChan,
// })
// }
registry := frameworkruntime.Registry{
// TODO(#92229): test more failure points that would trigger Unreserve in
// reserve plugins than just one pre-bind plugin.
preBindPluginName: newPlugin(preBindPlugin),
}
for _, pl := range reservePlugins {
registry[pl.Name()] = newPlugin(pl)
}
// registry := frameworkruntime.Registry{
// // TODO(#92229): test more failure points that would trigger Unreserve in
// // reserve plugins than just one pre-bind plugin.
// preBindPluginName: newPlugin(preBindPlugin),
// }
// for _, pl := range reservePlugins {
// registry[pl.Name()] = newPlugin(pl)
// }
// Setup initial reserve and prebind plugin for testing.
prof := schedulerconfig.KubeSchedulerProfile{
SchedulerName: v1.DefaultSchedulerName,
Plugins: &schedulerconfig.Plugins{
Reserve: schedulerconfig.PluginSet{
// filled by looping over reservePlugins
},
PreBind: schedulerconfig.PluginSet{
Enabled: []schedulerconfig.Plugin{
{
Name: preBindPluginName,
},
},
},
},
}
for _, pl := range reservePlugins {
prof.Plugins.Reserve.Enabled = append(prof.Plugins.Reserve.Enabled, schedulerconfig.Plugin{
Name: pl.Name(),
})
}
// // Setup initial reserve and prebind plugin for testing.
// prof := schedulerconfig.KubeSchedulerProfile{
// SchedulerName: v1.DefaultSchedulerName,
// Plugins: &schedulerconfig.Plugins{
// Reserve: schedulerconfig.PluginSet{
// // filled by looping over reservePlugins
// },
// PreBind: schedulerconfig.PluginSet{
// Enabled: []schedulerconfig.Plugin{
// {
// Name: preBindPluginName,
// },
// },
// },
// },
// }
// for _, pl := range reservePlugins {
// prof.Plugins.Reserve.Enabled = append(prof.Plugins.Reserve.Enabled, schedulerconfig.Plugin{
// Name: pl.Name(),
// })
// }
// Create the master and the scheduler with the test plugin set.
testCtx := initTestSchedulerForFrameworkTest(t, testutils.InitTestMaster(t, "reserve-plugin-unreserve", nil), 2,
scheduler.WithProfiles(prof),
scheduler.WithFrameworkOutOfTreeRegistry(registry))
defer testutils.CleanupTest(t, testCtx)
// // Create the master and the scheduler with the test plugin set.
// testCtx := initTestSchedulerForFrameworkTest(t, testutils.InitTestMaster(t, "reserve-plugin-unreserve", nil), 2,
// scheduler.WithProfiles(prof),
// scheduler.WithFrameworkOutOfTreeRegistry(registry))
// defer testutils.CleanupTest(t, testCtx)
preBindPlugin.failPreBind = test.failPreBind
if test.failReserve {
reservePlugins[test.failReserveIndex].failReserve = true
}
// Create a best effort pod.
pod, err := createPausePod(testCtx.ClientSet,
initPausePod(&pausePodConfig{Name: "test-pod", Namespace: testCtx.NS.Name}))
if err != nil {
t.Errorf("Error while creating a test pod: %v", err)
}
// preBindPlugin.failPreBind = test.failPreBind
// if test.failReserve {
// reservePlugins[test.failReserveIndex].failReserve = true
// }
// // Create a best effort pod.
// pod, err := createPausePod(testCtx.ClientSet,
// initPausePod(&pausePodConfig{Name: "test-pod", Namespace: testCtx.NS.Name}))
// if err != nil {
// t.Errorf("Error while creating a test pod: %v", err)
// }
if test.failPreBind || test.failReserve {
if err = wait.Poll(10*time.Millisecond, 30*time.Second, podSchedulingError(testCtx.ClientSet, pod.Namespace, pod.Name)); err != nil {
t.Errorf("Expected a scheduling error, but didn't get it: %v", err)
}
for i := numReservePlugins - 1; i >= 0; i-- {
select {
case event := <-pluginInvokeEventChan:
expectedPluginName := reservePlugins[i].Name()
if expectedPluginName != event.pluginName {
t.Errorf("event.pluginName = %s, want %s", event.pluginName, expectedPluginName)
}
case <-time.After(time.Second * 30):
t.Errorf("pluginInvokeEventChan receive timed out")
}
}
} else {
if err = testutils.WaitForPodToSchedule(testCtx.ClientSet, pod); err != nil {
t.Errorf("Expected the pod to be scheduled, got an error: %v", err)
}
for i, pl := range reservePlugins {
if pl.numUnreserveCalled != 0 {
t.Errorf("reservePlugins[%d].numUnreserveCalled = %d, want 0", i, pl.numUnreserveCalled)
}
}
}
testutils.CleanupPods(testCtx.ClientSet, t, []*v1.Pod{pod})
})
}
}
// if test.failPreBind || test.failReserve {
// if err = wait.Poll(10*time.Millisecond, 30*time.Second, podSchedulingError(testCtx.ClientSet, pod.Namespace, pod.Name)); err != nil {
// t.Errorf("Expected a scheduling error, but didn't get it: %v", err)
// }
// for i := numReservePlugins - 1; i >= 0; i-- {
// select {
// case event := <-pluginInvokeEventChan:
// expectedPluginName := reservePlugins[i].Name()
// if expectedPluginName != event.pluginName {
// t.Errorf("event.pluginName = %s, want %s", event.pluginName, expectedPluginName)
// }
// case <-time.After(time.Second * 30):
// t.Errorf("pluginInvokeEventChan receive timed out")
// }
// }
// } else {
// if err = testutils.WaitForPodToSchedule(testCtx.ClientSet, pod); err != nil {
// t.Errorf("Expected the pod to be scheduled, got an error: %v", err)
// }
// for i, pl := range reservePlugins {
// if pl.numUnreserveCalled != 0 {
// t.Errorf("reservePlugins[%d].numUnreserveCalled = %d, want 0", i, pl.numUnreserveCalled)
// }
// }
// }
// testutils.CleanupPods(testCtx.ClientSet, t, []*v1.Pod{pod})
// })
// }
// }
type pluginInvokeEvent struct {
pluginName string
@ -1155,26 +1166,28 @@ func TestBindPlugin(t *testing.T) {
}
// Setup initial unreserve and bind plugins for testing.
prof := schedulerconfig.KubeSchedulerProfile{
SchedulerName: v1.DefaultSchedulerName,
Plugins: &schedulerconfig.Plugins{
Reserve: schedulerconfig.PluginSet{
Enabled: []schedulerconfig.Plugin{{Name: reservePlugin.Name()}},
cfg := configtesting.V1beta2ToInternalWithDefaults(t, v1beta2.KubeSchedulerConfiguration{
Profiles: []v1beta2.KubeSchedulerProfile{{
SchedulerName: pointer.StringPtr(v1.DefaultSchedulerName),
Plugins: &v1beta2.Plugins{
Reserve: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{{Name: reservePlugin.Name()}},
},
Bind: schedulerconfig.PluginSet{
Bind: v1beta2.PluginSet{
// Put DefaultBinder last.
Enabled: []schedulerconfig.Plugin{{Name: bindPlugin1.Name()}, {Name: bindPlugin2.Name()}, {Name: defaultbinder.Name}},
Disabled: []schedulerconfig.Plugin{{Name: defaultbinder.Name}},
Enabled: []v1beta2.Plugin{{Name: bindPlugin1.Name()}, {Name: bindPlugin2.Name()}, {Name: defaultbinder.Name}},
Disabled: []v1beta2.Plugin{{Name: defaultbinder.Name}},
},
PostBind: schedulerconfig.PluginSet{
Enabled: []schedulerconfig.Plugin{{Name: postBindPlugin.Name()}},
PostBind: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{{Name: postBindPlugin.Name()}},
},
},
}
}},
})
// Create the scheduler with the test plugin set.
testCtx := testutils.InitTestSchedulerWithOptions(t, testContext, nil,
scheduler.WithProfiles(prof),
scheduler.WithProfiles(cfg.Profiles...),
scheduler.WithFrameworkOutOfTreeRegistry(registry))
testutils.SyncInformerFactory(testCtx)
go testCtx.Scheduler.Run(testCtx.Ctx)
@ -1341,29 +1354,31 @@ func TestPostBindPlugin(t *testing.T) {
}
// Setup initial prebind and postbind plugin for testing.
prof := schedulerconfig.KubeSchedulerProfile{
SchedulerName: v1.DefaultSchedulerName,
Plugins: &schedulerconfig.Plugins{
PreBind: schedulerconfig.PluginSet{
Enabled: []schedulerconfig.Plugin{
cfg := configtesting.V1beta2ToInternalWithDefaults(t, v1beta2.KubeSchedulerConfiguration{
Profiles: []v1beta2.KubeSchedulerProfile{{
SchedulerName: pointer.StringPtr(v1.DefaultSchedulerName),
Plugins: &v1beta2.Plugins{
PreBind: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{
Name: preBindPluginName,
},
},
},
PostBind: schedulerconfig.PluginSet{
Enabled: []schedulerconfig.Plugin{
PostBind: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{
Name: postBindPluginName,
},
},
},
},
}
}},
})
// Create the master and the scheduler with the test plugin set.
testCtx := initTestSchedulerForFrameworkTest(t, testutils.InitTestMaster(t, "postbind-plugin", nil), 2,
scheduler.WithProfiles(prof),
scheduler.WithProfiles(cfg.Profiles...),
scheduler.WithFrameworkOutOfTreeRegistry(registry))
defer testutils.CleanupTest(t, testCtx)
@ -1404,7 +1419,7 @@ func TestPostBindPlugin(t *testing.T) {
func TestPermitPlugin(t *testing.T) {
// Create a plugin registry for testing. Register only a permit plugin.
perPlugin := &PermitPlugin{name: permitPluginName}
registry, prof := initRegistryAndConfig(perPlugin)
registry, prof := initRegistryAndConfig(t, perPlugin)
// Create the master and the scheduler with the test plugin set.
testCtx := initTestSchedulerForFrameworkTest(t, testutils.InitTestMaster(t, "permit-plugin", nil), 2,
@ -1501,7 +1516,7 @@ func TestMultiplePermitPlugins(t *testing.T) {
// Create a plugin registry for testing.
perPlugin1 := &PermitPlugin{name: "permit-plugin-1"}
perPlugin2 := &PermitPlugin{name: "permit-plugin-2"}
registry, prof := initRegistryAndConfig(perPlugin1, perPlugin2)
registry, prof := initRegistryAndConfig(t, perPlugin1, perPlugin2)
// Create the master and the scheduler with the test plugin set.
testCtx := initTestSchedulerForFrameworkTest(t, testutils.InitTestMaster(t, "multi-permit-plugin", nil), 2,
@ -1556,7 +1571,7 @@ func TestPermitPluginsCancelled(t *testing.T) {
// Create a plugin registry for testing.
perPlugin1 := &PermitPlugin{name: "permit-plugin-1"}
perPlugin2 := &PermitPlugin{name: "permit-plugin-2"}
registry, prof := initRegistryAndConfig(perPlugin1, perPlugin2)
registry, prof := initRegistryAndConfig(t, perPlugin1, perPlugin2)
// Create the master and the scheduler with the test plugin set.
testCtx := initTestSchedulerForFrameworkTest(t, testutils.InitTestMaster(t, "permit-plugins", nil), 2,
@ -1597,7 +1612,7 @@ func TestPermitPluginsCancelled(t *testing.T) {
func TestCoSchedulingWithPermitPlugin(t *testing.T) {
// Create a plugin registry for testing. Register only a permit plugin.
permitPlugin := &PermitPlugin{name: permitPluginName}
registry, prof := initRegistryAndConfig(permitPlugin)
registry, prof := initRegistryAndConfig(t, permitPlugin)
// Create the master and the scheduler with the test plugin set.
// TODO Make the subtests not share scheduler instances.
@ -1687,22 +1702,22 @@ func TestFilterPlugin(t *testing.T) {
registry := frameworkruntime.Registry{filterPluginName: newPlugin(filterPlugin)}
// Setup initial filter plugin for testing.
prof := schedulerconfig.KubeSchedulerProfile{
SchedulerName: v1.DefaultSchedulerName,
Plugins: &schedulerconfig.Plugins{
Filter: schedulerconfig.PluginSet{
Enabled: []schedulerconfig.Plugin{
{
Name: filterPluginName,
cfg := configtesting.V1beta2ToInternalWithDefaults(t, v1beta2.KubeSchedulerConfiguration{
Profiles: []v1beta2.KubeSchedulerProfile{{
SchedulerName: pointer.StringPtr(v1.DefaultSchedulerName),
Plugins: &v1beta2.Plugins{
Filter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: filterPluginName},
},
},
},
},
}
}},
})
// Create the master and the scheduler with the test plugin set.
testCtx := initTestSchedulerForFrameworkTest(t, testutils.InitTestMaster(t, "filter-plugin", nil), 1,
scheduler.WithProfiles(prof),
scheduler.WithProfiles(cfg.Profiles...),
scheduler.WithFrameworkOutOfTreeRegistry(registry))
defer testutils.CleanupTest(t, testCtx)
@ -1759,22 +1774,22 @@ func TestPreScorePlugin(t *testing.T) {
registry := frameworkruntime.Registry{preScorePluginName: newPlugin(preScorePlugin)}
// Setup initial pre-score plugin for testing.
prof := schedulerconfig.KubeSchedulerProfile{
SchedulerName: v1.DefaultSchedulerName,
Plugins: &schedulerconfig.Plugins{
PreScore: schedulerconfig.PluginSet{
Enabled: []schedulerconfig.Plugin{
{
Name: preScorePluginName,
cfg := configtesting.V1beta2ToInternalWithDefaults(t, v1beta2.KubeSchedulerConfiguration{
Profiles: []v1beta2.KubeSchedulerProfile{{
SchedulerName: pointer.StringPtr(v1.DefaultSchedulerName),
Plugins: &v1beta2.Plugins{
PreScore: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: preScorePluginName},
},
},
},
},
}
}},
})
// Create the master and the scheduler with the test plugin set.
testCtx := initTestSchedulerForFrameworkTest(t, testutils.InitTestMaster(t, "pre-score-plugin", nil), 2,
scheduler.WithProfiles(prof),
scheduler.WithProfiles(cfg.Profiles...),
scheduler.WithFrameworkOutOfTreeRegistry(registry))
defer testutils.CleanupTest(t, testCtx)
@ -1826,7 +1841,7 @@ func TestPreScorePlugin(t *testing.T) {
func TestPreemptWithPermitPlugin(t *testing.T) {
// Create a plugin registry for testing. Register only a permit plugin.
permitPlugin := &PermitPlugin{}
registry, prof := initRegistryAndConfig(permitPlugin)
registry, prof := initRegistryAndConfig(t, permitPlugin)
// Create the master and the scheduler with the test plugin set.
testCtx := initTestSchedulerForFrameworkTest(t, testutils.InitTestMaster(t, "preempt-with-permit-plugin", nil), 0,
@ -1941,23 +1956,25 @@ func initTestSchedulerForFrameworkTest(t *testing.T, testCtx *testutils.TestCont
// initRegistryAndConfig returns registry and plugins config based on give plugins.
// TODO: refactor it to a more generic functions that accepts all kinds of Plugins as arguments
func initRegistryAndConfig(pp ...*PermitPlugin) (registry frameworkruntime.Registry, prof schedulerconfig.KubeSchedulerProfile) {
func initRegistryAndConfig(t *testing.T, pp ...*PermitPlugin) (frameworkruntime.Registry, schedulerconfig.KubeSchedulerProfile) {
var registry frameworkruntime.Registry
if len(pp) == 0 {
return
return frameworkruntime.Registry{}, schedulerconfig.KubeSchedulerProfile{}
}
versionedCfg := v1beta2.KubeSchedulerConfiguration{
Profiles: []v1beta2.KubeSchedulerProfile{{
SchedulerName: pointer.StringPtr(v1.DefaultSchedulerName),
Plugins: &v1beta2.Plugins{
Permit: v1beta2.PluginSet{},
},
}},
}
registry = frameworkruntime.Registry{}
var plugins []schedulerconfig.Plugin
for _, p := range pp {
registry.Register(p.Name(), newPermitPlugin(p))
plugins = append(plugins, schedulerconfig.Plugin{Name: p.Name()})
versionedCfg.Profiles[0].Plugins.Permit.Enabled = append(versionedCfg.Profiles[0].Plugins.Permit.Enabled, v1beta2.Plugin{Name: p.Name()})
}
prof.SchedulerName = v1.DefaultSchedulerName
prof.Plugins = &schedulerconfig.Plugins{
Permit: schedulerconfig.PluginSet{
Enabled: plugins,
},
}
return
cfg := configtesting.V1beta2ToInternalWithDefaults(t, versionedCfg)
return registry, cfg.Profiles[0]
}

View File

@ -40,16 +40,18 @@ import (
restclient "k8s.io/client-go/rest"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/klog/v2"
"k8s.io/kube-scheduler/config/v1beta2"
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
"k8s.io/kubernetes/pkg/apis/scheduling"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/scheduler"
schedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config"
configtesting "k8s.io/kubernetes/pkg/scheduler/apis/config/testing"
framework "k8s.io/kubernetes/pkg/scheduler/framework"
frameworkruntime "k8s.io/kubernetes/pkg/scheduler/framework/runtime"
st "k8s.io/kubernetes/pkg/scheduler/testing"
"k8s.io/kubernetes/plugin/pkg/admission/priority"
testutils "k8s.io/kubernetes/test/integration/util"
"k8s.io/utils/pointer"
)
var lowPriority, mediumPriority, highPriority = int32(100), int32(200), int32(300)
@ -132,25 +134,28 @@ func TestPreemption(t *testing.T) {
if err != nil {
t.Fatalf("Error registering a filter: %v", err)
}
prof := schedulerconfig.KubeSchedulerProfile{
SchedulerName: v1.DefaultSchedulerName,
Plugins: &schedulerconfig.Plugins{
Filter: schedulerconfig.PluginSet{
Enabled: []schedulerconfig.Plugin{
cfg := configtesting.V1beta2ToInternalWithDefaults(t, v1beta2.KubeSchedulerConfiguration{
Profiles: []v1beta2.KubeSchedulerProfile{{
SchedulerName: pointer.StringPtr(v1.DefaultSchedulerName),
Plugins: &v1beta2.Plugins{
Filter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: filterPluginName},
},
},
PreFilter: schedulerconfig.PluginSet{
Enabled: []schedulerconfig.Plugin{
PreFilter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: filterPluginName},
},
},
},
}
}},
})
testCtx := testutils.InitTestSchedulerWithOptions(t,
testutils.InitTestMaster(t, "preemption", nil),
nil,
scheduler.WithProfiles(prof),
scheduler.WithProfiles(cfg.Profiles...),
scheduler.WithFrameworkOutOfTreeRegistry(registry))
testutils.SyncInformerFactory(testCtx)
go testCtx.Scheduler.Run(testCtx.Ctx)

View File

@ -29,9 +29,10 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/kube-scheduler/config/v1beta2"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/scheduler"
schedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config"
configtesting "k8s.io/kubernetes/pkg/scheduler/apis/config/testing"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/imagelocality"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/interpodaffinity"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeaffinity"
@ -39,28 +40,31 @@ import (
st "k8s.io/kubernetes/pkg/scheduler/testing"
testutils "k8s.io/kubernetes/test/integration/util"
imageutils "k8s.io/kubernetes/test/utils/image"
"k8s.io/utils/pointer"
)
// This file tests the scheduler priority functions.
func initTestSchedulerForPriorityTest(t *testing.T, scorePluginName string) *testutils.TestContext {
prof := schedulerconfig.KubeSchedulerProfile{
SchedulerName: v1.DefaultSchedulerName,
Plugins: &schedulerconfig.Plugins{
Score: schedulerconfig.PluginSet{
Enabled: []schedulerconfig.Plugin{
{Name: scorePluginName, Weight: 1},
cfg := configtesting.V1beta2ToInternalWithDefaults(t, v1beta2.KubeSchedulerConfiguration{
Profiles: []v1beta2.KubeSchedulerProfile{{
SchedulerName: pointer.StringPtr(v1.DefaultSchedulerName),
Plugins: &v1beta2.Plugins{
Score: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: scorePluginName, Weight: pointer.Int32Ptr(1)},
},
Disabled: []schedulerconfig.Plugin{
Disabled: []v1beta2.Plugin{
{Name: "*"},
},
},
},
}
}},
})
testCtx := testutils.InitTestSchedulerWithOptions(
t,
testutils.InitTestMaster(t, strings.ToLower(scorePluginName), nil),
nil,
scheduler.WithProfiles(prof),
scheduler.WithProfiles(cfg.Profiles...),
)
testutils.SyncInformerFactory(testCtx)
go testCtx.Scheduler.Run(testCtx.Ctx)

View File

@ -34,9 +34,11 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"
"k8s.io/kube-scheduler/config/v1beta1"
"k8s.io/kube-scheduler/config/v1beta2"
apiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
"k8s.io/kubernetes/pkg/scheduler"
schedapi "k8s.io/kubernetes/pkg/scheduler/apis/config"
configtesting "k8s.io/kubernetes/pkg/scheduler/apis/config/testing"
"k8s.io/kubernetes/pkg/scheduler/framework"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/serviceaffinity"
frameworkruntime "k8s.io/kubernetes/pkg/scheduler/framework/runtime"
@ -44,38 +46,44 @@ import (
testfwk "k8s.io/kubernetes/test/integration/framework"
testutils "k8s.io/kubernetes/test/integration/util"
imageutils "k8s.io/kubernetes/test/utils/image"
"k8s.io/utils/pointer"
)
func TestServiceAffinityEnqueue(t *testing.T) {
profile := schedapi.KubeSchedulerProfile{
SchedulerName: v1.DefaultSchedulerName,
Plugins: &schedapi.Plugins{
PreFilter: schedapi.PluginSet{
Enabled: []schedapi.Plugin{
cfg := configtesting.V1beta1ToInternalWithDefaults(t, v1beta1.KubeSchedulerConfiguration{
Profiles: []v1beta1.KubeSchedulerProfile{{
SchedulerName: pointer.StringPtr(v1.DefaultSchedulerName),
Plugins: &v1beta1.Plugins{
PreFilter: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: serviceaffinity.Name},
},
},
Filter: schedapi.PluginSet{
Enabled: []schedapi.Plugin{
Filter: &v1beta1.PluginSet{
Enabled: []v1beta1.Plugin{
{Name: serviceaffinity.Name},
},
},
},
PluginConfig: []schedapi.PluginConfig{
PluginConfig: []v1beta1.PluginConfig{
{
Name: serviceaffinity.Name,
Args: &schedapi.ServiceAffinityArgs{
Args: runtime.RawExtension{
Object: &v1beta1.ServiceAffinityArgs{
AffinityLabels: []string{"hostname"},
},
},
},
}
},
}},
})
// Use zero backoff seconds to bypass backoffQ.
testCtx := testutils.InitTestSchedulerWithOptions(
t,
testutils.InitTestMaster(t, "serviceaffinity-enqueue", nil),
nil,
scheduler.WithProfiles(profile),
scheduler.WithProfiles(cfg.Profiles...),
scheduler.WithPodInitialBackoffSeconds(0),
scheduler.WithPodMaxBackoffSeconds(0),
)
@ -239,16 +247,17 @@ func TestCustomResourceEnqueue(t *testing.T) {
return &fakeCRPlugin{}, nil
},
}
profile := schedapi.KubeSchedulerProfile{
SchedulerName: v1.DefaultSchedulerName,
Plugins: &schedapi.Plugins{
Filter: schedapi.PluginSet{
Enabled: []schedapi.Plugin{
cfg := configtesting.V1beta2ToInternalWithDefaults(t, v1beta2.KubeSchedulerConfiguration{
Profiles: []v1beta2.KubeSchedulerProfile{{
SchedulerName: pointer.StringPtr(v1.DefaultSchedulerName),
Plugins: &v1beta2.Plugins{
Filter: v1beta2.PluginSet{
Enabled: []v1beta2.Plugin{
{Name: "fakeCRPlugin"},
},
},
},
}
}}})
testCtx.KubeConfig = server.ClientConfig
testCtx.ClientSet = kubernetes.NewForConfigOrDie(server.ClientConfig)
@ -263,7 +272,7 @@ func TestCustomResourceEnqueue(t *testing.T) {
t,
testCtx,
nil,
scheduler.WithProfiles(profile),
scheduler.WithProfiles(cfg.Profiles...),
scheduler.WithFrameworkOutOfTreeRegistry(registry),
scheduler.WithPodInitialBackoffSeconds(0),
scheduler.WithPodMaxBackoffSeconds(0),

View File

@ -28,6 +28,7 @@ import (
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apimachinery/pkg/watch"
@ -36,12 +37,15 @@ import (
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/events"
"k8s.io/kube-scheduler/config/v1beta2"
"k8s.io/kubernetes/pkg/scheduler"
kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config"
"k8s.io/kubernetes/pkg/scheduler/apis/config"
configtesting "k8s.io/kubernetes/pkg/scheduler/apis/config/testing"
"k8s.io/kubernetes/pkg/scheduler/profile"
st "k8s.io/kubernetes/pkg/scheduler/testing"
"k8s.io/kubernetes/test/integration/framework"
testutils "k8s.io/kubernetes/test/integration/util"
"k8s.io/utils/pointer"
)
type nodeMutationFunc func(t *testing.T, n *v1.Node, nodeLister corelisters.NodeLister, c clientset.Interface)
@ -67,7 +71,7 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) {
for i, test := range []struct {
policy string
expectedPlugins map[string][]kubeschedulerconfig.Plugin
expectedPlugins config.Plugins
}{
{
policy: `{
@ -80,21 +84,17 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) {
{"name" : "ImageLocalityPriority", "weight" : 1}
]
}`,
expectedPlugins: map[string][]kubeschedulerconfig.Plugin{
"QueueSortPlugin": {{Name: "PrioritySort"}},
"PreFilterPlugin": {
{Name: "NodeResourcesFit"},
},
"FilterPlugin": {
expectedPlugins: config.Plugins{
QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: "PrioritySort"}}},
PreFilter: config.PluginSet{Enabled: []config.Plugin{{Name: "NodeResourcesFit"}}},
Filter: config.PluginSet{Enabled: []config.Plugin{
{Name: "NodeUnschedulable"},
{Name: "NodeResourcesFit"},
{Name: "TaintToleration"},
},
"PostFilterPlugin": {{Name: "DefaultPreemption"}},
"ScorePlugin": {
{Name: "ImageLocality", Weight: 1},
},
"BindPlugin": {{Name: "DefaultBinder"}},
}},
PostFilter: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultPreemption"}}},
Score: config.PluginSet{Enabled: []config.Plugin{{Name: "ImageLocality", Weight: 1}}},
Bind: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultBinder"}}},
},
},
{
@ -102,17 +102,17 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) {
"kind" : "Policy",
"apiVersion" : "v1"
}`,
expectedPlugins: map[string][]kubeschedulerconfig.Plugin{
"QueueSortPlugin": {{Name: "PrioritySort"}},
"PreFilterPlugin": {
expectedPlugins: config.Plugins{
QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: "PrioritySort"}}},
PreFilter: config.PluginSet{Enabled: []config.Plugin{
{Name: "NodeResourcesFit"},
{Name: "NodePorts"},
{Name: "NodeAffinity"},
{Name: "VolumeBinding"},
{Name: "PodTopologySpread"},
{Name: "InterPodAffinity"},
},
"FilterPlugin": {
}},
Filter: config.PluginSet{Enabled: []config.Plugin{
{Name: "NodeUnschedulable"},
{Name: "NodeResourcesFit"},
{Name: "NodeName"},
@ -128,15 +128,15 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) {
{Name: "VolumeZone"},
{Name: "PodTopologySpread"},
{Name: "InterPodAffinity"},
},
"PostFilterPlugin": {{Name: "DefaultPreemption"}},
"PreScorePlugin": {
}},
PostFilter: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultPreemption"}}},
PreScore: config.PluginSet{Enabled: []config.Plugin{
{Name: "PodTopologySpread"},
{Name: "InterPodAffinity"},
{Name: "NodeAffinity"},
{Name: "TaintToleration"},
},
"ScorePlugin": {
}},
Score: config.PluginSet{Enabled: []config.Plugin{
{Name: "NodeResourcesBalancedAllocation", Weight: 1},
{Name: "PodTopologySpread", Weight: 2},
{Name: "ImageLocality", Weight: 1},
@ -145,10 +145,10 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) {
{Name: "NodeAffinity", Weight: 1},
{Name: "NodePreferAvoidPods", Weight: 10000},
{Name: "TaintToleration", Weight: 1},
},
"ReservePlugin": {{Name: "VolumeBinding"}},
"PreBindPlugin": {{Name: "VolumeBinding"}},
"BindPlugin": {{Name: "DefaultBinder"}},
}},
Reserve: config.PluginSet{Enabled: []config.Plugin{{Name: "VolumeBinding"}}},
PreBind: config.PluginSet{Enabled: []config.Plugin{{Name: "VolumeBinding"}}},
Bind: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultBinder"}}},
},
},
{
@ -158,14 +158,14 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) {
"predicates" : [],
"priorities" : []
}`,
expectedPlugins: map[string][]kubeschedulerconfig.Plugin{
"QueueSortPlugin": {{Name: "PrioritySort"}},
"FilterPlugin": {
expectedPlugins: config.Plugins{
QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: "PrioritySort"}}},
Filter: config.PluginSet{Enabled: []config.Plugin{
{Name: "NodeUnschedulable"},
{Name: "TaintToleration"},
},
"PostFilterPlugin": {{Name: "DefaultPreemption"}},
"BindPlugin": {{Name: "DefaultBinder"}},
}},
PostFilter: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultPreemption"}}},
Bind: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultBinder"}}},
},
},
{
@ -177,38 +177,38 @@ priorities:
- name: ImageLocalityPriority
weight: 1
`,
expectedPlugins: map[string][]kubeschedulerconfig.Plugin{
"QueueSortPlugin": {{Name: "PrioritySort"}},
"PreFilterPlugin": {
expectedPlugins: config.Plugins{
QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: "PrioritySort"}}},
PreFilter: config.PluginSet{Enabled: []config.Plugin{
{Name: "NodeResourcesFit"},
},
"FilterPlugin": {
}},
Filter: config.PluginSet{Enabled: []config.Plugin{
{Name: "NodeUnschedulable"},
{Name: "NodeResourcesFit"},
{Name: "TaintToleration"},
},
"PostFilterPlugin": {{Name: "DefaultPreemption"}},
"ScorePlugin": {
}},
PostFilter: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultPreemption"}}},
Score: config.PluginSet{Enabled: []config.Plugin{
{Name: "ImageLocality", Weight: 1},
},
"BindPlugin": {{Name: "DefaultBinder"}},
}},
Bind: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultBinder"}}},
},
},
{
policy: `apiVersion: v1
kind: Policy
`,
expectedPlugins: map[string][]kubeschedulerconfig.Plugin{
"QueueSortPlugin": {{Name: "PrioritySort"}},
"PreFilterPlugin": {
expectedPlugins: config.Plugins{
QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: "PrioritySort"}}},
PreFilter: config.PluginSet{Enabled: []config.Plugin{
{Name: "NodeResourcesFit"},
{Name: "NodePorts"},
{Name: "NodeAffinity"},
{Name: "VolumeBinding"},
{Name: "PodTopologySpread"},
{Name: "InterPodAffinity"},
},
"FilterPlugin": {
}},
Filter: config.PluginSet{Enabled: []config.Plugin{
{Name: "NodeUnschedulable"},
{Name: "NodeResourcesFit"},
{Name: "NodeName"},
@ -224,15 +224,15 @@ kind: Policy
{Name: "VolumeZone"},
{Name: "PodTopologySpread"},
{Name: "InterPodAffinity"},
},
"PostFilterPlugin": {{Name: "DefaultPreemption"}},
"PreScorePlugin": {
}},
PostFilter: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultPreemption"}}},
PreScore: config.PluginSet{Enabled: []config.Plugin{
{Name: "PodTopologySpread"},
{Name: "InterPodAffinity"},
{Name: "NodeAffinity"},
{Name: "TaintToleration"},
},
"ScorePlugin": {
}},
Score: config.PluginSet{Enabled: []config.Plugin{
{Name: "NodeResourcesBalancedAllocation", Weight: 1},
{Name: "PodTopologySpread", Weight: 2},
{Name: "ImageLocality", Weight: 1},
@ -241,10 +241,10 @@ kind: Policy
{Name: "NodeAffinity", Weight: 1},
{Name: "NodePreferAvoidPods", Weight: 10000},
{Name: "TaintToleration", Weight: 1},
},
"ReservePlugin": {{Name: "VolumeBinding"}},
"PreBindPlugin": {{Name: "VolumeBinding"}},
"BindPlugin": {{Name: "DefaultBinder"}},
}},
Reserve: config.PluginSet{Enabled: []config.Plugin{{Name: "VolumeBinding"}}},
PreBind: config.PluginSet{Enabled: []config.Plugin{{Name: "VolumeBinding"}}},
Bind: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultBinder"}}},
},
},
{
@ -253,14 +253,14 @@ kind: Policy
predicates: []
priorities: []
`,
expectedPlugins: map[string][]kubeschedulerconfig.Plugin{
"QueueSortPlugin": {{Name: "PrioritySort"}},
"FilterPlugin": {
expectedPlugins: config.Plugins{
QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: "PrioritySort"}}},
Filter: config.PluginSet{Enabled: []config.Plugin{
{Name: "NodeUnschedulable"},
{Name: "TaintToleration"},
},
"PostFilterPlugin": {{Name: "DefaultPreemption"}},
"BindPlugin": {{Name: "DefaultBinder"}},
}},
PostFilter: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultPreemption"}}},
Bind: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultBinder"}}},
},
},
} {
@ -268,7 +268,7 @@ priorities: []
configPolicyName := fmt.Sprintf("scheduler-custom-policy-config-%d", i)
policyConfigMap := v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: configPolicyName},
Data: map[string]string{kubeschedulerconfig.SchedulerPolicyConfigMapKey: test.policy},
Data: map[string]string{config.SchedulerPolicyConfigMapKey: test.policy},
}
policyConfigMap.APIVersion = "v1"
@ -282,30 +282,20 @@ priorities: []
informerFactory,
profile.NewRecorderFactory(eventBroadcaster),
nil,
scheduler.WithLegacyPolicySource(&kubeschedulerconfig.SchedulerPolicySource{
ConfigMap: &kubeschedulerconfig.SchedulerPolicyConfigMapSource{
scheduler.WithProfiles([]config.KubeSchedulerProfile(nil)...),
scheduler.WithLegacyPolicySource(&config.SchedulerPolicySource{
ConfigMap: &config.SchedulerPolicyConfigMapSource{
Namespace: policyConfigMap.Namespace,
Name: policyConfigMap.Name,
},
}),
scheduler.WithProfiles(kubeschedulerconfig.KubeSchedulerProfile{
SchedulerName: v1.DefaultSchedulerName,
PluginConfig: []kubeschedulerconfig.PluginConfig{
{
Name: "VolumeBinding",
Args: &kubeschedulerconfig.VolumeBindingArgs{
BindTimeoutSeconds: 30,
},
},
},
}),
)
if err != nil {
t.Fatalf("couldn't make scheduler config for test %d: %v", i, err)
}
schedPlugins := sched.Profiles[v1.DefaultSchedulerName].ListPlugins()
if diff := cmp.Diff(test.expectedPlugins, schedPlugins); diff != "" {
if diff := cmp.Diff(&test.expectedPlugins, schedPlugins); diff != "" {
t.Errorf("unexpected plugins diff (-want, +got): %s", diff)
}
}
@ -329,27 +319,33 @@ func TestSchedulerCreationFromNonExistentConfigMap(t *testing.T) {
stopCh := make(chan struct{})
eventBroadcaster.StartRecordingToSink(stopCh)
cfg := configtesting.V1beta2ToInternalWithDefaults(t, v1beta2.KubeSchedulerConfiguration{
Profiles: []v1beta2.KubeSchedulerProfile{{
SchedulerName: pointer.StringPtr(v1.DefaultSchedulerName),
PluginConfig: []v1beta2.PluginConfig{
{
Name: "VolumeBinding",
Args: runtime.RawExtension{
Object: &v1beta2.VolumeBindingArgs{
BindTimeoutSeconds: pointer.Int64Ptr(30),
},
},
},
}},
},
})
_, err := scheduler.New(clientSet,
informerFactory,
profile.NewRecorderFactory(eventBroadcaster),
nil,
scheduler.WithLegacyPolicySource(&kubeschedulerconfig.SchedulerPolicySource{
ConfigMap: &kubeschedulerconfig.SchedulerPolicyConfigMapSource{
scheduler.WithLegacyPolicySource(&config.SchedulerPolicySource{
ConfigMap: &config.SchedulerPolicyConfigMapSource{
Namespace: "non-existent-config",
Name: "non-existent-config",
},
}),
scheduler.WithProfiles(kubeschedulerconfig.KubeSchedulerProfile{
SchedulerName: v1.DefaultSchedulerName,
PluginConfig: []kubeschedulerconfig.PluginConfig{
{
Name: "VolumeBinding",
Args: &kubeschedulerconfig.VolumeBindingArgs{
BindTimeoutSeconds: 30,
},
},
},
}),
scheduler.WithProfiles(cfg.Profiles...),
)
if err == nil {
@ -560,8 +556,22 @@ func TestMultipleSchedulers(t *testing.T) {
}
// 5. create and start a scheduler with name "foo-scheduler"
fooProf := kubeschedulerconfig.KubeSchedulerProfile{SchedulerName: fooScheduler}
testCtx = testutils.InitTestSchedulerWithOptions(t, testCtx, nil, scheduler.WithProfiles(fooProf))
cfg := configtesting.V1beta2ToInternalWithDefaults(t, v1beta2.KubeSchedulerConfiguration{
Profiles: []v1beta2.KubeSchedulerProfile{{
SchedulerName: pointer.StringPtr(fooScheduler),
PluginConfig: []v1beta2.PluginConfig{
{
Name: "VolumeBinding",
Args: runtime.RawExtension{
Object: &v1beta2.VolumeBindingArgs{
BindTimeoutSeconds: pointer.Int64Ptr(30),
},
},
},
}},
},
})
testCtx = testutils.InitTestSchedulerWithOptions(t, testCtx, nil, scheduler.WithProfiles(cfg.Profiles...))
testutils.SyncInformerFactory(testCtx)
go testCtx.Scheduler.Run(testCtx.Ctx)
@ -622,14 +632,14 @@ func TestMultipleSchedulers(t *testing.T) {
}
func TestMultipleSchedulingProfiles(t *testing.T) {
testCtx := initTest(t, "multi-scheduler", scheduler.WithProfiles(
kubeschedulerconfig.KubeSchedulerProfile{
SchedulerName: "default-scheduler",
cfg := configtesting.V1beta2ToInternalWithDefaults(t, v1beta2.KubeSchedulerConfiguration{
Profiles: []v1beta2.KubeSchedulerProfile{
{SchedulerName: pointer.StringPtr("default-scheduler")},
{SchedulerName: pointer.StringPtr("custom-scheduler")},
},
kubeschedulerconfig.KubeSchedulerProfile{
SchedulerName: "custom-scheduler",
},
))
})
testCtx := initTest(t, "multi-scheduler", scheduler.WithProfiles(cfg.Profiles...))
defer testutils.CleanupTest(t, testCtx)
node := &v1.Node{

View File

@ -35,14 +35,16 @@ import (
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/restmapper"
"k8s.io/client-go/scale"
"k8s.io/kube-scheduler/config/v1beta2"
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
"k8s.io/kubernetes/pkg/controller/disruption"
"k8s.io/kubernetes/pkg/scheduler"
schedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config"
configtesting "k8s.io/kubernetes/pkg/scheduler/apis/config/testing"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultpreemption"
st "k8s.io/kubernetes/pkg/scheduler/testing"
testutils "k8s.io/kubernetes/test/integration/util"
imageutils "k8s.io/kubernetes/test/utils/image"
"k8s.io/utils/pointer"
)
// initDisruptionController initializes and runs a Disruption Controller to properly
@ -90,19 +92,21 @@ func initTest(t *testing.T, nsPrefix string, opts ...scheduler.Option) *testutil
// initTestDisablePreemption initializes a test environment and creates master and scheduler with default
// configuration but with pod preemption disabled.
func initTestDisablePreemption(t *testing.T, nsPrefix string) *testutils.TestContext {
prof := schedulerconfig.KubeSchedulerProfile{
SchedulerName: v1.DefaultSchedulerName,
Plugins: &schedulerconfig.Plugins{
PostFilter: schedulerconfig.PluginSet{
Disabled: []schedulerconfig.Plugin{
cfg := configtesting.V1beta2ToInternalWithDefaults(t, v1beta2.KubeSchedulerConfiguration{
Profiles: []v1beta2.KubeSchedulerProfile{{
SchedulerName: pointer.StringPtr(v1.DefaultSchedulerName),
Plugins: &v1beta2.Plugins{
PostFilter: v1beta2.PluginSet{
Disabled: []v1beta2.Plugin{
{Name: defaultpreemption.Name},
},
},
},
}
}},
})
testCtx := testutils.InitTestSchedulerWithOptions(
t, testutils.InitTestMaster(t, nsPrefix, nil), nil,
scheduler.WithProfiles(prof))
scheduler.WithProfiles(cfg.Profiles...))
testutils.SyncInformerFactory(testCtx)
go testCtx.Scheduler.Run(testCtx.Ctx)
return testCtx