Add MultiPoint scheduler plugin config field

This commit is contained in:
Mike Dame 2021-10-11 13:47:23 -04:00
parent 582c4ebe26
commit 420c5308bb
20 changed files with 873 additions and 344 deletions

View File

@ -690,24 +690,24 @@ profiles:
{
SchedulerName: "default-scheduler",
Plugins: &kubeschedulerconfig.Plugins{
QueueSort: defaults.PluginsV1beta3.QueueSort,
PreFilter: defaults.PluginsV1beta3.PreFilter,
Filter: defaults.PluginsV1beta3.Filter,
PostFilter: defaults.PluginsV1beta3.PostFilter,
PreScore: defaults.PluginsV1beta3.PreScore,
Score: defaults.PluginsV1beta3.Score,
Reserve: kubeschedulerconfig.PluginSet{
Enabled: []kubeschedulerconfig.Plugin{
{Name: "foo"},
{Name: "bar"},
},
Disabled: []kubeschedulerconfig.Plugin{
{Name: names.VolumeBinding},
},
},
PreBind: kubeschedulerconfig.PluginSet{
Enabled: []kubeschedulerconfig.Plugin{
{Name: "foo"},
},
Disabled: []kubeschedulerconfig.Plugin{
{Name: names.VolumeBinding},
},
},
Bind: defaults.PluginsV1beta3.Bind,
MultiPoint: defaults.PluginsV1beta3.MultiPoint,
},
PluginConfig: []kubeschedulerconfig.PluginConfig{
{
@ -918,19 +918,15 @@ profiles:
{
SchedulerName: "foo-profile",
Plugins: &kubeschedulerconfig.Plugins{
QueueSort: defaults.PluginsV1beta3.QueueSort,
PreFilter: defaults.PluginsV1beta3.PreFilter,
Filter: defaults.PluginsV1beta3.Filter,
PostFilter: defaults.PluginsV1beta3.PostFilter,
PreScore: defaults.PluginsV1beta3.PreScore,
Score: defaults.PluginsV1beta3.Score,
Bind: defaults.PluginsV1beta3.Bind,
PreBind: defaults.PluginsV1beta3.PreBind,
MultiPoint: defaults.PluginsV1beta3.MultiPoint,
Reserve: kubeschedulerconfig.PluginSet{
Enabled: []kubeschedulerconfig.Plugin{
{Name: "foo"},
{Name: names.VolumeBinding},
},
Disabled: []kubeschedulerconfig.Plugin{
{Name: names.VolumeBinding},
},
},
},
PluginConfig: defaults.PluginConfigsV1beta3,
@ -938,14 +934,12 @@ profiles:
{
SchedulerName: "bar-profile",
Plugins: &kubeschedulerconfig.Plugins{
QueueSort: defaults.PluginsV1beta3.QueueSort,
PreFilter: defaults.PluginsV1beta3.PreFilter,
Filter: defaults.PluginsV1beta3.Filter,
PostFilter: defaults.PluginsV1beta3.PostFilter,
PreScore: defaults.PluginsV1beta3.PreScore,
Score: defaults.PluginsV1beta3.Score,
Bind: defaults.PluginsV1beta3.Bind,
Reserve: defaults.PluginsV1beta3.Reserve,
MultiPoint: defaults.PluginsV1beta3.MultiPoint,
PreBind: kubeschedulerconfig.PluginSet{
Disabled: []kubeschedulerconfig.Plugin{
{Name: names.VolumeBinding},
},
},
},
PluginConfig: []kubeschedulerconfig.PluginConfig{
{

View File

@ -82,8 +82,46 @@ users:
}
// plugin config
pluginConfigFile := filepath.Join(tmpDir, "plugin.yaml")
if err := ioutil.WriteFile(pluginConfigFile, []byte(fmt.Sprintf(`
pluginConfigFilev1beta3 := filepath.Join(tmpDir, "pluginv1beta3.yaml")
if err := ioutil.WriteFile(pluginConfigFilev1beta3, []byte(fmt.Sprintf(`
apiVersion: kubescheduler.config.k8s.io/v1beta3
kind: KubeSchedulerConfiguration
clientConnection:
kubeconfig: "%s"
profiles:
- plugins:
multiPoint:
enabled:
- name: DefaultBinder
- name: PrioritySort
- name: DefaultPreemption
- name: VolumeBinding
- name: NodeResourcesFit
- name: NodePorts
- name: InterPodAffinity
- name: TaintToleration
disabled:
- name: "*"
preFilter:
disabled:
- name: VolumeBinding
- name: InterPodAffinity
filter:
disabled:
- name: VolumeBinding
- name: InterPodAffinity
- name: TaintToleration
score:
disabled:
- name: VolumeBinding
- name: NodeResourcesFit
`, configKubeconfig)), os.FileMode(0600)); err != nil {
t.Fatal(err)
}
// plugin config
pluginConfigFilev1beta2 := filepath.Join(tmpDir, "pluginv1beta2.yaml")
if err := ioutil.WriteFile(pluginConfigFilev1beta2, []byte(fmt.Sprintf(`
apiVersion: kubescheduler.config.k8s.io/v1beta2
kind: KubeSchedulerConfiguration
clientConnection:
@ -188,20 +226,19 @@ leaderElection:
wantPlugins: map[string]*config.Plugins{
"default-scheduler": func() *config.Plugins {
plugins := &config.Plugins{
QueueSort: defaults.PluginsV1beta3.QueueSort,
PreFilter: defaults.PluginsV1beta3.PreFilter,
Filter: defaults.PluginsV1beta3.Filter,
PostFilter: defaults.PluginsV1beta3.PostFilter,
PreScore: defaults.PluginsV1beta3.PreScore,
Score: defaults.PluginsV1beta3.Score,
Bind: defaults.PluginsV1beta3.Bind,
PreBind: defaults.PluginsV1beta3.PreBind,
Reserve: defaults.PluginsV1beta3.Reserve,
QueueSort: defaults.ExpandedPluginsV1beta3.QueueSort,
PreFilter: defaults.ExpandedPluginsV1beta3.PreFilter,
Filter: defaults.ExpandedPluginsV1beta3.Filter,
PostFilter: defaults.ExpandedPluginsV1beta3.PostFilter,
PreScore: defaults.ExpandedPluginsV1beta3.PreScore,
Score: defaults.ExpandedPluginsV1beta3.Score,
Bind: defaults.ExpandedPluginsV1beta3.Bind,
PreBind: defaults.ExpandedPluginsV1beta3.PreBind,
Reserve: defaults.ExpandedPluginsV1beta3.Reserve,
}
plugins.PreScore.Enabled = append(plugins.PreScore.Enabled, config.Plugin{Name: names.SelectorSpread, Weight: 0})
plugins.Score.Enabled = append(
plugins.Score.Enabled,
config.Plugin{Name: names.VolumeBinding, Weight: 1},
config.Plugin{Name: names.SelectorSpread, Weight: 1},
)
return plugins
@ -218,13 +255,53 @@ leaderElection:
"--kubeconfig", configKubeconfig,
},
wantPlugins: map[string]*config.Plugins{
"default-scheduler": defaults.PluginsV1beta3,
"default-scheduler": defaults.ExpandedPluginsV1beta3,
},
},
{
name: "component configuration",
name: "component configuration v1beta2",
flags: []string{
"--config", pluginConfigFile,
"--config", pluginConfigFilev1beta2,
"--kubeconfig", configKubeconfig,
},
wantPlugins: map[string]*config.Plugins{
"default-scheduler": {
Bind: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultBinder"}}},
Filter: config.PluginSet{
Enabled: []config.Plugin{
{Name: "NodeResourcesFit"},
{Name: "NodePorts"},
},
},
PreFilter: config.PluginSet{
Enabled: []config.Plugin{
{Name: "NodeResourcesFit"},
{Name: "NodePorts"},
},
},
PostFilter: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultPreemption"}}},
PreScore: config.PluginSet{
Enabled: []config.Plugin{
{Name: "InterPodAffinity"},
{Name: "TaintToleration"},
},
},
QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: "PrioritySort"}}},
Score: config.PluginSet{
Enabled: []config.Plugin{
{Name: "InterPodAffinity", Weight: 1},
{Name: "TaintToleration", Weight: 1},
},
},
Reserve: config.PluginSet{Enabled: []config.Plugin{{Name: "VolumeBinding"}}},
PreBind: config.PluginSet{Enabled: []config.Plugin{{Name: "VolumeBinding"}}},
},
},
},
{
name: "component configuration v1beta3",
flags: []string{
"--config", pluginConfigFilev1beta3,
"--kubeconfig", configKubeconfig,
},
wantPlugins: map[string]*config.Plugins{

View File

@ -151,8 +151,36 @@ var PluginConfigsV1beta2 = []config.PluginConfig{
},
}
// PluginsV1beta3 default set of v1beta3 plugins.
// PluginsV1beta3 is the set of default v1beta3 plugins (before MultiPoint expansion)
var PluginsV1beta3 = &config.Plugins{
MultiPoint: config.PluginSet{
Enabled: []config.Plugin{
{Name: names.PrioritySort},
{Name: names.NodeUnschedulable},
{Name: names.NodeName},
{Name: names.TaintToleration, Weight: 3},
{Name: names.NodeAffinity, Weight: 2},
{Name: names.NodePorts},
{Name: names.NodeResourcesFit, Weight: 1},
{Name: names.VolumeRestrictions},
{Name: names.EBSLimits},
{Name: names.GCEPDLimits},
{Name: names.NodeVolumeLimits},
{Name: names.AzureDiskLimits},
{Name: names.VolumeBinding},
{Name: names.VolumeZone},
{Name: names.PodTopologySpread, Weight: 2},
{Name: names.InterPodAffinity, Weight: 2},
{Name: names.DefaultPreemption},
{Name: names.NodeResourcesBalancedAllocation, Weight: 1},
{Name: names.ImageLocality, Weight: 1},
{Name: names.DefaultBinder},
},
},
}
// ExpandedPluginsV1beta3 default set of v1beta3 plugins after MultiPoint expansion
var ExpandedPluginsV1beta3 = &config.Plugins{
QueueSort: config.PluginSet{
Enabled: []config.Plugin{
{Name: names.PrioritySort},
@ -160,13 +188,13 @@ var PluginsV1beta3 = &config.Plugins{
},
PreFilter: config.PluginSet{
Enabled: []config.Plugin{
{Name: names.NodeResourcesFit},
{Name: names.NodeAffinity},
{Name: names.NodePorts},
{Name: names.NodeResourcesFit},
{Name: names.VolumeRestrictions},
{Name: names.VolumeBinding},
{Name: names.PodTopologySpread},
{Name: names.InterPodAffinity},
{Name: names.VolumeBinding},
{Name: names.NodeAffinity},
},
},
Filter: config.PluginSet{
@ -195,32 +223,37 @@ var PluginsV1beta3 = &config.Plugins{
},
PreScore: config.PluginSet{
Enabled: []config.Plugin{
{Name: names.InterPodAffinity},
{Name: names.PodTopologySpread},
{Name: names.TaintToleration},
{Name: names.NodeAffinity},
{Name: names.PodTopologySpread},
{Name: names.InterPodAffinity},
},
},
Score: config.PluginSet{
Enabled: []config.Plugin{
{Name: names.NodeResourcesBalancedAllocation, Weight: 1},
{Name: names.ImageLocality, Weight: 1},
{Name: names.NodeResourcesFit, Weight: 1},
// Weight is doubled because:
// Weight is tripled because:
// - This is a score coming from user preference.
{Name: names.InterPodAffinity, Weight: 2},
// - Usage of node tainting to group nodes in the cluster is increasing becoming a use-case
// for many user workloads
{Name: names.TaintToleration, Weight: 3},
// Weight is doubled because:
// - This is a score coming from user preference.
{Name: names.NodeAffinity, Weight: 2},
// Weight is doubled because:
// - This is a score coming from user preference.
// - It makes its signal comparable to NodeResourcesLeastAllocated.
{Name: names.PodTopologySpread, Weight: 2},
{Name: names.NodeResourcesFit, Weight: 1},
// Weight is tripled because:
// - This is a score coming from user preference.
// - Usage of node tainting to group nodes in the cluster is increasing becoming a use-case
// for many user workloads
{Name: names.TaintToleration, Weight: 3},
{Name: names.VolumeBinding, Weight: 1},
// Weight is doubled because:
// - This is a score coming from user preference.
// - It makes its signal comparable to NodeResourcesLeastAllocated.
{Name: names.PodTopologySpread, Weight: 2},
// Weight is doubled because:
// - This is a score coming from user preference.
{Name: names.InterPodAffinity, Weight: 2},
{Name: names.NodeResourcesBalancedAllocation, Weight: 1},
{Name: names.ImageLocality, Weight: 1},
},
},
Reserve: config.PluginSet{

View File

@ -161,6 +161,9 @@ type Plugins struct {
// PostBind is a list of plugins that should be invoked after a pod is successfully bound.
PostBind PluginSet
// MultiPoint is a simplified config field for enabling plugins for all valid extension points
MultiPoint PluginSet
}
// PluginSet specifies enabled and disabled plugins for an extension point.

View File

@ -757,6 +757,9 @@ func autoConvert_v1beta2_Plugins_To_config_Plugins(in *v1beta2.Plugins, out *con
if err := Convert_v1beta2_PluginSet_To_config_PluginSet(&in.PostBind, &out.PostBind, s); err != nil {
return err
}
if err := Convert_v1beta2_PluginSet_To_config_PluginSet(&in.MultiPoint, &out.MultiPoint, s); err != nil {
return err
}
return nil
}
@ -799,6 +802,9 @@ func autoConvert_config_Plugins_To_v1beta2_Plugins(in *config.Plugins, out *v1be
if err := Convert_config_PluginSet_To_v1beta2_PluginSet(&in.PostBind, &out.PostBind, s); err != nil {
return err
}
if err := Convert_config_PluginSet_To_v1beta2_PluginSet(&in.MultiPoint, &out.MultiPoint, s); err != nil {
return err
}
return nil
}

View File

@ -29,30 +29,15 @@ import (
// getDefaultPlugins returns the default set of plugins.
func getDefaultPlugins() *v1beta3.Plugins {
plugins := &v1beta3.Plugins{
QueueSort: v1beta3.PluginSet{
MultiPoint: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.PrioritySort},
},
},
PreFilter: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.NodeResourcesFit},
{Name: names.NodePorts},
{Name: names.VolumeRestrictions},
{Name: names.PodTopologySpread},
{Name: names.InterPodAffinity},
{Name: names.VolumeBinding},
{Name: names.NodeAffinity},
},
},
Filter: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.NodeUnschedulable},
{Name: names.NodeName},
{Name: names.TaintToleration},
{Name: names.NodeAffinity},
{Name: names.TaintToleration, Weight: pointer.Int32(3)},
{Name: names.NodeAffinity, Weight: pointer.Int32(2)},
{Name: names.NodePorts},
{Name: names.NodeResourcesFit},
{Name: names.NodeResourcesFit, Weight: pointer.Int32(1)},
{Name: names.VolumeRestrictions},
{Name: names.EBSLimits},
{Name: names.GCEPDLimits},
@ -60,57 +45,11 @@ func getDefaultPlugins() *v1beta3.Plugins {
{Name: names.AzureDiskLimits},
{Name: names.VolumeBinding},
{Name: names.VolumeZone},
{Name: names.PodTopologySpread},
{Name: names.InterPodAffinity},
},
},
PostFilter: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.PodTopologySpread, Weight: pointer.Int32(2)},
{Name: names.InterPodAffinity, Weight: pointer.Int32(2)},
{Name: names.DefaultPreemption},
},
},
PreScore: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.InterPodAffinity},
{Name: names.PodTopologySpread},
{Name: names.TaintToleration},
{Name: names.NodeAffinity},
},
},
Score: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.NodeResourcesBalancedAllocation, Weight: pointer.Int32Ptr(1)},
{Name: names.ImageLocality, Weight: pointer.Int32Ptr(1)},
{Name: names.NodeResourcesFit, Weight: pointer.Int32Ptr(1)},
// Weight is doubled because:
// - This is a score coming from user preference.
{Name: names.InterPodAffinity, Weight: pointer.Int32Ptr(2)},
// Weight is doubled because:
// - This is a score coming from user preference.
{Name: names.NodeAffinity, Weight: pointer.Int32Ptr(2)},
// Weight is doubled because:
// - This is a score coming from user preference.
// - It makes its signal comparable to NodeResourcesFit.LeastAllocated.
{Name: names.PodTopologySpread, Weight: pointer.Int32Ptr(2)},
// Weight is tripled because:
// - This is a score coming from user preference.
// - Usage of node tainting to group nodes in the cluster is becoming a valid use-case more often
// for many user workloads
{Name: names.TaintToleration, Weight: pointer.Int32Ptr(3)},
},
},
Reserve: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.VolumeBinding},
},
},
PreBind: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.VolumeBinding},
},
},
Bind: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.NodeResourcesBalancedAllocation, Weight: pointer.Int32(1)},
{Name: names.ImageLocality, Weight: pointer.Int32(1)},
{Name: names.DefaultBinder},
},
},
@ -121,18 +60,12 @@ func getDefaultPlugins() *v1beta3.Plugins {
}
func applyFeatureGates(config *v1beta3.Plugins) {
if utilfeature.DefaultFeatureGate.Enabled(features.VolumeCapacityPriority) {
config.Score.Enabled = append(config.Score.Enabled, v1beta3.Plugin{Name: names.VolumeBinding, Weight: pointer.Int32Ptr(1)})
}
if !utilfeature.DefaultFeatureGate.Enabled(features.DefaultPodTopologySpread) {
// When feature is enabled, the default spreading is done by
// PodTopologySpread plugin, which is enabled by default.
klog.InfoS("Registering SelectorSpread plugin")
s := v1beta3.Plugin{Name: names.SelectorSpread}
config.PreScore.Enabled = append(config.PreScore.Enabled, s)
s.Weight = pointer.Int32Ptr(1)
config.Score.Enabled = append(config.Score.Enabled, s)
s := v1beta3.Plugin{Name: names.SelectorSpread, Weight: pointer.Int32Ptr(1)}
config.MultiPoint.Enabled = append(config.MultiPoint.Enabled, s)
}
}
@ -142,6 +75,7 @@ func mergePlugins(defaultPlugins, customPlugins *v1beta3.Plugins) *v1beta3.Plugi
return defaultPlugins
}
defaultPlugins.MultiPoint = mergePluginSet(defaultPlugins.MultiPoint, customPlugins.MultiPoint)
defaultPlugins.QueueSort = mergePluginSet(defaultPlugins.QueueSort, customPlugins.QueueSort)
defaultPlugins.PreFilter = mergePluginSet(defaultPlugins.PreFilter, customPlugins.PreFilter)
defaultPlugins.Filter = mergePluginSet(defaultPlugins.Filter, customPlugins.Filter)
@ -166,9 +100,22 @@ func mergePluginSet(defaultPluginSet, customPluginSet v1beta3.PluginSet) v1beta3
enabledCustomPlugins := make(map[string]pluginIndex)
// replacedPluginIndex is a set of index of plugins, which have replaced the default plugins.
replacedPluginIndex := sets.NewInt()
var disabled []v1beta3.Plugin
for _, disabledPlugin := range customPluginSet.Disabled {
// if the user is manually disabling any (or all, with "*") default plugins for an extension point,
// we need to track that so that the MultiPoint extension logic in the framework can know to skip
// inserting unspecified default plugins to this point.
disabled = append(disabled, v1beta3.Plugin{Name: disabledPlugin.Name})
disabledPlugins.Insert(disabledPlugin.Name)
}
// With MultiPoint, we may now have some disabledPlugins in the default registry
// For example, we enable PluginX with Filter+Score through MultiPoint but disable its Score plugin by default.
for _, disabledPlugin := range defaultPluginSet.Disabled {
disabled = append(disabled, v1beta3.Plugin{Name: disabledPlugin.Name})
disabledPlugins.Insert(disabledPlugin.Name)
}
for index, enabledPlugin := range customPluginSet.Enabled {
enabledCustomPlugins[enabledPlugin.Name] = pluginIndex{index, enabledPlugin}
}
@ -197,5 +144,5 @@ func mergePluginSet(defaultPluginSet, customPluginSet v1beta3.PluginSet) v1beta3
enabledPlugins = append(enabledPlugins, plugin)
}
}
return v1beta3.PluginSet{Enabled: enabledPlugins}
return v1beta3.PluginSet{Enabled: enabledPlugins, Disabled: disabled}
}

View File

@ -38,30 +38,15 @@ func TestApplyFeatureGates(t *testing.T) {
{
name: "Feature gates disabled",
wantConfig: &v1beta3.Plugins{
QueueSort: v1beta3.PluginSet{
MultiPoint: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.PrioritySort},
},
},
PreFilter: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.NodeResourcesFit},
{Name: names.NodePorts},
{Name: names.VolumeRestrictions},
{Name: names.PodTopologySpread},
{Name: names.InterPodAffinity},
{Name: names.VolumeBinding},
{Name: names.NodeAffinity},
},
},
Filter: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.NodeUnschedulable},
{Name: names.NodeName},
{Name: names.TaintToleration},
{Name: names.NodeAffinity},
{Name: names.TaintToleration, Weight: pointer.Int32(3)},
{Name: names.NodeAffinity, Weight: pointer.Int32(2)},
{Name: names.NodePorts},
{Name: names.NodeResourcesFit},
{Name: names.NodeResourcesFit, Weight: pointer.Int32(1)},
{Name: names.VolumeRestrictions},
{Name: names.EBSLimits},
{Name: names.GCEPDLimits},
@ -69,46 +54,11 @@ func TestApplyFeatureGates(t *testing.T) {
{Name: names.AzureDiskLimits},
{Name: names.VolumeBinding},
{Name: names.VolumeZone},
{Name: names.PodTopologySpread},
{Name: names.InterPodAffinity},
},
},
PostFilter: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.PodTopologySpread, Weight: pointer.Int32(2)},
{Name: names.InterPodAffinity, Weight: pointer.Int32(2)},
{Name: names.DefaultPreemption},
},
},
PreScore: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.InterPodAffinity},
{Name: names.PodTopologySpread},
{Name: names.TaintToleration},
{Name: names.NodeAffinity},
},
},
Score: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.NodeResourcesBalancedAllocation, Weight: pointer.Int32Ptr(1)},
{Name: names.ImageLocality, Weight: pointer.Int32Ptr(1)},
{Name: names.NodeResourcesFit, Weight: pointer.Int32Ptr(1)},
{Name: names.InterPodAffinity, Weight: pointer.Int32Ptr(2)},
{Name: names.NodeAffinity, Weight: pointer.Int32Ptr(2)},
{Name: names.PodTopologySpread, Weight: pointer.Int32Ptr(2)},
{Name: names.TaintToleration, Weight: pointer.Int32Ptr(3)},
},
},
Reserve: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.VolumeBinding},
},
},
PreBind: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.VolumeBinding},
},
},
Bind: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.NodeResourcesBalancedAllocation, Weight: pointer.Int32(1)},
{Name: names.ImageLocality, Weight: pointer.Int32(1)},
{Name: names.DefaultBinder},
},
},
@ -120,30 +70,15 @@ func TestApplyFeatureGates(t *testing.T) {
features.DefaultPodTopologySpread: false,
},
wantConfig: &v1beta3.Plugins{
QueueSort: v1beta3.PluginSet{
MultiPoint: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.PrioritySort},
},
},
PreFilter: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.NodeResourcesFit},
{Name: names.NodePorts},
{Name: names.VolumeRestrictions},
{Name: names.PodTopologySpread},
{Name: names.InterPodAffinity},
{Name: names.VolumeBinding},
{Name: names.NodeAffinity},
},
},
Filter: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.NodeUnschedulable},
{Name: names.NodeName},
{Name: names.TaintToleration},
{Name: names.NodeAffinity},
{Name: names.TaintToleration, Weight: pointer.Int32(3)},
{Name: names.NodeAffinity, Weight: pointer.Int32(2)},
{Name: names.NodePorts},
{Name: names.NodeResourcesFit},
{Name: names.NodeResourcesFit, Weight: pointer.Int32(1)},
{Name: names.VolumeRestrictions},
{Name: names.EBSLimits},
{Name: names.GCEPDLimits},
@ -151,49 +86,13 @@ func TestApplyFeatureGates(t *testing.T) {
{Name: names.AzureDiskLimits},
{Name: names.VolumeBinding},
{Name: names.VolumeZone},
{Name: names.PodTopologySpread},
{Name: names.InterPodAffinity},
},
},
PostFilter: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.PodTopologySpread, Weight: pointer.Int32(2)},
{Name: names.InterPodAffinity, Weight: pointer.Int32(2)},
{Name: names.DefaultPreemption},
},
},
PreScore: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.InterPodAffinity},
{Name: names.PodTopologySpread},
{Name: names.TaintToleration},
{Name: names.NodeAffinity},
{Name: names.SelectorSpread},
},
},
Score: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.NodeResourcesBalancedAllocation, Weight: pointer.Int32Ptr(1)},
{Name: names.ImageLocality, Weight: pointer.Int32Ptr(1)},
{Name: names.NodeResourcesFit, Weight: pointer.Int32Ptr(1)},
{Name: names.InterPodAffinity, Weight: pointer.Int32Ptr(2)},
{Name: names.NodeAffinity, Weight: pointer.Int32Ptr(2)},
{Name: names.PodTopologySpread, Weight: pointer.Int32Ptr(2)},
{Name: names.TaintToleration, Weight: pointer.Int32Ptr(3)},
{Name: names.SelectorSpread, Weight: pointer.Int32Ptr(1)},
},
},
Reserve: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.VolumeBinding},
},
},
PreBind: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.VolumeBinding},
},
},
Bind: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.NodeResourcesBalancedAllocation, Weight: pointer.Int32(1)},
{Name: names.ImageLocality, Weight: pointer.Int32(1)},
{Name: names.DefaultBinder},
{Name: names.SelectorSpread, Weight: pointer.Int32(1)},
},
},
},
@ -276,6 +175,9 @@ func TestMergePlugins(t *testing.T) {
{Name: "CustomPlugin"},
{Name: "DefaultPlugin2"},
},
Disabled: []v1beta3.Plugin{
{Name: "DefaultPlugin2"},
},
},
},
},
@ -308,6 +210,9 @@ func TestMergePlugins(t *testing.T) {
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
},
Disabled: []v1beta3.Plugin{
{Name: "*"},
},
},
},
},
@ -338,6 +243,9 @@ func TestMergePlugins(t *testing.T) {
{Name: "DefaultPlugin2"},
{Name: "DefaultPlugin1"},
},
Disabled: []v1beta3.Plugin{
{Name: "*"},
},
},
},
},
@ -451,12 +359,197 @@ func TestMergePlugins(t *testing.T) {
},
},
},
{
name: "Append custom MultiPoint plugin",
customPlugins: &v1beta3.Plugins{
MultiPoint: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: "CustomPlugin"},
},
},
},
defaultPlugins: &v1beta3.Plugins{
MultiPoint: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
},
},
},
expectedPlugins: &v1beta3.Plugins{
MultiPoint: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
{Name: "CustomPlugin"},
},
},
},
},
{
name: "Append disabled Multipoint plugins",
customPlugins: &v1beta3.Plugins{
MultiPoint: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: "CustomPlugin"},
{Name: "CustomPlugin2"},
},
Disabled: []v1beta3.Plugin{
{Name: "DefaultPlugin2"},
},
},
Score: v1beta3.PluginSet{
Disabled: []v1beta3.Plugin{
{Name: "CustomPlugin2"},
},
},
},
defaultPlugins: &v1beta3.Plugins{
MultiPoint: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: "DefaultPlugin1"},
{Name: "DefaultPlugin2"},
},
},
},
expectedPlugins: &v1beta3.Plugins{
MultiPoint: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: "DefaultPlugin1"},
{Name: "CustomPlugin"},
{Name: "CustomPlugin2"},
},
Disabled: []v1beta3.Plugin{
{Name: "DefaultPlugin2"},
},
},
Score: v1beta3.PluginSet{
Disabled: []v1beta3.Plugin{
{Name: "CustomPlugin2"},
},
},
},
},
{
name: "override default MultiPoint plugins with custom value",
customPlugins: &v1beta3.Plugins{
MultiPoint: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: "DefaultPlugin", Weight: pointer.Int32(5)},
},
},
},
defaultPlugins: &v1beta3.Plugins{
MultiPoint: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: "DefaultPlugin"},
},
},
},
expectedPlugins: &v1beta3.Plugins{
MultiPoint: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: "DefaultPlugin", Weight: pointer.Int32(5)},
},
},
},
},
{
name: "disabled MultiPoint plugin in default set",
defaultPlugins: &v1beta3.Plugins{
MultiPoint: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: "DefaultPlugin"},
},
Disabled: []v1beta3.Plugin{
{Name: "DefaultPlugin2"},
},
},
},
customPlugins: &v1beta3.Plugins{
MultiPoint: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: "CustomPlugin"},
},
},
},
expectedPlugins: &v1beta3.Plugins{
MultiPoint: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: "DefaultPlugin"},
{Name: "CustomPlugin"},
},
Disabled: []v1beta3.Plugin{
{Name: "DefaultPlugin2"},
},
},
},
},
{
name: "disabled MultiPoint plugin in default set for specific extension point",
defaultPlugins: &v1beta3.Plugins{
MultiPoint: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: "DefaultPlugin"},
},
},
Score: v1beta3.PluginSet{
Disabled: []v1beta3.Plugin{
{Name: "DefaultPlugin2"},
},
},
},
customPlugins: &v1beta3.Plugins{
MultiPoint: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: "CustomPlugin"},
},
},
},
expectedPlugins: &v1beta3.Plugins{
MultiPoint: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: "DefaultPlugin"},
{Name: "CustomPlugin"},
},
},
Score: v1beta3.PluginSet{
Disabled: []v1beta3.Plugin{
{Name: "DefaultPlugin2"},
},
},
},
},
{
name: "multipoint with only disabled gets merged",
defaultPlugins: &v1beta3.Plugins{
MultiPoint: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: "DefaultPlugin"},
},
},
},
customPlugins: &v1beta3.Plugins{
MultiPoint: v1beta3.PluginSet{
Disabled: []v1beta3.Plugin{
{Name: "DefaultPlugin"},
},
},
},
expectedPlugins: &v1beta3.Plugins{
MultiPoint: v1beta3.PluginSet{
Disabled: []v1beta3.Plugin{
{Name: "DefaultPlugin"},
},
},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
test.defaultPlugins = mergePlugins(test.defaultPlugins, test.customPlugins)
if d := cmp.Diff(test.expectedPlugins, test.defaultPlugins); d != "" {
gotPlugins := mergePlugins(test.defaultPlugins, test.customPlugins)
if d := cmp.Diff(test.expectedPlugins, gotPlugins); d != "" {
t.Fatalf("plugins mismatch (-want +got):\n%s", d)
}
})

View File

@ -42,6 +42,7 @@ func pluginsNames(p *v1beta3.Plugins) []string {
return nil
}
extensions := []v1beta3.PluginSet{
p.MultiPoint,
p.PreFilter,
p.Filter,
p.PostFilter,
@ -66,7 +67,6 @@ func pluginsNames(p *v1beta3.Plugins) []string {
func setDefaults_KubeSchedulerProfile(prof *v1beta3.KubeSchedulerProfile) {
// Set default plugins.
prof.Plugins = mergePlugins(getDefaultPlugins(), prof.Plugins)
// Set default plugin configs.
scheme := GetPluginArgConversionScheme()
existingConfigs := sets.NewString()

View File

@ -325,30 +325,15 @@ func TestSchedulerDefaults(t *testing.T) {
{
SchedulerName: pointer.StringPtr("custom-scheduler"),
Plugins: &v1beta3.Plugins{
QueueSort: v1beta3.PluginSet{
MultiPoint: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.PrioritySort},
},
},
PreFilter: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.NodeResourcesFit},
{Name: names.NodePorts},
{Name: names.VolumeRestrictions},
{Name: names.PodTopologySpread},
{Name: names.InterPodAffinity},
{Name: names.VolumeBinding},
{Name: names.NodeAffinity},
},
},
Filter: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.NodeUnschedulable},
{Name: names.NodeName},
{Name: names.TaintToleration},
{Name: names.NodeAffinity},
{Name: names.TaintToleration, Weight: pointer.Int32(3)},
{Name: names.NodeAffinity, Weight: pointer.Int32(2)},
{Name: names.NodePorts},
{Name: names.NodeResourcesFit},
{Name: names.NodeResourcesFit, Weight: pointer.Int32(1)},
{Name: names.VolumeRestrictions},
{Name: names.EBSLimits},
{Name: names.GCEPDLimits},
@ -356,48 +341,21 @@ func TestSchedulerDefaults(t *testing.T) {
{Name: names.AzureDiskLimits},
{Name: names.VolumeBinding},
{Name: names.VolumeZone},
{Name: names.PodTopologySpread},
{Name: names.InterPodAffinity},
},
},
PostFilter: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.PodTopologySpread, Weight: pointer.Int32(2)},
{Name: names.InterPodAffinity, Weight: pointer.Int32(2)},
{Name: names.DefaultPreemption},
},
},
PreScore: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.InterPodAffinity},
{Name: names.PodTopologySpread},
{Name: names.TaintToleration},
{Name: names.NodeAffinity},
},
},
Score: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.NodeResourcesBalancedAllocation, Weight: pointer.Int32Ptr(1)},
{Name: names.ImageLocality, Weight: pointer.Int32Ptr(1)},
{Name: names.NodeResourcesFit, Weight: pointer.Int32Ptr(1)},
{Name: names.InterPodAffinity, Weight: pointer.Int32Ptr(2)},
{Name: names.NodeAffinity, Weight: pointer.Int32Ptr(2)},
{Name: names.PodTopologySpread, Weight: pointer.Int32Ptr(2)},
{Name: names.TaintToleration, Weight: pointer.Int32Ptr(3)},
},
},
Reserve: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.VolumeBinding},
},
},
PreBind: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: names.VolumeBinding},
{Name: names.NodeResourcesBalancedAllocation, Weight: pointer.Int32(1)},
{Name: names.ImageLocality, Weight: pointer.Int32(1)},
{Name: names.DefaultBinder},
},
},
Bind: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{
{Name: "BarPlugin"},
},
Disabled: []v1beta3.Plugin{
{Name: names.DefaultBinder},
},
},
},
PluginConfig: pluginConfigs,

View File

@ -747,6 +747,9 @@ func autoConvert_v1beta3_Plugins_To_config_Plugins(in *v1beta3.Plugins, out *con
if err := Convert_v1beta3_PluginSet_To_config_PluginSet(&in.PostBind, &out.PostBind, s); err != nil {
return err
}
if err := Convert_v1beta3_PluginSet_To_config_PluginSet(&in.MultiPoint, &out.MultiPoint, s); err != nil {
return err
}
return nil
}
@ -789,6 +792,9 @@ func autoConvert_config_Plugins_To_v1beta3_Plugins(in *config.Plugins, out *v1be
if err := Convert_config_PluginSet_To_v1beta3_PluginSet(&in.PostBind, &out.PostBind, s); err != nil {
return err
}
if err := Convert_config_PluginSet_To_v1beta3_PluginSet(&in.MultiPoint, &out.MultiPoint, s); err != nil {
return err
}
return nil
}

View File

@ -395,6 +395,7 @@ func (in *Plugins) DeepCopyInto(out *Plugins) {
in.PreBind.DeepCopyInto(&out.PreBind)
in.Bind.DeepCopyInto(&out.Bind)
in.PostBind.DeepCopyInto(&out.PostBind)
in.MultiPoint.DeepCopyInto(&out.MultiPoint)
return
}

View File

@ -289,22 +289,6 @@ func NewFramework(r Registry, profile *config.KubeSchedulerProfile, opts ...Opti
return f, nil
}
var totalPriority int64
for _, e := range profile.Plugins.Score.Enabled {
// a weight of zero is not permitted, plugins can be disabled explicitly
// when configured.
f.scorePluginWeight[e.Name] = int(e.Weight)
if f.scorePluginWeight[e.Name] == 0 {
f.scorePluginWeight[e.Name] = 1
}
// Checks totalPriority against MaxTotalScore to avoid overflow
if int64(f.scorePluginWeight[e.Name])*framework.MaxNodeScore > framework.MaxTotalScore-totalPriority {
return nil, fmt.Errorf("total score of Score plugins could overflow")
}
totalPriority += int64(f.scorePluginWeight[e.Name]) * framework.MaxNodeScore
}
// get needed plugins from config
pg := f.pluginsNeeded(profile.Plugins)
@ -346,12 +330,28 @@ func NewFramework(r Registry, profile *config.KubeSchedulerProfile, opts ...Opti
fillEventToPluginMap(p, options.clusterEventMap)
}
// initialize plugins per individual extension points
for _, e := range f.getExtensionPoints(profile.Plugins) {
if err := updatePluginList(e.slicePtr, *e.plugins, pluginsMap); err != nil {
return nil, err
}
}
// initialize multiPoint plugins to their expanded extension points
if len(profile.Plugins.MultiPoint.Enabled) > 0 {
if err := f.expandMultiPointPlugins(profile, pluginsMap); err != nil {
return nil, err
}
}
if len(f.queueSortPlugins) != 1 {
return nil, fmt.Errorf("one queue sort plugin required for profile with scheduler name %q", profile.SchedulerName)
}
if err := getScoreWeights(f, pluginsMap, append(profile.Plugins.Score.Enabled, profile.Plugins.MultiPoint.Enabled...)); err != nil {
return nil, err
}
// Verifying the score weights again since Plugin.Name() could return a different
// value from the one used in the configuration.
for _, scorePlugin := range f.scorePlugins {
@ -384,6 +384,106 @@ func NewFramework(r Registry, profile *config.KubeSchedulerProfile, opts ...Opti
return f, nil
}
// getScoreWeights makes sure that, between MultiPoint-Score plugin weights and individual Score
// plugin weights there is not an overflow of MaxTotalScore.
func getScoreWeights(f *frameworkImpl, pluginsMap map[string]framework.Plugin, plugins []config.Plugin) error {
var totalPriority int64
scorePlugins := reflect.ValueOf(&f.scorePlugins).Elem()
pluginType := scorePlugins.Type().Elem()
for _, e := range plugins {
pg := pluginsMap[e.Name]
if !reflect.TypeOf(pg).Implements(pluginType) {
continue
}
// We append MultiPoint plugins to the list of Score plugins. So if this plugin has already been
// encountered, let the individual Score weight take precedence.
if _, ok := f.scorePluginWeight[e.Name]; ok {
continue
}
// a weight of zero is not permitted, plugins can be disabled explicitly
// when configured.
f.scorePluginWeight[e.Name] = int(e.Weight)
if f.scorePluginWeight[e.Name] == 0 {
f.scorePluginWeight[e.Name] = 1
}
// Checks totalPriority against MaxTotalScore to avoid overflow
if int64(f.scorePluginWeight[e.Name])*framework.MaxNodeScore > framework.MaxTotalScore-totalPriority {
return fmt.Errorf("total score of Score plugins could overflow")
}
totalPriority += int64(f.scorePluginWeight[e.Name]) * framework.MaxNodeScore
}
return nil
}
func (f *frameworkImpl) expandMultiPointPlugins(profile *config.KubeSchedulerProfile, pluginsMap map[string]framework.Plugin) error {
// initialize MultiPoint plugins
for _, e := range f.getExtensionPoints(profile.Plugins) {
plugins := reflect.ValueOf(e.slicePtr).Elem()
pluginType := plugins.Type().Elem()
// build enabledSet of plugins already registered via normal extension points
// to check double registration
enabledSet := sets.NewString()
for _, plugin := range e.plugins.Enabled {
enabledSet.Insert(plugin.Name)
}
disabledSet := sets.NewString()
for _, disabledPlugin := range e.plugins.Disabled {
disabledSet.Insert(disabledPlugin.Name)
}
if disabledSet.Has("*") {
klog.V(4).InfoS("all plugins disabled for extension point, skipping MultiPoint expansion", "extension", pluginType)
continue
}
// track plugins enabled via multipoint separately from those enabled by specific extensions,
// so that we can distinguish between double-registration and explicit overrides
multiPointEnabled := sets.NewString()
for _, ep := range profile.Plugins.MultiPoint.Enabled {
pg, ok := pluginsMap[ep.Name]
if !ok {
return fmt.Errorf("%s %q does not exist", pluginType.Name(), ep.Name)
}
// if this plugin doesn't implement the type for the current extension we're trying to expand, skip
if !reflect.TypeOf(pg).Implements(pluginType) {
continue
}
// a plugin that's enabled via MultiPoint can still be disabled for specific extension points
if disabledSet.Has(ep.Name) {
klog.V(4).InfoS("plugin disabled for extension point", "plugin", ep.Name, "extension", pluginType)
continue
}
// if this plugin has already been enabled by the specific extension point,
// the user intent is to override the default plugin or make some other explicit setting.
// Either way, discard the MultiPoint value for this plugin.
// This maintains expected behavior for overriding default plugins (see https://github.com/kubernetes/kubernetes/pull/99582)
if enabledSet.Has(ep.Name) {
klog.InfoS("MultiPoint plugin is explicitly re-configured; overriding", "plugin", ep.Name)
continue
}
// if this plugin is already registered via MultiPoint, then this is
// a double registration and an error in the config.
if multiPointEnabled.Has(ep.Name) {
return fmt.Errorf("plugin %q already registered as %q", ep.Name, pluginType.Name())
}
// we only need to update the multipoint set, since we already have the specific extension set from above
multiPointEnabled.Insert(ep.Name)
newPlugins := reflect.Append(plugins, reflect.ValueOf(pg))
plugins.Set(newPlugins)
}
}
return nil
}
func fillEventToPluginMap(p framework.Plugin, eventToPlugins map[framework.ClusterEvent]sets.String) {
ext, ok := p.(framework.EnqueueExtensions)
if !ok {
@ -1174,6 +1274,9 @@ func (f *frameworkImpl) pluginsNeeded(plugins *config.Plugins) map[string]config
for _, e := range f.getExtensionPoints(plugins) {
find(e.plugins)
}
// Parse MultiPoint separately since they are not returned by f.getExtensionPoints()
find(&plugins.MultiPoint)
return pgMap
}

View File

@ -119,6 +119,10 @@ func (pl *TestScorePlugin) Name() string {
return pl.name
}
func (pl *TestScorePlugin) PreScore(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodes []*v1.Node) *framework.Status {
return framework.NewStatus(framework.Code(pl.inj.PreScoreStatus), "injected status")
}
func (pl *TestScorePlugin) Score(ctx context.Context, state *framework.CycleState, p *v1.Pod, nodeName string) (int64, *framework.Status) {
return setScoreRes(pl.inj)
}
@ -490,6 +494,281 @@ func TestNewFrameworkErrors(t *testing.T) {
}
}
func TestNewFrameworkMultiPointExpansion(t *testing.T) {
tests := []struct {
name string
plugins *config.Plugins
wantPlugins *config.Plugins
wantErr string
}{
{
name: "plugin expansion",
plugins: &config.Plugins{
MultiPoint: config.PluginSet{
Enabled: []config.Plugin{
{Name: testPlugin, Weight: 5},
},
},
},
wantPlugins: &config.Plugins{
QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
PreFilter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
Filter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
PostFilter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
PreScore: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
Score: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin, Weight: 5}}},
Reserve: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
Permit: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
PreBind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
Bind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
PostBind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
},
},
{
name: "disable MultiPoint plugin at some extension points",
plugins: &config.Plugins{
MultiPoint: config.PluginSet{
Enabled: []config.Plugin{
{Name: testPlugin},
},
},
PreScore: config.PluginSet{
Disabled: []config.Plugin{
{Name: testPlugin},
},
},
Score: config.PluginSet{
Disabled: []config.Plugin{
{Name: testPlugin},
},
},
},
wantPlugins: &config.Plugins{
QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
PreFilter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
Filter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
PostFilter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
Reserve: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
Permit: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
PreBind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
Bind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
PostBind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
},
},
{
name: "Multiple MultiPoint plugins",
plugins: &config.Plugins{
MultiPoint: config.PluginSet{
Enabled: []config.Plugin{
{Name: testPlugin},
{Name: scorePlugin1},
},
},
},
wantPlugins: &config.Plugins{
QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
PreFilter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
Filter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
PostFilter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
PreScore: config.PluginSet{Enabled: []config.Plugin{
{Name: testPlugin},
{Name: scorePlugin1},
}},
Score: config.PluginSet{Enabled: []config.Plugin{
{Name: testPlugin, Weight: 1},
{Name: scorePlugin1, Weight: 1},
}},
Reserve: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
Permit: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
PreBind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
Bind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
PostBind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
},
},
{
name: "disable MultiPoint extension",
plugins: &config.Plugins{
MultiPoint: config.PluginSet{
Enabled: []config.Plugin{
{Name: testPlugin},
{Name: scorePlugin1},
},
},
PreScore: config.PluginSet{
Disabled: []config.Plugin{
{Name: "*"},
},
},
},
wantPlugins: &config.Plugins{
QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
PreFilter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
Filter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
PostFilter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
Score: config.PluginSet{Enabled: []config.Plugin{
{Name: testPlugin, Weight: 1},
{Name: scorePlugin1, Weight: 1},
}},
Reserve: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
Permit: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
PreBind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
Bind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
PostBind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
},
},
{
name: "Reorder MultiPoint plugins (specified extension takes precedence)",
plugins: &config.Plugins{
MultiPoint: config.PluginSet{
Enabled: []config.Plugin{
{Name: scoreWithNormalizePlugin1},
{Name: testPlugin},
{Name: scorePlugin1},
},
},
Score: config.PluginSet{
Enabled: []config.Plugin{
{Name: scorePlugin1},
{Name: testPlugin},
},
},
},
wantPlugins: &config.Plugins{
QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
PreFilter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
Filter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
PostFilter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
PreScore: config.PluginSet{Enabled: []config.Plugin{
{Name: testPlugin},
{Name: scorePlugin1},
}},
Score: config.PluginSet{Enabled: []config.Plugin{
{Name: scorePlugin1, Weight: 1},
{Name: testPlugin, Weight: 1},
{Name: scoreWithNormalizePlugin1, Weight: 1},
}},
Reserve: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
Permit: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
PreBind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
Bind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
PostBind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
},
},
{
name: "Override MultiPoint plugins weights",
plugins: &config.Plugins{
MultiPoint: config.PluginSet{
Enabled: []config.Plugin{
{Name: testPlugin},
{Name: scorePlugin1},
},
},
Score: config.PluginSet{
Enabled: []config.Plugin{
{Name: scorePlugin1, Weight: 5},
{Name: testPlugin, Weight: 3},
},
},
},
wantPlugins: &config.Plugins{
QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
PreFilter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
Filter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
PostFilter: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
PreScore: config.PluginSet{Enabled: []config.Plugin{
{Name: testPlugin},
{Name: scorePlugin1},
}},
Score: config.PluginSet{Enabled: []config.Plugin{
{Name: scorePlugin1, Weight: 5},
{Name: testPlugin, Weight: 3},
}},
Reserve: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
Permit: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
PreBind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
Bind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
PostBind: config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin}}},
},
},
{
name: "disable and enable MultiPoint plugins with '*'",
plugins: &config.Plugins{
MultiPoint: config.PluginSet{
Enabled: []config.Plugin{
{Name: queueSortPlugin},
{Name: bindPlugin},
{Name: scorePlugin1},
},
Disabled: []config.Plugin{
{Name: "*"},
},
},
},
wantPlugins: &config.Plugins{
QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: queueSortPlugin}}},
PreScore: config.PluginSet{Enabled: []config.Plugin{
{Name: scorePlugin1},
}},
Score: config.PluginSet{Enabled: []config.Plugin{
{Name: scorePlugin1, Weight: 1},
}},
Bind: config.PluginSet{Enabled: []config.Plugin{{Name: bindPlugin}}},
},
},
{
name: "disable and enable MultiPoint plugin by name",
plugins: &config.Plugins{
MultiPoint: config.PluginSet{
Enabled: []config.Plugin{
{Name: bindPlugin},
{Name: queueSortPlugin},
{Name: scorePlugin1},
},
Disabled: []config.Plugin{
{Name: scorePlugin1},
},
},
},
wantPlugins: &config.Plugins{
QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: queueSortPlugin}}},
PreScore: config.PluginSet{Enabled: []config.Plugin{
{Name: scorePlugin1},
}},
Score: config.PluginSet{Enabled: []config.Plugin{
{Name: scorePlugin1, Weight: 1},
}},
Bind: config.PluginSet{Enabled: []config.Plugin{{Name: bindPlugin}}},
},
},
{
name: "Expect 'already registered' error",
plugins: &config.Plugins{
MultiPoint: config.PluginSet{
Enabled: []config.Plugin{
{Name: testPlugin},
{Name: testPlugin},
},
},
},
wantErr: "already registered",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
fw, err := NewFramework(registry, &config.KubeSchedulerProfile{Plugins: tc.plugins})
if (err != nil && tc.wantErr == "") || (err == nil && tc.wantErr != "") || (err != nil && !strings.Contains(err.Error(), tc.wantErr)) {
t.Errorf("Unexpected error, got %v, expect: %s", err, tc.wantErr)
}
if tc.wantErr == "" {
if diff := cmp.Diff(tc.wantPlugins, fw.ListPlugins()); diff != "" {
t.Errorf("Unexpected eventToPlugin map (-want,+got):%s", diff)
}
}
})
}
}
// fakeNoopPlugin doesn't implement interface framework.EnqueueExtensions.
type fakeNoopPlugin struct{}

View File

@ -20,7 +20,6 @@ package profile
import (
"errors"
"fmt"
"github.com/google/go-cmp/cmp"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes/scheme"
@ -55,13 +54,13 @@ func NewMap(cfgs []config.KubeSchedulerProfile, r frameworkruntime.Registry, rec
v := cfgValidator{m: m}
for _, cfg := range cfgs {
if err := v.validate(cfg); err != nil {
return nil, err
}
p, err := newProfile(cfg, r, recorderFact, opts...)
if err != nil {
return nil, fmt.Errorf("creating profile for scheduler name %s: %v", cfg.SchedulerName, err)
}
if err := v.validate(cfg, p); err != nil {
return nil, err
}
m[cfg.SchedulerName] = p
}
return m, nil
@ -86,20 +85,18 @@ type cfgValidator struct {
queueSortArgs runtime.Object
}
func (v *cfgValidator) validate(cfg config.KubeSchedulerProfile) error {
if len(cfg.SchedulerName) == 0 {
func (v *cfgValidator) validate(cfg config.KubeSchedulerProfile, f framework.Framework) error {
if len(f.ProfileName()) == 0 {
return errors.New("scheduler name is needed")
}
if cfg.Plugins == nil {
return fmt.Errorf("plugins required for profile with scheduler name %q", cfg.SchedulerName)
return fmt.Errorf("plugins required for profile with scheduler name %q", f.ProfileName())
}
if v.m[cfg.SchedulerName] != nil {
return fmt.Errorf("duplicate profile with scheduler name %q", cfg.SchedulerName)
if v.m[f.ProfileName()] != nil {
return fmt.Errorf("duplicate profile with scheduler name %q", f.ProfileName())
}
if len(cfg.Plugins.QueueSort.Enabled) != 1 {
return fmt.Errorf("one queue sort plugin required for profile with scheduler name %q", cfg.SchedulerName)
}
queueSort := cfg.Plugins.QueueSort.Enabled[0].Name
queueSort := f.ListPlugins().QueueSort.Enabled[0].Name
var queueSortArgs runtime.Object
for _, plCfg := range cfg.PluginConfig {
if plCfg.Name == queueSort {

View File

@ -31,10 +31,10 @@ import (
)
var fakeRegistry = frameworkruntime.Registry{
"QueueSort": newFakePlugin,
"Bind1": newFakePlugin,
"Bind2": newFakePlugin,
"Another": newFakePlugin,
"QueueSort": newFakePlugin("QueueSort"),
"Bind1": newFakePlugin("Bind1"),
"Bind2": newFakePlugin("Bind2"),
"Another": newFakePlugin("Another"),
}
func TestNewMap(t *testing.T) {
@ -260,10 +260,12 @@ func TestNewMap(t *testing.T) {
}
}
type fakePlugin struct{}
type fakePlugin struct {
name string
}
func (p *fakePlugin) Name() string {
return ""
return p.name
}
func (p *fakePlugin) Less(*framework.QueuedPodInfo, *framework.QueuedPodInfo) bool {
@ -274,8 +276,10 @@ func (p *fakePlugin) Bind(context.Context, *framework.CycleState, *v1.Pod, strin
return nil
}
func newFakePlugin(_ runtime.Object, _ framework.Handle) (framework.Plugin, error) {
return &fakePlugin{}, nil
func newFakePlugin(name string) func(object runtime.Object, handle framework.Handle) (framework.Plugin, error) {
return func(_ runtime.Object, _ framework.Handle) (framework.Plugin, error) {
return &fakePlugin{name: name}, nil
}
}
func nilRecorderFactory(_ string) events.EventRecorder {

View File

@ -191,6 +191,9 @@ type Plugins struct {
// PostBind is a list of plugins that should be invoked after a pod is successfully bound.
PostBind PluginSet `json:"postBind,omitempty"`
// MultiPoint is a simplified config section to enable plugins for all valid extension points.
MultiPoint PluginSet `json:"multiPoint,omitempty"`
}
// PluginSet specifies enabled and disabled plugins for an extension point.

View File

@ -452,6 +452,7 @@ func (in *Plugins) DeepCopyInto(out *Plugins) {
in.PreBind.DeepCopyInto(&out.PreBind)
in.Bind.DeepCopyInto(&out.Bind)
in.PostBind.DeepCopyInto(&out.PostBind)
in.MultiPoint.DeepCopyInto(&out.MultiPoint)
return
}

View File

@ -184,6 +184,24 @@ type Plugins struct {
// PostBind is a list of plugins that should be invoked after a pod is successfully bound.
PostBind PluginSet `json:"postBind,omitempty"`
// MultiPoint is a simplified config section to enable plugins for all valid extension points.
// Plugins enabled through MultiPoint will automatically register for every individual extension
// point the plugin has implemented. Disabling a plugin through MultiPoint disables that behavior.
// The same is true for disabling "*" through MultiPoint (no default plugins will be automatically registered).
// Plugins can still be disabled through their individual extension points.
//
// In terms of precedence, plugin config follows this basic hierarchy
// 1. Specific extension points
// 2. Explicitly configured MultiPoint plugins
// 3. The set of default plugins, as MultiPoint plugins
// This implies that a higher precedence plugin will run first and overwrite any settings within MultiPoint.
// Explicitly user-configured plugins also take a higher precedence over default plugins.
// Within this hierarchy, an Enabled setting takes precedence over Disabled. For example, if a plugin is
// set in both `multiPoint.Enabled` and `multiPoint.Disabled`, the plugin will be enabled. Similarly,
// including `multiPoint.Disabled = '*'` and `multiPoint.Enabled = pluginA` will still register that specific
// plugin through MultiPoint. This follows the same behavior as all other extension point configurations.
MultiPoint PluginSet `json:"multiPoint,omitempty"`
}
// PluginSet specifies enabled and disabled plugins for an extension point.

View File

@ -442,6 +442,7 @@ func (in *Plugins) DeepCopyInto(out *Plugins) {
in.PreBind.DeepCopyInto(&out.PreBind)
in.Bind.DeepCopyInto(&out.Bind)
in.PostBind.DeepCopyInto(&out.PostBind)
in.MultiPoint.DeepCopyInto(&out.MultiPoint)
return
}

View File

@ -1259,6 +1259,11 @@ func TestBindPlugin(t *testing.T) {
Profiles: []v1beta3.KubeSchedulerProfile{{
SchedulerName: pointer.StringPtr(v1.DefaultSchedulerName),
Plugins: &v1beta3.Plugins{
MultiPoint: v1beta3.PluginSet{
Disabled: []v1beta3.Plugin{
{Name: defaultbinder.Name},
},
},
Reserve: v1beta3.PluginSet{
Enabled: []v1beta3.Plugin{{Name: reservePlugin.Name()}},
},