diff --git a/pkg/scheduler/factory.go b/pkg/scheduler/factory.go index 249a0ee980e..65e9fcf5de6 100644 --- a/pkg/scheduler/factory.go +++ b/pkg/scheduler/factory.go @@ -20,7 +20,6 @@ import ( "context" "errors" "fmt" - "sort" "time" "github.com/google/go-cmp/cmp" @@ -268,138 +267,56 @@ func (c *Configurator) createFromConfig(policy schedulerapi.Policy) (*Scheduler, klog.V(2).Infof("Creating scheduler with fit predicates '%v' and priority functions '%v'", predicateKeys, priorityKeys) - pluginsForPredicates, pluginConfigForPredicates, err := getPredicateConfigs(predicateKeys, lr, args) - if err != nil { - return nil, err - } - - pluginsForPriorities, pluginConfigForPriorities, err := getPriorityConfigs(priorityKeys, lr, args) - if err != nil { - return nil, err - } // Combine all framework configurations. If this results in any duplication, framework // instantiation should fail. - var defPlugins schedulerapi.Plugins + // "PrioritySort" and "DefaultBinder" were neither predicates nor priorities // before. We add them by default. - defPlugins.Append(&schedulerapi.Plugins{ + plugins := schedulerapi.Plugins{ QueueSort: &schedulerapi.PluginSet{ Enabled: []schedulerapi.Plugin{{Name: queuesort.Name}}, }, Bind: &schedulerapi.PluginSet{ Enabled: []schedulerapi.Plugin{{Name: defaultbinder.Name}}, }, - }) - defPlugins.Append(pluginsForPredicates) - defPlugins.Append(pluginsForPriorities) - defPluginConfig, err := mergePluginConfigsFromPolicy(pluginConfigForPredicates, pluginConfigForPriorities) - if err != nil { + } + var pluginConfig []schedulerapi.PluginConfig + var err error + if plugins, pluginConfig, err = lr.AppendPredicateConfigs(predicateKeys, args, plugins, pluginConfig); err != nil { + return nil, err + } + if plugins, pluginConfig, err = lr.AppendPriorityConfigs(priorityKeys, args, plugins, pluginConfig); err != nil { + return nil, err + } + if pluginConfig, err = dedupPluginConfigs(pluginConfig); err != nil { return nil, err } for i := range c.profiles { prof := &c.profiles[i] - // Plugins are empty when using Policy. + // Plugins and PluginConfig are empty when using Policy; overriding. prof.Plugins = &schedulerapi.Plugins{} - prof.Plugins.Append(&defPlugins) - - // PluginConfig is ignored when using Policy. - prof.PluginConfig = defPluginConfig + prof.Plugins.Append(&plugins) + prof.PluginConfig = pluginConfig } return c.create() } -// mergePluginConfigsFromPolicy merges the giving plugin configs ensuring that, +// dedupPluginConfigs removes duplicates from pluginConfig, ensuring that, // if a plugin name is repeated, the arguments are the same. -func mergePluginConfigsFromPolicy(pc1, pc2 []schedulerapi.PluginConfig) ([]schedulerapi.PluginConfig, error) { +func dedupPluginConfigs(pc []schedulerapi.PluginConfig) ([]schedulerapi.PluginConfig, error) { args := make(map[string]runtime.Object) - for _, c := range pc1 { - args[c.Name] = c.Args - } - for _, c := range pc2 { - if v, ok := args[c.Name]; ok && !cmp.Equal(v, c.Args) { + result := make([]schedulerapi.PluginConfig, 0, len(pc)) + for _, c := range pc { + if v, found := args[c.Name]; !found { + result = append(result, c) + args[c.Name] = c.Args + } else if !cmp.Equal(v, c.Args) { // This should be unreachable. return nil, fmt.Errorf("inconsistent configuration produced for plugin %s", c.Name) } - args[c.Name] = c.Args } - pc := make([]schedulerapi.PluginConfig, 0, len(args)) - for k, v := range args { - pc = append(pc, schedulerapi.PluginConfig{ - Name: k, - Args: v, - }) - } - return pc, nil -} - -// getPriorityConfigs returns priorities configuration: ones that will run as priorities and ones that will run -// as framework plugins. Specifically, a priority will run as a framework plugin if a plugin config producer was -// registered for that priority. -func getPriorityConfigs(keys map[string]int64, lr *frameworkplugins.LegacyRegistry, args *frameworkplugins.ConfigProducerArgs) (*schedulerapi.Plugins, []schedulerapi.PluginConfig, error) { - var plugins schedulerapi.Plugins - var pluginConfig []schedulerapi.PluginConfig - - // Sort the keys so that it is easier for unit tests to do compare. - var sortedKeys []string - for k := range keys { - sortedKeys = append(sortedKeys, k) - } - sort.Strings(sortedKeys) - - for _, priority := range sortedKeys { - weight := keys[priority] - producer, exist := lr.PriorityToConfigProducer[priority] - if !exist { - return nil, nil, fmt.Errorf("no config producer registered for %q", priority) - } - a := *args - a.Weight = int32(weight) - pl, plc := producer(a) - plugins.Append(&pl) - pluginConfig = append(pluginConfig, plc...) - } - return &plugins, pluginConfig, nil -} - -// getPredicateConfigs returns predicates configuration: ones that will run as fitPredicates and ones that will run -// as framework plugins. Specifically, a predicate will run as a framework plugin if a plugin config producer was -// registered for that predicate. -// Note that the framework executes plugins according to their order in the Plugins list, and so predicates run as plugins -// are added to the Plugins list according to the order specified in predicates.Ordering(). -func getPredicateConfigs(keys sets.String, lr *frameworkplugins.LegacyRegistry, args *frameworkplugins.ConfigProducerArgs) (*schedulerapi.Plugins, []schedulerapi.PluginConfig, error) { - allPredicates := keys.Union(lr.MandatoryPredicates) - - // Create the framework plugin configurations, and place them in the order - // that the corresponding predicates were supposed to run. - var plugins schedulerapi.Plugins - var pluginConfig []schedulerapi.PluginConfig - - for _, predicateKey := range frameworkplugins.PredicateOrdering() { - if allPredicates.Has(predicateKey) { - producer, exist := lr.PredicateToConfigProducer[predicateKey] - if !exist { - return nil, nil, fmt.Errorf("no framework config producer registered for %q", predicateKey) - } - pl, plc := producer(*args) - plugins.Append(&pl) - pluginConfig = append(pluginConfig, plc...) - allPredicates.Delete(predicateKey) - } - } - - // Third, add the rest in no specific order. - for predicateKey := range allPredicates { - producer, exist := lr.PredicateToConfigProducer[predicateKey] - if !exist { - return nil, nil, fmt.Errorf("no framework config producer registered for %q", predicateKey) - } - pl, plc := producer(*args) - plugins.Append(&pl) - pluginConfig = append(pluginConfig, plc...) - } - - return &plugins, pluginConfig, nil + return result, nil } // MakeDefaultErrorFunc construct a function to handle pod scheduler error diff --git a/pkg/scheduler/framework/plugins/BUILD b/pkg/scheduler/framework/plugins/BUILD index 91620cf86fc..8327c03cfd6 100644 --- a/pkg/scheduler/framework/plugins/BUILD +++ b/pkg/scheduler/framework/plugins/BUILD @@ -80,6 +80,9 @@ go_test( embed = [":go_default_library"], deps = [ "//pkg/scheduler/apis/config:go_default_library", + "//pkg/scheduler/framework/plugins/nodeunschedulable:go_default_library", + "//pkg/scheduler/framework/plugins/tainttoleration:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/github.com/google/go-cmp/cmp:go_default_library", ], ) diff --git a/pkg/scheduler/framework/plugins/legacy_registry.go b/pkg/scheduler/framework/plugins/legacy_registry.go index dc2ca2f8684..ce753c2d6b3 100644 --- a/pkg/scheduler/framework/plugins/legacy_registry.go +++ b/pkg/scheduler/framework/plugins/legacy_registry.go @@ -17,8 +17,10 @@ limitations under the License. package plugins import ( - "k8s.io/apimachinery/pkg/util/sets" + "fmt" + "sort" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/imagelocality" @@ -127,24 +129,24 @@ const ( EvenPodsSpreadPred = "EvenPodsSpread" ) -// PredicateOrdering returns the ordering of predicate execution. -func PredicateOrdering() []string { - return []string{CheckNodeUnschedulablePred, - GeneralPred, HostNamePred, PodFitsHostPortsPred, - MatchNodeSelectorPred, PodFitsResourcesPred, NoDiskConflictPred, - PodToleratesNodeTaintsPred, CheckNodeLabelPresencePred, - CheckServiceAffinityPred, MaxEBSVolumeCountPred, MaxGCEPDVolumeCountPred, MaxCSIVolumeCountPred, - MaxAzureDiskVolumeCountPred, MaxCinderVolumeCountPred, CheckVolumeBindingPred, NoVolumeZoneConflictPred, - EvenPodsSpreadPred, MatchInterPodAffinityPred} +// predicateOrdering is the ordering of predicate execution. +var predicateOrdering = []string{ + CheckNodeUnschedulablePred, + GeneralPred, HostNamePred, PodFitsHostPortsPred, + MatchNodeSelectorPred, PodFitsResourcesPred, NoDiskConflictPred, + PodToleratesNodeTaintsPred, CheckNodeLabelPresencePred, + CheckServiceAffinityPred, MaxEBSVolumeCountPred, MaxGCEPDVolumeCountPred, MaxCSIVolumeCountPred, + MaxAzureDiskVolumeCountPred, MaxCinderVolumeCountPred, CheckVolumeBindingPred, NoVolumeZoneConflictPred, + EvenPodsSpreadPred, MatchInterPodAffinityPred, } // LegacyRegistry is used to store current state of registered predicates and priorities. type LegacyRegistry struct { // maps that associate predicates/priorities with framework plugin configurations. - PredicateToConfigProducer map[string]ConfigProducer - PriorityToConfigProducer map[string]ConfigProducer + predicateToConfigProducer map[string]configProducer + priorityToConfigProducer map[string]configProducer // predicates that will always be configured. - MandatoryPredicates sets.String + mandatoryPredicates sets.String // predicates and priorities that will be used if either was set to nil in a // given v1.Policy configuration. DefaultPredicates sets.String @@ -169,16 +171,16 @@ type ConfigProducerArgs struct { InterPodAffinityArgs *config.InterPodAffinityArgs } -// ConfigProducer returns the set of plugins and their configuration for a +// configProducer appends the set of plugins and their configuration for a // predicate/priority given the args. -type ConfigProducer func(args ConfigProducerArgs) (config.Plugins, []config.PluginConfig) +type configProducer func(ConfigProducerArgs, *config.Plugins, *[]config.PluginConfig) // NewLegacyRegistry returns a legacy algorithm registry of predicates and priorities. func NewLegacyRegistry() *LegacyRegistry { registry := &LegacyRegistry{ - // MandatoryPredicates the set of keys for predicates that the scheduler will + // mandatoryPredicates the set of keys for predicates that the scheduler will // be configured with all the time. - MandatoryPredicates: sets.NewString( + mandatoryPredicates: sets.NewString( PodToleratesNodeTaintsPred, CheckNodeUnschedulablePred, ), @@ -212,248 +214,275 @@ func NewLegacyRegistry() *LegacyRegistry { EvenPodsSpreadPriority: 2, }, - PredicateToConfigProducer: make(map[string]ConfigProducer), - PriorityToConfigProducer: make(map[string]ConfigProducer), + predicateToConfigProducer: make(map[string]configProducer), + priorityToConfigProducer: make(map[string]configProducer), } registry.registerPredicateConfigProducer(GeneralPred, - func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(args ConfigProducerArgs, plugins *config.Plugins, pluginConfig *[]config.PluginConfig) { // GeneralPredicate is a combination of predicates. plugins.Filter = appendToPluginSet(plugins.Filter, noderesources.FitName, nil) plugins.PreFilter = appendToPluginSet(plugins.PreFilter, noderesources.FitName, nil) if args.NodeResourcesFitArgs != nil { - pluginConfig = append(pluginConfig, + *pluginConfig = append(*pluginConfig, config.PluginConfig{Name: noderesources.FitName, Args: args.NodeResourcesFitArgs}) } plugins.Filter = appendToPluginSet(plugins.Filter, nodename.Name, nil) plugins.Filter = appendToPluginSet(plugins.Filter, nodeports.Name, nil) plugins.PreFilter = appendToPluginSet(plugins.PreFilter, nodeports.Name, nil) plugins.Filter = appendToPluginSet(plugins.Filter, nodeaffinity.Name, nil) - return }) registry.registerPredicateConfigProducer(PodToleratesNodeTaintsPred, - func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, tainttoleration.Name, nil) - return }) registry.registerPredicateConfigProducer(PodFitsResourcesPred, - func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(args ConfigProducerArgs, plugins *config.Plugins, pluginConfig *[]config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, noderesources.FitName, nil) plugins.PreFilter = appendToPluginSet(plugins.PreFilter, noderesources.FitName, nil) if args.NodeResourcesFitArgs != nil { - pluginConfig = append(pluginConfig, + *pluginConfig = append(*pluginConfig, config.PluginConfig{Name: noderesources.FitName, Args: args.NodeResourcesFitArgs}) } - return }) registry.registerPredicateConfigProducer(HostNamePred, - func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, nodename.Name, nil) - return }) registry.registerPredicateConfigProducer(PodFitsHostPortsPred, - func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, nodeports.Name, nil) plugins.PreFilter = appendToPluginSet(plugins.PreFilter, nodeports.Name, nil) - return }) registry.registerPredicateConfigProducer(MatchNodeSelectorPred, - func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, nodeaffinity.Name, nil) - return }) registry.registerPredicateConfigProducer(CheckNodeUnschedulablePred, - func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, nodeunschedulable.Name, nil) - return }) registry.registerPredicateConfigProducer(CheckVolumeBindingPred, - func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) { plugins.PreFilter = appendToPluginSet(plugins.PreFilter, volumebinding.Name, nil) plugins.Filter = appendToPluginSet(plugins.Filter, volumebinding.Name, nil) plugins.Reserve = appendToPluginSet(plugins.Reserve, volumebinding.Name, nil) plugins.PreBind = appendToPluginSet(plugins.PreBind, volumebinding.Name, nil) - return }) registry.registerPredicateConfigProducer(NoDiskConflictPred, - func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, volumerestrictions.Name, nil) - return }) registry.registerPredicateConfigProducer(NoVolumeZoneConflictPred, - func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, volumezone.Name, nil) - return }) registry.registerPredicateConfigProducer(MaxCSIVolumeCountPred, - func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, nodevolumelimits.CSIName, nil) - return }) registry.registerPredicateConfigProducer(MaxEBSVolumeCountPred, - func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, nodevolumelimits.EBSName, nil) - return }) registry.registerPredicateConfigProducer(MaxGCEPDVolumeCountPred, - func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, nodevolumelimits.GCEPDName, nil) - return }) registry.registerPredicateConfigProducer(MaxAzureDiskVolumeCountPred, - func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, nodevolumelimits.AzureDiskName, nil) - return }) registry.registerPredicateConfigProducer(MaxCinderVolumeCountPred, - func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, nodevolumelimits.CinderName, nil) - return }) registry.registerPredicateConfigProducer(MatchInterPodAffinityPred, - func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, interpodaffinity.Name, nil) plugins.PreFilter = appendToPluginSet(plugins.PreFilter, interpodaffinity.Name, nil) - return }) registry.registerPredicateConfigProducer(CheckNodeLabelPresencePred, - func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(args ConfigProducerArgs, plugins *config.Plugins, pluginConfig *[]config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, nodelabel.Name, nil) if args.NodeLabelArgs != nil { - pluginConfig = append(pluginConfig, + *pluginConfig = append(*pluginConfig, config.PluginConfig{Name: nodelabel.Name, Args: args.NodeLabelArgs}) } - return }) registry.registerPredicateConfigProducer(CheckServiceAffinityPred, - func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(args ConfigProducerArgs, plugins *config.Plugins, pluginConfig *[]config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, serviceaffinity.Name, nil) if args.ServiceAffinityArgs != nil { - pluginConfig = append(pluginConfig, + *pluginConfig = append(*pluginConfig, config.PluginConfig{Name: serviceaffinity.Name, Args: args.ServiceAffinityArgs}) } plugins.PreFilter = appendToPluginSet(plugins.PreFilter, serviceaffinity.Name, nil) - return }) registry.registerPredicateConfigProducer(EvenPodsSpreadPred, - func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) { plugins.PreFilter = appendToPluginSet(plugins.PreFilter, podtopologyspread.Name, nil) plugins.Filter = appendToPluginSet(plugins.Filter, podtopologyspread.Name, nil) - return }) // Register Priorities. registry.registerPriorityConfigProducer(SelectorSpreadPriority, - func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(args ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) { plugins.Score = appendToPluginSet(plugins.Score, selectorspread.Name, &args.Weight) plugins.PreScore = appendToPluginSet(plugins.PreScore, selectorspread.Name, nil) - return }) registry.registerPriorityConfigProducer(TaintTolerationPriority, - func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(args ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) { plugins.PreScore = appendToPluginSet(plugins.PreScore, tainttoleration.Name, nil) plugins.Score = appendToPluginSet(plugins.Score, tainttoleration.Name, &args.Weight) - return }) registry.registerPriorityConfigProducer(NodeAffinityPriority, - func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(args ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) { plugins.Score = appendToPluginSet(plugins.Score, nodeaffinity.Name, &args.Weight) - return }) registry.registerPriorityConfigProducer(ImageLocalityPriority, - func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(args ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) { plugins.Score = appendToPluginSet(plugins.Score, imagelocality.Name, &args.Weight) - return }) registry.registerPriorityConfigProducer(InterPodAffinityPriority, - func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(args ConfigProducerArgs, plugins *config.Plugins, pluginConfig *[]config.PluginConfig) { plugins.PreScore = appendToPluginSet(plugins.PreScore, interpodaffinity.Name, nil) plugins.Score = appendToPluginSet(plugins.Score, interpodaffinity.Name, &args.Weight) if args.InterPodAffinityArgs != nil { - pluginConfig = append(pluginConfig, + *pluginConfig = append(*pluginConfig, config.PluginConfig{Name: interpodaffinity.Name, Args: args.InterPodAffinityArgs}) } - return }) registry.registerPriorityConfigProducer(NodePreferAvoidPodsPriority, - func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(args ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) { plugins.Score = appendToPluginSet(plugins.Score, nodepreferavoidpods.Name, &args.Weight) - return }) registry.registerPriorityConfigProducer(MostRequestedPriority, - func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(args ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) { plugins.Score = appendToPluginSet(plugins.Score, noderesources.MostAllocatedName, &args.Weight) - return }) registry.registerPriorityConfigProducer(BalancedResourceAllocation, - func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(args ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) { plugins.Score = appendToPluginSet(plugins.Score, noderesources.BalancedAllocationName, &args.Weight) - return }) registry.registerPriorityConfigProducer(LeastRequestedPriority, - func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(args ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) { plugins.Score = appendToPluginSet(plugins.Score, noderesources.LeastAllocatedName, &args.Weight) - return }) registry.registerPriorityConfigProducer(noderesources.RequestedToCapacityRatioName, - func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(args ConfigProducerArgs, plugins *config.Plugins, pluginConfig *[]config.PluginConfig) { plugins.Score = appendToPluginSet(plugins.Score, noderesources.RequestedToCapacityRatioName, &args.Weight) if args.RequestedToCapacityRatioArgs != nil { - pluginConfig = append(pluginConfig, + *pluginConfig = append(*pluginConfig, config.PluginConfig{Name: noderesources.RequestedToCapacityRatioName, Args: args.RequestedToCapacityRatioArgs}) } - return }) registry.registerPriorityConfigProducer(nodelabel.Name, - func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(args ConfigProducerArgs, plugins *config.Plugins, pluginConfig *[]config.PluginConfig) { // If there are n LabelPreference priorities in the policy, the weight for the corresponding // score plugin is n*weight (note that the validation logic verifies that all LabelPreference // priorities specified in Policy have the same weight). weight := args.Weight * int32(len(args.NodeLabelArgs.PresentLabelsPreference)+len(args.NodeLabelArgs.AbsentLabelsPreference)) plugins.Score = appendToPluginSet(plugins.Score, nodelabel.Name, &weight) if args.NodeLabelArgs != nil { - pluginConfig = append(pluginConfig, + *pluginConfig = append(*pluginConfig, config.PluginConfig{Name: nodelabel.Name, Args: args.NodeLabelArgs}) } - return }) registry.registerPriorityConfigProducer(serviceaffinity.Name, - func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(args ConfigProducerArgs, plugins *config.Plugins, pluginConfig *[]config.PluginConfig) { // If there are n ServiceAffinity priorities in the policy, the weight for the corresponding // score plugin is n*weight (note that the validation logic verifies that all ServiceAffinity // priorities specified in Policy have the same weight). weight := args.Weight * int32(len(args.ServiceAffinityArgs.AntiAffinityLabelsPreference)) plugins.Score = appendToPluginSet(plugins.Score, serviceaffinity.Name, &weight) if args.ServiceAffinityArgs != nil { - pluginConfig = append(pluginConfig, + *pluginConfig = append(*pluginConfig, config.PluginConfig{Name: serviceaffinity.Name, Args: args.ServiceAffinityArgs}) } - return }) registry.registerPriorityConfigProducer(EvenPodsSpreadPriority, - func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(args ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) { plugins.PreScore = appendToPluginSet(plugins.PreScore, podtopologyspread.Name, nil) plugins.Score = appendToPluginSet(plugins.Score, podtopologyspread.Name, &args.Weight) - return }) return registry } +// AppendPredicateConfigs returns predicates configuration that will run as framework plugins. +// Note that the framework executes plugins according to their order in the Plugins list, and so predicates run as plugins +// are added to the Plugins list according to the order specified in predicateOrdering. +func (lr *LegacyRegistry) AppendPredicateConfigs(keys sets.String, args *ConfigProducerArgs, plugins config.Plugins, pluginConfig []config.PluginConfig) (config.Plugins, []config.PluginConfig, error) { + allPredicates := keys.Union(lr.mandatoryPredicates) + + // Create the framework plugin configurations, and place them in the order + // that the corresponding predicates were supposed to run. + for _, predicateKey := range predicateOrdering { + if allPredicates.Has(predicateKey) { + producer, exist := lr.predicateToConfigProducer[predicateKey] + if !exist { + return config.Plugins{}, nil, fmt.Errorf("no framework config producer registered for %q", predicateKey) + } + producer(*args, &plugins, &pluginConfig) + allPredicates.Delete(predicateKey) + } + } + + // Sort the keys so that it is easier for unit tests to do compare. + sortedKeys := make([]string, 0, len(allPredicates)) + for k := range allPredicates { + sortedKeys = append(sortedKeys, k) + } + sort.Strings(sortedKeys) + + for _, predicateKey := range sortedKeys { + producer, exist := lr.predicateToConfigProducer[predicateKey] + if !exist { + return config.Plugins{}, nil, fmt.Errorf("no framework config producer registered for %q", predicateKey) + } + producer(*args, &plugins, &pluginConfig) + } + + return plugins, pluginConfig, nil +} + +// AppendPriorityConfigs returns priorities configuration that will run as framework plugins. +func (lr *LegacyRegistry) AppendPriorityConfigs(keys map[string]int64, args *ConfigProducerArgs, plugins config.Plugins, pluginConfig []config.PluginConfig) (config.Plugins, []config.PluginConfig, error) { + // Sort the keys so that it is easier for unit tests to do compare. + sortedKeys := make([]string, 0, len(keys)) + for k := range keys { + sortedKeys = append(sortedKeys, k) + } + sort.Strings(sortedKeys) + + for _, priority := range sortedKeys { + weight := keys[priority] + producer, exist := lr.priorityToConfigProducer[priority] + if !exist { + return config.Plugins{}, nil, fmt.Errorf("no config producer registered for %q", priority) + } + a := *args + a.Weight = int32(weight) + producer(a, &plugins, &pluginConfig) + } + return plugins, pluginConfig, nil +} + // registers a config producer for a predicate. -func (lr *LegacyRegistry) registerPredicateConfigProducer(name string, producer ConfigProducer) { - if _, exist := lr.PredicateToConfigProducer[name]; exist { +func (lr *LegacyRegistry) registerPredicateConfigProducer(name string, producer configProducer) { + if _, exist := lr.predicateToConfigProducer[name]; exist { klog.Fatalf("already registered %q", name) } - lr.PredicateToConfigProducer[name] = producer + lr.predicateToConfigProducer[name] = producer } // registers a framework config producer for a priority. -func (lr *LegacyRegistry) registerPriorityConfigProducer(name string, producer ConfigProducer) { - if _, exist := lr.PriorityToConfigProducer[name]; exist { +func (lr *LegacyRegistry) registerPriorityConfigProducer(name string, producer configProducer) { + if _, exist := lr.priorityToConfigProducer[name]; exist { klog.Fatalf("already registered %q", name) } - lr.PriorityToConfigProducer[name] = producer + lr.priorityToConfigProducer[name] = producer } func appendToPluginSet(set *config.PluginSet, name string, weight *int32) *config.PluginSet { @@ -479,7 +508,7 @@ func (lr *LegacyRegistry) ProcessPredicatePolicy(policy config.PredicatePolicy, predicateName = PodFitsHostPortsPred } - if _, ok := lr.PredicateToConfigProducer[predicateName]; ok { + if _, ok := lr.predicateToConfigProducer[predicateName]; ok { // checking to see if a pre-defined predicate is requested klog.V(2).Infof("Predicate type %s already registered, reusing.", policy.Name) return predicateName @@ -535,7 +564,7 @@ func (lr *LegacyRegistry) ProcessPriorityPolicy(policy config.PriorityPolicy, co priorityName = SelectorSpreadPriority } - if _, ok := lr.PriorityToConfigProducer[priorityName]; ok { + if _, ok := lr.priorityToConfigProducer[priorityName]; ok { klog.V(2).Infof("Priority type %s already registered, reusing.", priorityName) return priorityName } diff --git a/pkg/scheduler/framework/plugins/legacy_registry_test.go b/pkg/scheduler/framework/plugins/legacy_registry_test.go index 3d2fb47a818..306d00210b3 100644 --- a/pkg/scheduler/framework/plugins/legacy_registry_test.go +++ b/pkg/scheduler/framework/plugins/legacy_registry_test.go @@ -17,104 +17,76 @@ limitations under the License. package plugins import ( - "fmt" "testing" "github.com/google/go-cmp/cmp" - + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kubernetes/pkg/scheduler/apis/config" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeunschedulable" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/tainttoleration" ) -func produceConfig(keys []string, producersMap map[string]ConfigProducer, args ConfigProducerArgs) (*config.Plugins, []config.PluginConfig, error) { - var plugins config.Plugins - var pluginConfig []config.PluginConfig - for _, k := range keys { - p, exist := producersMap[k] - if !exist { - return nil, nil, fmt.Errorf("finding key %q", k) - } - pl, plc := p(args) - plugins.Append(&pl) - pluginConfig = append(pluginConfig, plc...) - } - return &plugins, pluginConfig, nil -} - func TestRegisterConfigProducers(t *testing.T) { registry := NewLegacyRegistry() testPredicateName1 := "testPredicate1" testFilterName1 := "testFilter1" registry.registerPredicateConfigProducer(testPredicateName1, - func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, testFilterName1, nil) - return }) testPredicateName2 := "testPredicate2" testFilterName2 := "testFilter2" registry.registerPredicateConfigProducer(testPredicateName2, - func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, testFilterName2, nil) - return }) testPriorityName1 := "testPriority1" testScoreName1 := "testScore1" registry.registerPriorityConfigProducer(testPriorityName1, - func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(args ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) { plugins.Score = appendToPluginSet(plugins.Score, testScoreName1, &args.Weight) - return }) testPriorityName2 := "testPriority2" testScoreName2 := "testScore2" registry.registerPriorityConfigProducer(testPriorityName2, - func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + func(args ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) { plugins.Score = appendToPluginSet(plugins.Score, testScoreName2, &args.Weight) - return }) args := ConfigProducerArgs{Weight: 1} - predicatePlugins, _, err := produceConfig( - []string{testPredicateName1, testPredicateName2}, registry.PredicateToConfigProducer, args) - if err != nil { - t.Fatalf("producing predicate framework configs: %v.", err) - } - - priorityPlugins, _, err := produceConfig( - []string{testPriorityName1, testPriorityName2}, registry.PriorityToConfigProducer, args) - if err != nil { - t.Fatalf("producing predicate framework configs: %v.", err) - } - - // Verify that predicates and priorities are in the map and produce the expected score configurations. var gotPlugins config.Plugins - gotPlugins.Append(predicatePlugins) - gotPlugins.Append(priorityPlugins) + gotPlugins, _, err := registry.AppendPredicateConfigs(sets.NewString(testPredicateName1, testPredicateName2), &args, gotPlugins, nil) + if err != nil { + t.Fatalf("producing predicate framework configs: %v.", err) + } + + priorities := map[string]int64{ + testPriorityName1: 1, + testPriorityName2: 1, + } + gotPlugins, _, err = registry.AppendPriorityConfigs(priorities, &args, gotPlugins, nil) + if err != nil { + t.Fatalf("producing priority framework configs: %v.", err) + } - // Verify the aggregated configuration. wantPlugins := config.Plugins{ - QueueSort: &config.PluginSet{}, - PreFilter: &config.PluginSet{}, Filter: &config.PluginSet{ Enabled: []config.Plugin{ + {Name: nodeunschedulable.Name}, + {Name: tainttoleration.Name}, {Name: testFilterName1}, {Name: testFilterName2}, }, }, - PostFilter: &config.PluginSet{}, - PreScore: &config.PluginSet{}, Score: &config.PluginSet{ Enabled: []config.Plugin{ {Name: testScoreName1, Weight: 1}, {Name: testScoreName2, Weight: 1}, }, }, - Reserve: &config.PluginSet{}, - Permit: &config.PluginSet{}, - PreBind: &config.PluginSet{}, - Bind: &config.PluginSet{}, - PostBind: &config.PluginSet{}, } if diff := cmp.Diff(wantPlugins, gotPlugins); diff != "" {