diff --git a/pkg/scheduler/algorithmprovider/registry_test.go b/pkg/scheduler/algorithmprovider/registry_test.go index 6b3daac1aa8..46be6b3d5a3 100644 --- a/pkg/scheduler/algorithmprovider/registry_test.go +++ b/pkg/scheduler/algorithmprovider/registry_test.go @@ -109,71 +109,142 @@ func TestClusterAutoscalerProvider(t *testing.T) { func TestApplyFeatureGates(t *testing.T) { hardPodAffinityWeight := int64(1) - wantConfig := &Config{ - FrameworkPlugins: &schedulerapi.Plugins{ - PreFilter: &schedulerapi.PluginSet{ - Enabled: []schedulerapi.Plugin{ - {Name: noderesources.FitName}, - {Name: nodeports.Name}, - {Name: interpodaffinity.Name}, - {Name: podtopologyspread.Name}, + tests := []struct { + name string + featuresEnabled bool + wantConfig *Config + }{ + { + name: "Feature gates disabled", + featuresEnabled: false, + wantConfig: &Config{ + FrameworkPlugins: &schedulerapi.Plugins{ + PreFilter: &schedulerapi.PluginSet{ + Enabled: []schedulerapi.Plugin{ + {Name: noderesources.FitName}, + {Name: nodeports.Name}, + {Name: interpodaffinity.Name}, + }, + }, + Filter: &schedulerapi.PluginSet{ + Enabled: []schedulerapi.Plugin{ + {Name: nodeunschedulable.Name}, + {Name: noderesources.FitName}, + {Name: nodename.Name}, + {Name: nodeports.Name}, + {Name: nodeaffinity.Name}, + {Name: volumerestrictions.Name}, + {Name: tainttoleration.Name}, + {Name: nodevolumelimits.EBSName}, + {Name: nodevolumelimits.GCEPDName}, + {Name: nodevolumelimits.CSIName}, + {Name: nodevolumelimits.AzureDiskName}, + {Name: volumebinding.Name}, + {Name: volumezone.Name}, + {Name: interpodaffinity.Name}, + }, + }, + PostFilter: &schedulerapi.PluginSet{ + Enabled: []schedulerapi.Plugin{ + {Name: interpodaffinity.Name}, + {Name: tainttoleration.Name}, + }, + }, + Score: &schedulerapi.PluginSet{ + Enabled: []schedulerapi.Plugin{ + {Name: noderesources.BalancedAllocationName, Weight: 1}, + {Name: imagelocality.Name, Weight: 1}, + {Name: interpodaffinity.Name, Weight: 1}, + {Name: noderesources.LeastAllocatedName, Weight: 1}, + {Name: nodeaffinity.Name, Weight: 1}, + {Name: nodepreferavoidpods.Name, Weight: 10000}, + {Name: defaultpodtopologyspread.Name, Weight: 1}, + {Name: tainttoleration.Name, Weight: 1}, + }, + }, }, - }, - Filter: &schedulerapi.PluginSet{ - Enabled: []schedulerapi.Plugin{ - {Name: nodeunschedulable.Name}, - {Name: noderesources.FitName}, - {Name: nodename.Name}, - {Name: nodeports.Name}, - {Name: nodeaffinity.Name}, - {Name: volumerestrictions.Name}, - {Name: tainttoleration.Name}, - {Name: nodevolumelimits.EBSName}, - {Name: nodevolumelimits.GCEPDName}, - {Name: nodevolumelimits.CSIName}, - {Name: nodevolumelimits.AzureDiskName}, - {Name: volumebinding.Name}, - {Name: volumezone.Name}, - {Name: interpodaffinity.Name}, - {Name: podtopologyspread.Name}, - }, - }, - PostFilter: &schedulerapi.PluginSet{ - Enabled: []schedulerapi.Plugin{ - {Name: interpodaffinity.Name}, - {Name: tainttoleration.Name}, - {Name: podtopologyspread.Name}, - }, - }, - Score: &schedulerapi.PluginSet{ - Enabled: []schedulerapi.Plugin{ - {Name: noderesources.BalancedAllocationName, Weight: 1}, - {Name: imagelocality.Name, Weight: 1}, - {Name: interpodaffinity.Name, Weight: 1}, - {Name: noderesources.LeastAllocatedName, Weight: 1}, - {Name: nodeaffinity.Name, Weight: 1}, - {Name: nodepreferavoidpods.Name, Weight: 10000}, - {Name: defaultpodtopologyspread.Name, Weight: 1}, - {Name: tainttoleration.Name, Weight: 1}, - {Name: podtopologyspread.Name, Weight: 1}, - {Name: noderesources.ResourceLimitsName, Weight: 1}, + FrameworkPluginConfig: []schedulerapi.PluginConfig{ + { + Name: interpodaffinity.Name, + Args: runtime.Unknown{Raw: []byte(fmt.Sprintf(`{"hardPodAffinityWeight":%d}`, hardPodAffinityWeight))}, + }, }, }, }, - FrameworkPluginConfig: []schedulerapi.PluginConfig{ - { - Name: interpodaffinity.Name, - Args: runtime.Unknown{Raw: []byte(fmt.Sprintf(`{"hardPodAffinityWeight":%d}`, hardPodAffinityWeight))}, + { + name: "Feature gates enabled", + featuresEnabled: true, + wantConfig: &Config{ + FrameworkPlugins: &schedulerapi.Plugins{ + PreFilter: &schedulerapi.PluginSet{ + Enabled: []schedulerapi.Plugin{ + {Name: noderesources.FitName}, + {Name: nodeports.Name}, + {Name: interpodaffinity.Name}, + {Name: podtopologyspread.Name}, + }, + }, + Filter: &schedulerapi.PluginSet{ + Enabled: []schedulerapi.Plugin{ + {Name: nodeunschedulable.Name}, + {Name: noderesources.FitName}, + {Name: nodename.Name}, + {Name: nodeports.Name}, + {Name: nodeaffinity.Name}, + {Name: volumerestrictions.Name}, + {Name: tainttoleration.Name}, + {Name: nodevolumelimits.EBSName}, + {Name: nodevolumelimits.GCEPDName}, + {Name: nodevolumelimits.CSIName}, + {Name: nodevolumelimits.AzureDiskName}, + {Name: volumebinding.Name}, + {Name: volumezone.Name}, + {Name: interpodaffinity.Name}, + {Name: podtopologyspread.Name}, + }, + }, + PostFilter: &schedulerapi.PluginSet{ + Enabled: []schedulerapi.Plugin{ + {Name: interpodaffinity.Name}, + {Name: tainttoleration.Name}, + {Name: podtopologyspread.Name}, + }, + }, + Score: &schedulerapi.PluginSet{ + Enabled: []schedulerapi.Plugin{ + {Name: noderesources.BalancedAllocationName, Weight: 1}, + {Name: imagelocality.Name, Weight: 1}, + {Name: interpodaffinity.Name, Weight: 1}, + {Name: noderesources.LeastAllocatedName, Weight: 1}, + {Name: nodeaffinity.Name, Weight: 1}, + {Name: nodepreferavoidpods.Name, Weight: 10000}, + {Name: defaultpodtopologyspread.Name, Weight: 1}, + {Name: tainttoleration.Name, Weight: 1}, + {Name: podtopologyspread.Name, Weight: 1}, + {Name: noderesources.ResourceLimitsName, Weight: 1}, + }, + }, + }, + FrameworkPluginConfig: []schedulerapi.PluginConfig{ + { + Name: interpodaffinity.Name, + Args: runtime.Unknown{Raw: []byte(fmt.Sprintf(`{"hardPodAffinityWeight":%d}`, hardPodAffinityWeight))}, + }, + }, }, }, } - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ResourceLimitsPriorityFunction, true)() - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.EvenPodsSpread, true)() + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ResourceLimitsPriorityFunction, test.featuresEnabled)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.EvenPodsSpread, test.featuresEnabled)() - r := NewRegistry(hardPodAffinityWeight) - gotConfig := r[schedulerapi.SchedulerDefaultProviderName] - if diff := cmp.Diff(wantConfig, gotConfig); diff != "" { - t.Errorf("unexpected config diff (-want, +got): %s", diff) + r := NewRegistry(hardPodAffinityWeight) + gotConfig := r[schedulerapi.SchedulerDefaultProviderName] + if diff := cmp.Diff(test.wantConfig, gotConfig); diff != "" { + t.Errorf("unexpected config diff (-want, +got): %s", diff) + } + }) } } diff --git a/pkg/scheduler/apis/config/testing/compatibility_test.go b/pkg/scheduler/apis/config/testing/compatibility_test.go index f43f15d6440..b24ff55b532 100644 --- a/pkg/scheduler/apis/config/testing/compatibility_test.go +++ b/pkg/scheduler/apis/config/testing/compatibility_test.go @@ -133,6 +133,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "ScorePlugin": { {Name: "NodeResourcesLeastAllocated", Weight: 1}, {Name: "NodeLabel", Weight: 4}, + {Name: "DefaultPodTopologySpread", Weight: 2}, {Name: "ServiceAffinity", Weight: 3}, }, }, @@ -158,8 +159,10 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {"name": "LeastRequestedPriority", "weight": 2}, {"name": "BalancedResourceAllocation", "weight": 2}, {"name": "SelectorSpreadPriority", "weight": 2}, - {"name": "TestServiceAntiAffinity", "weight": 3, "argument": {"serviceAntiAffinity": {"label": "zone"}}}, - {"name": "TestLabelPreference", "weight": 4, "argument": {"labelPreference": {"label": "bar", "presence":true}}} + {"name": "TestServiceAntiAffinity1", "weight": 3, "argument": {"serviceAntiAffinity": {"label": "zone"}}}, + {"name": "TestServiceAntiAffinity2", "weight": 3, "argument": {"serviceAntiAffinity": {"label": "region"}}}, + {"name": "TestLabelPreference1", "weight": 4, "argument": {"labelPreference": {"label": "bar", "presence":true}}}, + {"name": "TestLabelPreference2", "weight": 4, "argument": {"labelPreference": {"label": "foo", "presence":false}}} ] }`, wantPlugins: map[string][]config.Plugin{ @@ -182,9 +185,9 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "ScorePlugin": { {Name: "NodeResourcesBalancedAllocation", Weight: 2}, {Name: "NodeResourcesLeastAllocated", Weight: 2}, - {Name: "NodeLabel", Weight: 4}, + {Name: "NodeLabel", Weight: 8}, // Weight is 4 * number of LabelPreference priorities {Name: "DefaultPodTopologySpread", Weight: 2}, - {Name: "ServiceAffinity", Weight: 3}, + {Name: "ServiceAffinity", Weight: 6}, // Weight is the 3 * number of custom ServiceAntiAffinity priorities }, }, }, @@ -1365,91 +1368,57 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { func TestAlgorithmProviderCompatibility(t *testing.T) { // Add serialized versions of scheduler config that exercise available options to ensure compatibility between releases + defaultPlugins := map[string][]config.Plugin{ + "PreFilterPlugin": { + {Name: "NodeResourcesFit"}, + {Name: "NodePorts"}, + {Name: "InterPodAffinity"}, + }, + "FilterPlugin": { + {Name: "NodeUnschedulable"}, + {Name: "NodeResourcesFit"}, + {Name: "NodeName"}, + {Name: "NodePorts"}, + {Name: "NodeAffinity"}, + {Name: "VolumeRestrictions"}, + {Name: "TaintToleration"}, + {Name: "EBSLimits"}, + {Name: "GCEPDLimits"}, + {Name: "NodeVolumeLimits"}, + {Name: "AzureDiskLimits"}, + {Name: "VolumeBinding"}, + {Name: "VolumeZone"}, + {Name: "InterPodAffinity"}, + }, + "PostFilterPlugin": { + {Name: "InterPodAffinity"}, + {Name: "TaintToleration"}, + }, + "ScorePlugin": { + {Name: "NodeResourcesBalancedAllocation", Weight: 1}, + {Name: "ImageLocality", Weight: 1}, + {Name: "InterPodAffinity", Weight: 1}, + {Name: "NodeResourcesLeastAllocated", Weight: 1}, + {Name: "NodeAffinity", Weight: 1}, + {Name: "NodePreferAvoidPods", Weight: 10000}, + {Name: "DefaultPodTopologySpread", Weight: 1}, + {Name: "TaintToleration", Weight: 1}, + }, + } + testcases := []struct { name string provider string wantPlugins map[string][]config.Plugin }{ { - name: "No Provider specified", - wantPlugins: map[string][]config.Plugin{ - "PreFilterPlugin": { - {Name: "NodeResourcesFit"}, - {Name: "NodePorts"}, - {Name: "InterPodAffinity"}, - }, - "FilterPlugin": { - {Name: "NodeUnschedulable"}, - {Name: "NodeResourcesFit"}, - {Name: "NodeName"}, - {Name: "NodePorts"}, - {Name: "NodeAffinity"}, - {Name: "VolumeRestrictions"}, - {Name: "TaintToleration"}, - {Name: "EBSLimits"}, - {Name: "GCEPDLimits"}, - {Name: "NodeVolumeLimits"}, - {Name: "AzureDiskLimits"}, - {Name: "VolumeBinding"}, - {Name: "VolumeZone"}, - {Name: "InterPodAffinity"}, - }, - "PostFilterPlugin": { - {Name: "InterPodAffinity"}, - {Name: "TaintToleration"}, - }, - "ScorePlugin": { - {Name: "NodeResourcesBalancedAllocation", Weight: 1}, - {Name: "ImageLocality", Weight: 1}, - {Name: "InterPodAffinity", Weight: 1}, - {Name: "NodeResourcesLeastAllocated", Weight: 1}, - {Name: "NodeAffinity", Weight: 1}, - {Name: "NodePreferAvoidPods", Weight: 10000}, - {Name: "DefaultPodTopologySpread", Weight: 1}, - {Name: "TaintToleration", Weight: 1}, - }, - }, + name: "No Provider specified", + wantPlugins: defaultPlugins, }, { - name: "DefaultProvider", - provider: config.SchedulerDefaultProviderName, - wantPlugins: map[string][]config.Plugin{ - "PreFilterPlugin": { - {Name: "NodeResourcesFit"}, - {Name: "NodePorts"}, - {Name: "InterPodAffinity"}, - }, - "FilterPlugin": { - {Name: "NodeUnschedulable"}, - {Name: "NodeResourcesFit"}, - {Name: "NodeName"}, - {Name: "NodePorts"}, - {Name: "NodeAffinity"}, - {Name: "VolumeRestrictions"}, - {Name: "TaintToleration"}, - {Name: "EBSLimits"}, - {Name: "GCEPDLimits"}, - {Name: "NodeVolumeLimits"}, - {Name: "AzureDiskLimits"}, - {Name: "VolumeBinding"}, - {Name: "VolumeZone"}, - {Name: "InterPodAffinity"}, - }, - "PostFilterPlugin": { - {Name: "InterPodAffinity"}, - {Name: "TaintToleration"}, - }, - "ScorePlugin": { - {Name: "NodeResourcesBalancedAllocation", Weight: 1}, - {Name: "ImageLocality", Weight: 1}, - {Name: "InterPodAffinity", Weight: 1}, - {Name: "NodeResourcesLeastAllocated", Weight: 1}, - {Name: "NodeAffinity", Weight: 1}, - {Name: "NodePreferAvoidPods", Weight: 10000}, - {Name: "DefaultPodTopologySpread", Weight: 1}, - {Name: "TaintToleration", Weight: 1}, - }, - }, + name: "DefaultProvider", + provider: config.SchedulerDefaultProviderName, + wantPlugins: defaultPlugins, }, { name: "ClusterAutoscalerProvider", diff --git a/pkg/scheduler/factory.go b/pkg/scheduler/factory.go index aabc583ea4f..5da4106c267 100644 --- a/pkg/scheduler/factory.go +++ b/pkg/scheduler/factory.go @@ -18,6 +18,7 @@ package scheduler import ( "fmt" + "sort" "time" v1 "k8s.io/api/core/v1" @@ -111,38 +112,12 @@ type Configurator struct { nodeInfoSnapshot *nodeinfosnapshot.Snapshot } -// Create creates a scheduler with the default algorithm provider. -func (c *Configurator) Create() (*Scheduler, error) { - return c.CreateFromProvider(schedulerapi.SchedulerDefaultProviderName) -} - -// CreateFromProvider creates a scheduler from the name of a registered algorithm provider. -func (c *Configurator) CreateFromProvider(providerName string) (*Scheduler, error) { - klog.V(2).Infof("Creating scheduler from algorithm provider '%v'", providerName) - r := algorithmprovider.NewRegistry(int64(c.hardPodAffinitySymmetricWeight)) - provider, exist := r[providerName] - if !exist { - return nil, fmt.Errorf("algorithm provider %q is not registered", providerName) - } - - // Combine the provided plugins with the ones from component config. - // TODO(#86162): address disabled plugins. - var plugins schedulerapi.Plugins - plugins.Append(provider.FrameworkPlugins) - plugins.Append(c.plugins) - var pluginConfig []schedulerapi.PluginConfig - pluginConfig = append(pluginConfig, provider.FrameworkPluginConfig...) - pluginConfig = append(pluginConfig, c.pluginConfig...) - - return c.CreateFromPlugins(&plugins, pluginConfig, []algorithm.SchedulerExtender{}) -} - -// CreateFromPlugins creates a scheduler from a set of registered plugins. -func (c *Configurator) CreateFromPlugins(plugins *schedulerapi.Plugins, pluginConfig []schedulerapi.PluginConfig, extenders []algorithm.SchedulerExtender) (*Scheduler, error) { +// create a scheduler from a set of registered plugins. +func (c *Configurator) create(extenders []algorithm.SchedulerExtender) (*Scheduler, error) { framework, err := framework.NewFramework( c.registry, - plugins, - pluginConfig, + c.plugins, + c.pluginConfig, framework.WithClientSet(c.client), framework.WithInformerFactory(c.informerFactory), framework.WithSnapshotSharedLister(c.nodeInfoSnapshot), @@ -198,13 +173,34 @@ func (c *Configurator) CreateFromPlugins(plugins *schedulerapi.Plugins, pluginCo StopEverything: c.StopEverything, VolumeBinder: c.volumeBinder, SchedulingQueue: podQueue, - Plugins: plugins, - PluginConfig: pluginConfig, }, nil } +// createFromProvider creates a scheduler from the name of a registered algorithm provider. +func (c *Configurator) createFromProvider(providerName string) (*Scheduler, error) { + klog.V(2).Infof("Creating scheduler from algorithm provider '%v'", providerName) + r := algorithmprovider.NewRegistry(int64(c.hardPodAffinitySymmetricWeight)) + provider, exist := r[providerName] + if !exist { + return nil, fmt.Errorf("algorithm provider %q is not registered", providerName) + } + + // Combine the provided plugins with the ones from component config. + // TODO(#86789): address disabled plugins. + var plugins schedulerapi.Plugins + plugins.Append(provider.FrameworkPlugins) + plugins.Append(c.plugins) + c.plugins = &plugins + var pluginConfig []schedulerapi.PluginConfig + pluginConfig = append(pluginConfig, provider.FrameworkPluginConfig...) + pluginConfig = append(pluginConfig, c.pluginConfig...) + c.pluginConfig = pluginConfig + + return c.create([]algorithm.SchedulerExtender{}) +} + // CreateFromConfig creates a scheduler from the configuration file -func (c *Configurator) CreateFromConfig(policy schedulerapi.Policy) (*Scheduler, error) { +func (c *Configurator) createFromConfig(policy schedulerapi.Policy) (*Scheduler, error) { lr := plugins.NewLegacyRegistry() args := &plugins.ConfigProducerArgs{} @@ -222,13 +218,13 @@ func (c *Configurator) CreateFromConfig(policy schedulerapi.Policy) (*Scheduler, } else { for _, predicate := range policy.Predicates { klog.V(2).Infof("Registering predicate: %s", predicate.Name) - predicateKeys.Insert(lr.RegisterCustomPredicate(predicate, args)) + predicateKeys.Insert(lr.ProcessPredicatePolicy(predicate, args)) } } - priorityKeys := sets.NewString() + priorityKeys := make(map[string]int64) if policy.Priorities == nil { - klog.V(2).Infof("Using priorities from algorithm provider '%v'", schedulerapi.SchedulerDefaultProviderName) + klog.V(2).Infof("Using default priorities") priorityKeys = lr.DefaultPriorities } else { for _, priority := range policy.Priorities { @@ -237,7 +233,7 @@ func (c *Configurator) CreateFromConfig(policy schedulerapi.Policy) (*Scheduler, continue } klog.V(2).Infof("Registering priority: %s", priority.Name) - priorityKeys.Insert(lr.RegisterCustomPriority(priority, args)) + priorityKeys[lr.ProcessPriorityPolicy(priority, args)] = priority.Weight } } @@ -305,33 +301,43 @@ func (c *Configurator) CreateFromConfig(policy schedulerapi.Policy) (*Scheduler, plugins.Append(pluginsForPredicates) plugins.Append(pluginsForPriorities) plugins.Append(c.plugins) + c.plugins = &plugins + var pluginConfig []schedulerapi.PluginConfig pluginConfig = append(pluginConfig, pluginConfigForPredicates...) pluginConfig = append(pluginConfig, pluginConfigForPriorities...) pluginConfig = append(pluginConfig, c.pluginConfig...) + c.pluginConfig = pluginConfig - return c.CreateFromPlugins(&plugins, pluginConfig, extenders) + return c.create(extenders) } // getPriorityConfigs // getPriorityConfigs returns priorities configuration: ones that will run as priorities and ones that will run // as framework plugins. Specifically, a priority will run as a framework plugin if a plugin config producer was // registered for that priority. -func getPriorityConfigs(keys sets.String, lr *plugins.LegacyRegistry, args *plugins.ConfigProducerArgs) (*schedulerapi.Plugins, []schedulerapi.PluginConfig, error) { +func getPriorityConfigs(keys map[string]int64, lr *plugins.LegacyRegistry, args *plugins.ConfigProducerArgs) (*schedulerapi.Plugins, []schedulerapi.PluginConfig, error) { var plugins schedulerapi.Plugins var pluginConfig []schedulerapi.PluginConfig - for _, p := range keys.List() { - weight, exist := lr.Priorities[p] + + // Sort the keys so that it is easier for unit tests to do compare. + var sortedKeys []string + for k := range keys { + sortedKeys = append(sortedKeys, k) + } + sort.Strings(sortedKeys) + + for _, priority := range sortedKeys { + weight := keys[priority] + producer, exist := lr.PriorityToConfigProducer[priority] if !exist { - return nil, nil, fmt.Errorf("priority key %q is not registered", p) - } - if producer, exist := lr.ConfigProducerRegistry.PriorityToConfigProducer[p]; exist { - a := *args - a.Weight = int32(weight) - pl, pc := producer(a) - plugins.Append(&pl) - pluginConfig = append(pluginConfig, pc...) + return nil, nil, fmt.Errorf("no config producer registered for %q", priority) } + a := *args + a.Weight = int32(weight) + pl, plc := producer(a) + plugins.Append(&pl) + pluginConfig = append(pluginConfig, plc...) } return &plugins, pluginConfig, nil } @@ -343,11 +349,6 @@ func getPriorityConfigs(keys sets.String, lr *plugins.LegacyRegistry, args *plug // are added to the Plugins list according to the order specified in predicates.Ordering(). func getPredicateConfigs(keys sets.String, lr *plugins.LegacyRegistry, args *plugins.ConfigProducerArgs) (*schedulerapi.Plugins, []schedulerapi.PluginConfig, error) { allPredicates := keys.Union(lr.MandatoryPredicates) - if allPredicates.Has("PodFitsPorts") { - // For compatibility reasons, "PodFitsPorts" as a key is still supported. - allPredicates.Delete("PodFitsPorts") - allPredicates.Insert(predicates.PodFitsHostPortsPred) - } // Create the framework plugin configurations, and place them in the order // that the corresponding predicates were supposed to run. @@ -356,26 +357,26 @@ func getPredicateConfigs(keys sets.String, lr *plugins.LegacyRegistry, args *plu for _, predicateKey := range predicates.Ordering() { if allPredicates.Has(predicateKey) { - producer, exist := lr.ConfigProducerRegistry.PredicateToConfigProducer[predicateKey] + producer, exist := lr.PredicateToConfigProducer[predicateKey] if !exist { return nil, nil, fmt.Errorf("no framework config producer registered for %q", predicateKey) } - p, pc := producer(*args) - plugins.Append(&p) - pluginConfig = append(pluginConfig, pc...) + pl, plc := producer(*args) + plugins.Append(&pl) + pluginConfig = append(pluginConfig, plc...) allPredicates.Delete(predicateKey) } } // Third, add the rest in no specific order. for predicateKey := range allPredicates { - producer, exist := lr.ConfigProducerRegistry.PredicateToConfigProducer[predicateKey] + producer, exist := lr.PredicateToConfigProducer[predicateKey] if !exist { return nil, nil, fmt.Errorf("no framework config producer registered for %q", predicateKey) } - p, pc := producer(*args) - plugins.Append(&p) - pluginConfig = append(pluginConfig, pc...) + pl, plc := producer(*args) + plugins.Append(&pl) + pluginConfig = append(pluginConfig, plc...) } return &plugins, pluginConfig, nil diff --git a/pkg/scheduler/factory_test.go b/pkg/scheduler/factory_test.go index 1745b13a585..db9ddbe118b 100644 --- a/pkg/scheduler/factory_test.go +++ b/pkg/scheduler/factory_test.go @@ -64,7 +64,7 @@ func TestCreate(t *testing.T) { stopCh := make(chan struct{}) defer close(stopCh) factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight, stopCh) - factory.Create() + factory.createFromProvider(schedulerapi.SchedulerDefaultProviderName) } // Test configures a scheduler from a policies defined in a file @@ -101,9 +101,9 @@ func TestCreateFromConfig(t *testing.T) { t.Errorf("Invalid configuration: %v", err) } - sched, err := factory.CreateFromConfig(policy) + sched, err := factory.createFromConfig(policy) if err != nil { - t.Fatalf("CreateFromConfig failed: %v", err) + t.Fatalf("createFromConfig failed: %v", err) } hpa := factory.hardPodAffinitySymmetricWeight if hpa != v1.DefaultHardPodAffinitySymmetricWeight { @@ -112,13 +112,13 @@ func TestCreateFromConfig(t *testing.T) { // Verify that node label predicate/priority are converted to framework plugins. wantArgs := `{"Name":"NodeLabel","Args":{"presentLabels":["zone"],"absentLabels":["foo"],"presentLabelsPreference":["l1"],"absentLabelsPreference":["l2"]}}` - verifyPluginConvertion(t, nodelabel.Name, []string{"FilterPlugin", "ScorePlugin"}, sched, 6, wantArgs) + verifyPluginConvertion(t, nodelabel.Name, []string{"FilterPlugin", "ScorePlugin"}, sched, factory, 6, wantArgs) // Verify that service affinity custom predicate/priority is converted to framework plugin. wantArgs = `{"Name":"ServiceAffinity","Args":{"labels":["zone","foo"],"antiAffinityLabelsPreference":["rack","zone"]}}` - verifyPluginConvertion(t, serviceaffinity.Name, []string{"FilterPlugin", "ScorePlugin"}, sched, 6, wantArgs) + verifyPluginConvertion(t, serviceaffinity.Name, []string{"FilterPlugin", "ScorePlugin"}, sched, factory, 6, wantArgs) } -func verifyPluginConvertion(t *testing.T, name string, extentionPoints []string, sched *Scheduler, wantWeight int32, wantArgs string) { +func verifyPluginConvertion(t *testing.T, name string, extentionPoints []string, sched *Scheduler, configurator *Configurator, wantWeight int32, wantArgs string) { for _, extensionPoint := range extentionPoints { plugin, ok := findPlugin(name, extensionPoint, sched) if !ok { @@ -130,7 +130,7 @@ func verifyPluginConvertion(t *testing.T, name string, extentionPoints []string, } } // Verify that the policy config is converted to plugin config. - pluginConfig := findPluginConfig(name, sched) + pluginConfig := findPluginConfig(name, configurator) encoding, err := json.Marshal(pluginConfig) if err != nil { t.Errorf("Failed to marshal %+v: %v", pluginConfig, err) @@ -150,8 +150,8 @@ func findPlugin(name, extensionPoint string, sched *Scheduler) (schedulerapi.Plu return schedulerapi.Plugin{}, false } -func findPluginConfig(name string, sched *Scheduler) schedulerapi.PluginConfig { - for _, c := range sched.PluginConfig { +func findPluginConfig(name string, configurator *Configurator) schedulerapi.PluginConfig { + for _, c := range configurator.pluginConfig { if c.Name == name { return c } @@ -187,7 +187,7 @@ func TestCreateFromConfigWithHardPodAffinitySymmetricWeight(t *testing.T) { if err := runtime.DecodeInto(scheme.Codecs.UniversalDecoder(), configData, &policy); err != nil { t.Errorf("Invalid configuration: %v", err) } - factory.CreateFromConfig(policy) + factory.createFromConfig(policy) hpa := factory.hardPodAffinitySymmetricWeight if hpa != 10 { t.Errorf("Wrong hardPodAffinitySymmetricWeight, ecpected: %d, got: %d", 10, hpa) @@ -208,7 +208,7 @@ func TestCreateFromEmptyConfig(t *testing.T) { t.Errorf("Invalid configuration: %v", err) } - factory.CreateFromConfig(policy) + factory.createFromConfig(policy) } // Test configures a scheduler from a policy that does not specify any @@ -229,24 +229,15 @@ func TestCreateFromConfigWithUnspecifiedPredicatesOrPriorities(t *testing.T) { t.Fatalf("Invalid configuration: %v", err) } - c, err := factory.CreateFromConfig(policy) + sched, err := factory.createFromConfig(policy) if err != nil { t.Fatalf("Failed to create scheduler from configuration: %v", err) } - if !foundPlugin(c.Plugins.Filter.Enabled, "NodeResourcesFit") { + if _, exist := findPlugin("NodeResourcesFit", "FilterPlugin", sched); !exist { t.Errorf("Expected plugin NodeResourcesFit") } } -func foundPlugin(plugins []schedulerapi.Plugin, name string) bool { - for _, plugin := range plugins { - if plugin.Name == name { - return true - } - } - return false -} - func TestDefaultErrorFunc(t *testing.T) { testPod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "bar"}, diff --git a/pkg/scheduler/framework/plugins/BUILD b/pkg/scheduler/framework/plugins/BUILD index 68220c26b14..4cf9d401b47 100644 --- a/pkg/scheduler/framework/plugins/BUILD +++ b/pkg/scheduler/framework/plugins/BUILD @@ -81,7 +81,6 @@ go_test( srcs = ["legacy_registry_test.go"], embed = [":go_default_library"], deps = [ - "//pkg/scheduler/algorithm/priorities:go_default_library", "//pkg/scheduler/apis/config:go_default_library", "//pkg/scheduler/framework/plugins/noderesources:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", diff --git a/pkg/scheduler/framework/plugins/legacy_registry.go b/pkg/scheduler/framework/plugins/legacy_registry.go index 79e282dcdbc..4beffa1a97b 100644 --- a/pkg/scheduler/framework/plugins/legacy_registry.go +++ b/pkg/scheduler/framework/plugins/legacy_registry.go @@ -1,5 +1,5 @@ /* -Copyright 2014 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -51,12 +51,15 @@ import ( // LegacyRegistry is used to store current state of registered predicates and priorities. type LegacyRegistry struct { - Predicates sets.String - Priorities map[string]int64 - MandatoryPredicates sets.String - DefaultPredicates sets.String - DefaultPriorities sets.String - ConfigProducerRegistry *ConfigProducerRegistry + // maps that associate predicates/priorities with framework plugin configurations. + PredicateToConfigProducer map[string]ConfigProducer + PriorityToConfigProducer map[string]ConfigProducer + // predicates that will always be configured. + MandatoryPredicates sets.String + // predicates and priorities that will be used if either was set to nil in a + // given v1.Policy configuration. + DefaultPredicates sets.String + DefaultPriorities map[string]int64 } // ConfigProducerArgs contains arguments that are passed to the producer. @@ -77,24 +80,52 @@ type ConfigProducerArgs struct { InterPodAffinityArgs *interpodaffinity.Args } -// ConfigProducer produces a framework's configuration. +// ConfigProducer returns the set of plugins and their configuration for a +// predicate/priority given the args. type ConfigProducer func(args ConfigProducerArgs) (config.Plugins, []config.PluginConfig) -// ConfigProducerRegistry tracks mappings from predicates/priorities to framework config producers. -type ConfigProducerRegistry struct { - // maps that associate predicates/priorities with framework plugin configurations. - PredicateToConfigProducer map[string]ConfigProducer - PriorityToConfigProducer map[string]ConfigProducer -} +// NewLegacyRegistry returns a legacy algorithm registry of predicates and priorities. +func NewLegacyRegistry() *LegacyRegistry { + registry := &LegacyRegistry{ + // MandatoryPredicates the set of keys for predicates that the scheduler will + // be configured with all the time. + MandatoryPredicates: sets.NewString( + predicates.PodToleratesNodeTaintsPred, + predicates.CheckNodeUnschedulablePred, + ), + + // Used as the default set of predicates if Policy was specified, but predicates was nil. + DefaultPredicates: sets.NewString( + predicates.NoVolumeZoneConflictPred, + predicates.MaxEBSVolumeCountPred, + predicates.MaxGCEPDVolumeCountPred, + predicates.MaxAzureDiskVolumeCountPred, + predicates.MaxCSIVolumeCountPred, + predicates.MatchInterPodAffinityPred, + predicates.NoDiskConflictPred, + predicates.GeneralPred, + predicates.PodToleratesNodeTaintsPred, + predicates.CheckVolumeBindingPred, + predicates.CheckNodeUnschedulablePred, + ), + + // Used as the default set of predicates if Policy was specified, but priorities was nil. + DefaultPriorities: map[string]int64{ + priorities.SelectorSpreadPriority: 1, + priorities.InterPodAffinityPriority: 1, + priorities.LeastRequestedPriority: 1, + priorities.BalancedResourceAllocation: 1, + priorities.NodePreferAvoidPodsPriority: 10000, + priorities.NodeAffinityPriority: 1, + priorities.TaintTolerationPriority: 1, + priorities.ImageLocalityPriority: 1, + }, -// newConfigProducerRegistry creates a new producer registry. -func newConfigProducerRegistry() *ConfigProducerRegistry { - registry := &ConfigProducerRegistry{ PredicateToConfigProducer: make(map[string]ConfigProducer), PriorityToConfigProducer: make(map[string]ConfigProducer), } - // Register Predicates. - registry.RegisterPredicate(predicates.GeneralPred, + + registry.registerPredicateConfigProducer(predicates.GeneralPred, func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { // GeneralPredicate is a combination of predicates. plugins.Filter = appendToPluginSet(plugins.Filter, noderesources.FitName, nil) @@ -106,98 +137,92 @@ func newConfigProducerRegistry() *ConfigProducerRegistry { plugins.Filter = appendToPluginSet(plugins.Filter, nodeaffinity.Name, nil) return }) - registry.RegisterPredicate(predicates.PodToleratesNodeTaintsPred, + registry.registerPredicateConfigProducer(predicates.PodToleratesNodeTaintsPred, func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, tainttoleration.Name, nil) return }) - registry.RegisterPredicate(predicates.PodFitsResourcesPred, + registry.registerPredicateConfigProducer(predicates.PodFitsResourcesPred, func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, noderesources.FitName, nil) plugins.PreFilter = appendToPluginSet(plugins.PreFilter, noderesources.FitName, nil) pluginConfig = append(pluginConfig, makePluginConfig(noderesources.FitName, args.NodeResourcesFitArgs)) return }) - registry.RegisterPredicate(predicates.HostNamePred, + registry.registerPredicateConfigProducer(predicates.HostNamePred, func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, nodename.Name, nil) return }) - registry.RegisterPredicate(predicates.PodFitsHostPortsPred, + registry.registerPredicateConfigProducer(predicates.PodFitsHostPortsPred, func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, nodeports.Name, nil) plugins.PreFilter = appendToPluginSet(plugins.PreFilter, nodeports.Name, nil) return }) - registry.RegisterPredicate(predicates.MatchNodeSelectorPred, + registry.registerPredicateConfigProducer(predicates.MatchNodeSelectorPred, func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, nodeaffinity.Name, nil) return }) - registry.RegisterPredicate(predicates.CheckNodeUnschedulablePred, + registry.registerPredicateConfigProducer(predicates.CheckNodeUnschedulablePred, func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, nodeunschedulable.Name, nil) return }) - registry.RegisterPredicate(predicates.CheckVolumeBindingPred, + registry.registerPredicateConfigProducer(predicates.CheckVolumeBindingPred, func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, volumebinding.Name, nil) return }) - registry.RegisterPredicate(predicates.NoDiskConflictPred, + registry.registerPredicateConfigProducer(predicates.NoDiskConflictPred, func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, volumerestrictions.Name, nil) return }) - registry.RegisterPredicate(predicates.NoVolumeZoneConflictPred, + registry.registerPredicateConfigProducer(predicates.NoVolumeZoneConflictPred, func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, volumezone.Name, nil) return }) - registry.RegisterPredicate(predicates.MaxCSIVolumeCountPred, + registry.registerPredicateConfigProducer(predicates.MaxCSIVolumeCountPred, func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, nodevolumelimits.CSIName, nil) return }) - registry.RegisterPredicate(predicates.MaxEBSVolumeCountPred, + registry.registerPredicateConfigProducer(predicates.MaxEBSVolumeCountPred, func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, nodevolumelimits.EBSName, nil) return }) - registry.RegisterPredicate(predicates.MaxGCEPDVolumeCountPred, + registry.registerPredicateConfigProducer(predicates.MaxGCEPDVolumeCountPred, func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, nodevolumelimits.GCEPDName, nil) return }) - registry.RegisterPredicate(predicates.MaxAzureDiskVolumeCountPred, + registry.registerPredicateConfigProducer(predicates.MaxAzureDiskVolumeCountPred, func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, nodevolumelimits.AzureDiskName, nil) return }) - registry.RegisterPredicate(predicates.MaxCinderVolumeCountPred, + registry.registerPredicateConfigProducer(predicates.MaxCinderVolumeCountPred, func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, nodevolumelimits.CinderName, nil) return }) - registry.RegisterPredicate(predicates.MatchInterPodAffinityPred, + registry.registerPredicateConfigProducer(predicates.MatchInterPodAffinityPred, func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, interpodaffinity.Name, nil) plugins.PreFilter = appendToPluginSet(plugins.PreFilter, interpodaffinity.Name, nil) return }) - registry.RegisterPredicate(predicates.EvenPodsSpreadPred, - func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { - plugins.PreFilter = appendToPluginSet(plugins.PreFilter, podtopologyspread.Name, nil) - plugins.Filter = appendToPluginSet(plugins.Filter, podtopologyspread.Name, nil) - return - }) - registry.RegisterPredicate(predicates.CheckNodeLabelPresencePred, + registry.registerPredicateConfigProducer(predicates.CheckNodeLabelPresencePred, func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, nodelabel.Name, nil) pluginConfig = append(pluginConfig, makePluginConfig(nodelabel.Name, args.NodeLabelArgs)) return }) - registry.RegisterPredicate(predicates.CheckServiceAffinityPred, + registry.registerPredicateConfigProducer(predicates.CheckServiceAffinityPred, func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, serviceaffinity.Name, nil) pluginConfig = append(pluginConfig, makePluginConfig(serviceaffinity.Name, args.ServiceAffinityArgs)) @@ -206,104 +231,135 @@ func newConfigProducerRegistry() *ConfigProducerRegistry { }) // Register Priorities. - registry.RegisterPriority(priorities.SelectorSpreadPriority, + registry.registerPriorityConfigProducer(priorities.SelectorSpreadPriority, func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Score = appendToPluginSet(plugins.Score, defaultpodtopologyspread.Name, &args.Weight) return }) - registry.RegisterPriority(priorities.TaintTolerationPriority, + registry.registerPriorityConfigProducer(priorities.TaintTolerationPriority, func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.PostFilter = appendToPluginSet(plugins.PostFilter, tainttoleration.Name, nil) plugins.Score = appendToPluginSet(plugins.Score, tainttoleration.Name, &args.Weight) return }) - registry.RegisterPriority(priorities.NodeAffinityPriority, + registry.registerPriorityConfigProducer(priorities.NodeAffinityPriority, func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Score = appendToPluginSet(plugins.Score, nodeaffinity.Name, &args.Weight) return }) - registry.RegisterPriority(priorities.ImageLocalityPriority, + registry.registerPriorityConfigProducer(priorities.ImageLocalityPriority, func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Score = appendToPluginSet(plugins.Score, imagelocality.Name, &args.Weight) return }) - registry.RegisterPriority(priorities.InterPodAffinityPriority, + registry.registerPriorityConfigProducer(priorities.InterPodAffinityPriority, func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.PostFilter = appendToPluginSet(plugins.PostFilter, interpodaffinity.Name, nil) plugins.Score = appendToPluginSet(plugins.Score, interpodaffinity.Name, &args.Weight) pluginConfig = append(pluginConfig, makePluginConfig(interpodaffinity.Name, args.InterPodAffinityArgs)) return }) - registry.RegisterPriority(priorities.NodePreferAvoidPodsPriority, + registry.registerPriorityConfigProducer(priorities.NodePreferAvoidPodsPriority, func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Score = appendToPluginSet(plugins.Score, nodepreferavoidpods.Name, &args.Weight) return }) - registry.RegisterPriority(priorities.MostRequestedPriority, + registry.registerPriorityConfigProducer(priorities.MostRequestedPriority, func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Score = appendToPluginSet(plugins.Score, noderesources.MostAllocatedName, &args.Weight) return }) - registry.RegisterPriority(priorities.BalancedResourceAllocation, + registry.registerPriorityConfigProducer(priorities.BalancedResourceAllocation, func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Score = appendToPluginSet(plugins.Score, noderesources.BalancedAllocationName, &args.Weight) return }) - registry.RegisterPriority(priorities.LeastRequestedPriority, + registry.registerPriorityConfigProducer(priorities.LeastRequestedPriority, func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Score = appendToPluginSet(plugins.Score, noderesources.LeastAllocatedName, &args.Weight) return }) - registry.RegisterPriority(priorities.EvenPodsSpreadPriority, - func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { - plugins.PostFilter = appendToPluginSet(plugins.PostFilter, podtopologyspread.Name, nil) - plugins.Score = appendToPluginSet(plugins.Score, podtopologyspread.Name, &args.Weight) - return - }) - registry.RegisterPriority(noderesources.RequestedToCapacityRatioName, + registry.registerPriorityConfigProducer(noderesources.RequestedToCapacityRatioName, func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Score = appendToPluginSet(plugins.Score, noderesources.RequestedToCapacityRatioName, &args.Weight) pluginConfig = append(pluginConfig, makePluginConfig(noderesources.RequestedToCapacityRatioName, args.RequestedToCapacityRatioArgs)) return }) - registry.RegisterPriority(nodelabel.Name, + registry.registerPriorityConfigProducer(nodelabel.Name, func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { - plugins.Score = appendToPluginSet(plugins.Score, nodelabel.Name, &args.Weight) + // If there are n LabelPreference priorities in the policy, the weight for the corresponding + // score plugin is n*weight (note that the validation logic verifies that all LabelPreference + // priorities specified in Policy have the same weight). + weight := args.Weight * int32(len(args.NodeLabelArgs.PresentLabelsPreference)+len(args.NodeLabelArgs.AbsentLabelsPreference)) + plugins.Score = appendToPluginSet(plugins.Score, nodelabel.Name, &weight) pluginConfig = append(pluginConfig, makePluginConfig(nodelabel.Name, args.NodeLabelArgs)) return }) - registry.RegisterPriority(serviceaffinity.Name, + registry.registerPriorityConfigProducer(serviceaffinity.Name, func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { - plugins.Score = appendToPluginSet(plugins.Score, serviceaffinity.Name, &args.Weight) + // If there are n ServiceAffinity priorities in the policy, the weight for the corresponding + // score plugin is n*weight (note that the validation logic verifies that all ServiceAffinity + // priorities specified in Policy have the same weight). + weight := args.Weight * int32(len(args.ServiceAffinityArgs.AntiAffinityLabelsPreference)) + plugins.Score = appendToPluginSet(plugins.Score, serviceaffinity.Name, &weight) pluginConfig = append(pluginConfig, makePluginConfig(serviceaffinity.Name, args.ServiceAffinityArgs)) return }) - registry.RegisterPriority(priorities.ResourceLimitsPriority, - func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { - plugins.PostFilter = appendToPluginSet(plugins.PostFilter, noderesources.ResourceLimitsName, nil) - plugins.Score = appendToPluginSet(plugins.Score, noderesources.ResourceLimitsName, &args.Weight) - return - }) + + // The following two features are the last ones to be supported as predicate/priority. + // Once they graduate to GA, there will be no more checking for featue gates here. + // Only register EvenPodsSpread predicate & priority if the feature is enabled + if utilfeature.DefaultFeatureGate.Enabled(features.EvenPodsSpread) { + klog.Infof("Registering EvenPodsSpread predicate and priority function") + + registry.registerPredicateConfigProducer(predicates.EvenPodsSpreadPred, + func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + plugins.PreFilter = appendToPluginSet(plugins.PreFilter, podtopologyspread.Name, nil) + plugins.Filter = appendToPluginSet(plugins.Filter, podtopologyspread.Name, nil) + return + }) + registry.DefaultPredicates.Insert(predicates.EvenPodsSpreadPred) + + registry.registerPriorityConfigProducer(priorities.EvenPodsSpreadPriority, + func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + plugins.PostFilter = appendToPluginSet(plugins.PostFilter, podtopologyspread.Name, nil) + plugins.Score = appendToPluginSet(plugins.Score, podtopologyspread.Name, &args.Weight) + return + }) + registry.DefaultPriorities[priorities.EvenPodsSpreadPriority] = 1 + } + + // Prioritizes nodes that satisfy pod's resource limits + if utilfeature.DefaultFeatureGate.Enabled(features.ResourceLimitsPriorityFunction) { + klog.Infof("Registering resourcelimits priority function") + + registry.registerPriorityConfigProducer(priorities.ResourceLimitsPriority, + func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + plugins.PostFilter = appendToPluginSet(plugins.PostFilter, noderesources.ResourceLimitsName, nil) + plugins.Score = appendToPluginSet(plugins.Score, noderesources.ResourceLimitsName, &args.Weight) + return + }) + registry.DefaultPriorities[priorities.ResourceLimitsPriority] = 1 + } + return registry } -func registerProducer(name string, producer ConfigProducer, producersMap map[string]ConfigProducer) error { - if _, exist := producersMap[name]; exist { - return fmt.Errorf("already registered %q", name) +// registers a config producer for a predicate. +func (lr *LegacyRegistry) registerPredicateConfigProducer(name string, producer ConfigProducer) { + if _, exist := lr.PredicateToConfigProducer[name]; exist { + klog.Fatalf("already registered %q", name) } - producersMap[name] = producer - return nil + lr.PredicateToConfigProducer[name] = producer } -// RegisterPredicate registers a config producer for a predicate. -func (f *ConfigProducerRegistry) RegisterPredicate(name string, producer ConfigProducer) error { - return registerProducer(name, producer, f.PredicateToConfigProducer) -} - -// RegisterPriority registers a framework config producer for a priority. -func (f *ConfigProducerRegistry) RegisterPriority(name string, producer ConfigProducer) error { - return registerProducer(name, producer, f.PriorityToConfigProducer) +// registers a framework config producer for a priority. +func (lr *LegacyRegistry) registerPriorityConfigProducer(name string, producer ConfigProducer) { + if _, exist := lr.PriorityToConfigProducer[name]; exist { + klog.Fatalf("already registered %q", name) + } + lr.PriorityToConfigProducer[name] = producer } func appendToPluginSet(set *config.PluginSet, name string, weight *int32) *config.PluginSet { @@ -321,7 +377,7 @@ func appendToPluginSet(set *config.PluginSet, name string, weight *int32) *confi func makePluginConfig(pluginName string, args interface{}) config.PluginConfig { encoding, err := json.Marshal(args) if err != nil { - klog.Fatal(fmt.Errorf("Failed to marshal %+v: %v", args, err)) + klog.Fatal(fmt.Errorf("failed to marshal %+v: %v", args, err)) return config.PluginConfig{} } config := config.PluginConfig{ @@ -331,226 +387,136 @@ func makePluginConfig(pluginName string, args interface{}) config.PluginConfig { return config } -// NewLegacyRegistry returns a legacy algorithm registry of predicates and priorities. -func NewLegacyRegistry() *LegacyRegistry { - registry := &LegacyRegistry{ - // predicate keys supported for backward compatibility with v1.Policy. - Predicates: sets.NewString( - "PodFitsPorts", // This exists for compatibility reasons. - predicates.PodFitsHostPortsPred, - predicates.PodFitsResourcesPred, - predicates.HostNamePred, - predicates.MatchNodeSelectorPred, - predicates.NoVolumeZoneConflictPred, - predicates.MaxEBSVolumeCountPred, - predicates.MaxGCEPDVolumeCountPred, - predicates.MaxAzureDiskVolumeCountPred, - predicates.MaxCSIVolumeCountPred, - predicates.MaxCinderVolumeCountPred, - predicates.MatchInterPodAffinityPred, - predicates.NoDiskConflictPred, - predicates.GeneralPred, - predicates.PodToleratesNodeTaintsPred, - predicates.CheckNodeUnschedulablePred, - predicates.CheckVolumeBindingPred, - ), - - // priority keys to weights, this exist for backward compatibility with v1.Policy. - Priorities: map[string]int64{ - priorities.LeastRequestedPriority: 1, - priorities.BalancedResourceAllocation: 1, - priorities.MostRequestedPriority: 1, - priorities.ImageLocalityPriority: 1, - priorities.NodeAffinityPriority: 1, - priorities.SelectorSpreadPriority: 1, - priorities.ServiceSpreadingPriority: 1, - priorities.TaintTolerationPriority: 1, - priorities.InterPodAffinityPriority: 1, - priorities.NodePreferAvoidPodsPriority: 10000, - }, - - // MandatoryPredicates the set of keys for predicates that the scheduler will - // be configured with all the time. - MandatoryPredicates: sets.NewString( - predicates.PodToleratesNodeTaintsPred, - predicates.CheckNodeUnschedulablePred, - ), - - DefaultPredicates: sets.NewString( - predicates.NoVolumeZoneConflictPred, - predicates.MaxEBSVolumeCountPred, - predicates.MaxGCEPDVolumeCountPred, - predicates.MaxAzureDiskVolumeCountPred, - predicates.MaxCSIVolumeCountPred, - predicates.MatchInterPodAffinityPred, - predicates.NoDiskConflictPred, - predicates.GeneralPred, - predicates.PodToleratesNodeTaintsPred, - predicates.CheckVolumeBindingPred, - predicates.CheckNodeUnschedulablePred, - ), - - DefaultPriorities: sets.NewString( - priorities.SelectorSpreadPriority, - priorities.InterPodAffinityPriority, - priorities.LeastRequestedPriority, - priorities.BalancedResourceAllocation, - priorities.NodePreferAvoidPodsPriority, - priorities.NodeAffinityPriority, - priorities.TaintTolerationPriority, - priorities.ImageLocalityPriority, - ), - - ConfigProducerRegistry: newConfigProducerRegistry(), - } - - // The following two features are the last ones to be supported as predicate/priority. - // Once they graduate to GA, there will be no more checking for featue gates here. - // Only register EvenPodsSpread predicate & priority if the feature is enabled - if utilfeature.DefaultFeatureGate.Enabled(features.EvenPodsSpread) { - klog.Infof("Registering EvenPodsSpread predicate and priority function") - - registry.Predicates.Insert(predicates.EvenPodsSpreadPred) - registry.DefaultPredicates.Insert(predicates.EvenPodsSpreadPred) - - registry.Priorities[priorities.EvenPodsSpreadPriority] = 1 - registry.DefaultPriorities.Insert(priorities.EvenPodsSpreadPriority) - } - - // Prioritizes nodes that satisfy pod's resource limits - if utilfeature.DefaultFeatureGate.Enabled(features.ResourceLimitsPriorityFunction) { - klog.Infof("Registering resourcelimits priority function") - - registry.Priorities[priorities.ResourceLimitsPriority] = 1 - registry.DefaultPriorities.Insert(priorities.ResourceLimitsPriority) - } - - return registry -} - -// RegisterCustomPredicate registers a custom fit predicate with the algorithm registry. -// Returns the name, with which the predicate was registered. -func (a *LegacyRegistry) RegisterCustomPredicate(policy config.PredicatePolicy, pluginArgs *ConfigProducerArgs) string { - var ok bool - var predicate string - +// ProcessPredicatePolicy given a PredicatePolicy, return the plugin name implementing the predicate and update +// the ConfigProducerArgs if necessary. +func (lr *LegacyRegistry) ProcessPredicatePolicy(policy config.PredicatePolicy, pluginArgs *ConfigProducerArgs) string { validatePredicateOrDie(policy) - // generate the predicate function, if a custom type is requested - if policy.Argument != nil { - if policy.Argument.ServiceAffinity != nil { - // We use the ServiceAffinity predicate name for all ServiceAffinity custom predicates. - // It may get called multiple times but we essentially only register one instance of ServiceAffinity predicate. - // This name is then used to find the registered plugin and run the plugin instead of the predicate. - predicate = predicates.CheckServiceAffinityPred + predicateName := policy.Name + if policy.Name == "PodFitsPorts" { + // For compatibility reasons, "PodFitsPorts" as a key is still supported. + predicateName = predicates.PodFitsHostPortsPred + } - // map LabelsPresence policy to ConfigProducerArgs that's used to configure the ServiceAffinity plugin. - if pluginArgs.ServiceAffinityArgs == nil { - pluginArgs.ServiceAffinityArgs = &serviceaffinity.Args{} - } - - pluginArgs.ServiceAffinityArgs.AffinityLabels = append(pluginArgs.ServiceAffinityArgs.AffinityLabels, policy.Argument.ServiceAffinity.Labels...) - } else if policy.Argument.LabelsPresence != nil { - // We use the CheckNodeLabelPresencePred predicate name for all kNodeLabel custom predicates. - // It may get called multiple times but we essentially only register one instance of NodeLabel predicate. - // This name is then used to find the registered plugin and run the plugin instead of the predicate. - predicate = predicates.CheckNodeLabelPresencePred - - // Map LabelPresence policy to ConfigProducerArgs that's used to configure the NodeLabel plugin. - if pluginArgs.NodeLabelArgs == nil { - pluginArgs.NodeLabelArgs = &nodelabel.Args{} - } - if policy.Argument.LabelsPresence.Presence { - pluginArgs.NodeLabelArgs.PresentLabels = append(pluginArgs.NodeLabelArgs.PresentLabels, policy.Argument.LabelsPresence.Labels...) - } else { - pluginArgs.NodeLabelArgs.AbsentLabels = append(pluginArgs.NodeLabelArgs.AbsentLabels, policy.Argument.LabelsPresence.Labels...) - } - } - } else if _, ok = a.Predicates[policy.Name]; ok { + if _, ok := lr.PredicateToConfigProducer[predicateName]; ok { // checking to see if a pre-defined predicate is requested klog.V(2).Infof("Predicate type %s already registered, reusing.", policy.Name) - return policy.Name + return predicateName } - if len(predicate) == 0 { - klog.Fatalf("Invalid configuration: Predicate type not found for %s", policy.Name) + if policy.Argument == nil || (policy.Argument.ServiceAffinity == nil && + policy.Argument.LabelsPresence == nil) { + klog.Fatalf("Invalid configuration: Predicate type not found for %q", policy.Name) } - return predicate + // generate the predicate function, if a custom type is requested + if policy.Argument.ServiceAffinity != nil { + // map LabelsPresence policy to ConfigProducerArgs that's used to configure the ServiceAffinity plugin. + if pluginArgs.ServiceAffinityArgs == nil { + pluginArgs.ServiceAffinityArgs = &serviceaffinity.Args{} + } + pluginArgs.ServiceAffinityArgs.AffinityLabels = append(pluginArgs.ServiceAffinityArgs.AffinityLabels, policy.Argument.ServiceAffinity.Labels...) + + // We use the ServiceAffinity predicate name for all ServiceAffinity custom predicates. + // It may get called multiple times but we essentially only register one instance of ServiceAffinity predicate. + // This name is then used to find the registered plugin and run the plugin instead of the predicate. + predicateName = predicates.CheckServiceAffinityPred + } + + if policy.Argument.LabelsPresence != nil { + // Map LabelPresence policy to ConfigProducerArgs that's used to configure the NodeLabel plugin. + if pluginArgs.NodeLabelArgs == nil { + pluginArgs.NodeLabelArgs = &nodelabel.Args{} + } + if policy.Argument.LabelsPresence.Presence { + pluginArgs.NodeLabelArgs.PresentLabels = append(pluginArgs.NodeLabelArgs.PresentLabels, policy.Argument.LabelsPresence.Labels...) + } else { + pluginArgs.NodeLabelArgs.AbsentLabels = append(pluginArgs.NodeLabelArgs.AbsentLabels, policy.Argument.LabelsPresence.Labels...) + } + + // We use the CheckNodeLabelPresencePred predicate name for all kNodeLabel custom predicates. + // It may get called multiple times but we essentially only register one instance of NodeLabel predicate. + // This name is then used to find the registered plugin and run the plugin instead of the predicate. + predicateName = predicates.CheckNodeLabelPresencePred + + } + return predicateName } -// RegisterCustomPriority registers a custom priority with the algorithm registry. -// Returns the name, with which the priority function was registered. -func (a *LegacyRegistry) RegisterCustomPriority(policy config.PriorityPolicy, configProducerArgs *ConfigProducerArgs) string { - var priority string - var weight int64 - +// ProcessPriorityPolicy given a PriorityPolicy, return the plugin name implementing the priority and update +// the ConfigProducerArgs if necessary. +func (lr *LegacyRegistry) ProcessPriorityPolicy(policy config.PriorityPolicy, configProducerArgs *ConfigProducerArgs) string { validatePriorityOrDie(policy) + priorityName := policy.Name + if policy.Name == priorities.ServiceSpreadingPriority { + // For compatibility reasons, "ServiceSpreadingPriority" as a key is still supported. + priorityName = priorities.SelectorSpreadPriority + } + + if _, ok := lr.PriorityToConfigProducer[priorityName]; ok { + klog.V(2).Infof("Priority type %s already registered, reusing.", priorityName) + return priorityName + } + // generate the priority function, if a custom priority is requested - if policy.Argument != nil { - if policy.Argument.ServiceAntiAffinity != nil { - // We use the ServiceAffinity plugin name for all ServiceAffinity custom priorities. - // It may get called multiple times but we essentially only register one instance of - // ServiceAffinity priority. - // This name is then used to find the registered plugin and run the plugin instead of the priority. - priority = serviceaffinity.Name + if policy.Argument == nil || + (policy.Argument.ServiceAntiAffinity == nil && + policy.Argument.RequestedToCapacityRatioArguments == nil && + policy.Argument.LabelPreference == nil) { + klog.Fatalf("Invalid configuration: Priority type not found for %q", priorityName) + } - if configProducerArgs.ServiceAffinityArgs == nil { - configProducerArgs.ServiceAffinityArgs = &serviceaffinity.Args{} - } - configProducerArgs.ServiceAffinityArgs.AntiAffinityLabelsPreference = append(configProducerArgs.ServiceAffinityArgs.AntiAffinityLabelsPreference, policy.Argument.ServiceAntiAffinity.Label) - - weight = policy.Weight - if existingWeight, ok := a.Priorities[priority]; ok { - // If there are n ServiceAffinity priorities in the policy, the weight for the corresponding - // score plugin is n*(weight of each priority). - weight += existingWeight - } - } else if policy.Argument.LabelPreference != nil { - // We use the NodeLabel plugin name for all NodeLabel custom priorities. - // It may get called multiple times but we essentially only register one instance of NodeLabel priority. - // This name is then used to find the registered plugin and run the plugin instead of the priority. - priority = nodelabel.Name - if configProducerArgs.NodeLabelArgs == nil { - configProducerArgs.NodeLabelArgs = &nodelabel.Args{} - } - if policy.Argument.LabelPreference.Presence { - configProducerArgs.NodeLabelArgs.PresentLabelsPreference = append(configProducerArgs.NodeLabelArgs.PresentLabelsPreference, policy.Argument.LabelPreference.Label) - } else { - configProducerArgs.NodeLabelArgs.AbsentLabelsPreference = append(configProducerArgs.NodeLabelArgs.AbsentLabelsPreference, policy.Argument.LabelPreference.Label) - } - weight = policy.Weight - if existingWeight, ok := a.Priorities[priority]; ok { - // If there are n NodeLabel priority configured in the policy, the weight for the corresponding - // priority is n*(weight of each priority in policy). - weight += existingWeight - } - } else if policy.Argument.RequestedToCapacityRatioArguments != nil { - scoringFunctionShape, resources := buildScoringFunctionShapeFromRequestedToCapacityRatioArguments(policy.Argument.RequestedToCapacityRatioArguments) - configProducerArgs.RequestedToCapacityRatioArgs = &noderesources.RequestedToCapacityRatioArgs{ - FunctionShape: scoringFunctionShape, - ResourceToWeightMap: resources, - } - // We do not allow specifying the name for custom plugins, see #83472 - priority = noderesources.RequestedToCapacityRatioName - weight = policy.Weight + if policy.Argument.ServiceAntiAffinity != nil { + // We use the ServiceAffinity plugin name for all ServiceAffinity custom priorities. + // It may get called multiple times but we essentially only register one instance of + // ServiceAffinity priority. + // This name is then used to find the registered plugin and run the plugin instead of the priority. + priorityName = serviceaffinity.Name + if configProducerArgs.ServiceAffinityArgs == nil { + configProducerArgs.ServiceAffinityArgs = &serviceaffinity.Args{} } - } else if _, ok := a.Priorities[policy.Name]; ok { - klog.V(2).Infof("Priority type %s already registered, reusing.", policy.Name) - // set/update the weight based on the policy - priority = policy.Name - weight = policy.Weight + configProducerArgs.ServiceAffinityArgs.AntiAffinityLabelsPreference = append( + configProducerArgs.ServiceAffinityArgs.AntiAffinityLabelsPreference, + policy.Argument.ServiceAntiAffinity.Label, + ) } - if len(priority) == 0 { - klog.Fatalf("Invalid configuration: Priority type not found for %s", policy.Name) + if policy.Argument.LabelPreference != nil { + // We use the NodeLabel plugin name for all NodeLabel custom priorities. + // It may get called multiple times but we essentially only register one instance of NodeLabel priority. + // This name is then used to find the registered plugin and run the plugin instead of the priority. + priorityName = nodelabel.Name + if configProducerArgs.NodeLabelArgs == nil { + configProducerArgs.NodeLabelArgs = &nodelabel.Args{} + } + if policy.Argument.LabelPreference.Presence { + configProducerArgs.NodeLabelArgs.PresentLabelsPreference = append( + configProducerArgs.NodeLabelArgs.PresentLabelsPreference, + policy.Argument.LabelPreference.Label, + ) + } else { + configProducerArgs.NodeLabelArgs.AbsentLabelsPreference = append( + configProducerArgs.NodeLabelArgs.AbsentLabelsPreference, + policy.Argument.LabelPreference.Label, + ) + } } - a.Priorities[priority] = weight - return priority + if policy.Argument.RequestedToCapacityRatioArguments != nil { + scoringFunctionShape, resources := buildScoringFunctionShapeFromRequestedToCapacityRatioArguments(policy.Argument.RequestedToCapacityRatioArguments) + configProducerArgs.RequestedToCapacityRatioArgs = &noderesources.RequestedToCapacityRatioArgs{ + FunctionShape: scoringFunctionShape, + ResourceToWeightMap: resources, + } + // We do not allow specifying the name for custom plugins, see #83472 + priorityName = noderesources.RequestedToCapacityRatioName + } + + return priorityName } +// TODO(ahg-g): move to RequestedToCapacityRatio plugin. func buildScoringFunctionShapeFromRequestedToCapacityRatioArguments(arguments *config.RequestedToCapacityRatioArguments) (noderesources.FunctionShape, noderesources.ResourceToWeightMap) { n := len(arguments.Shape) points := make([]noderesources.FunctionShapePoint, 0, n) @@ -567,7 +533,7 @@ func buildScoringFunctionShapeFromRequestedToCapacityRatioArguments(arguments *c if err != nil { klog.Fatalf("invalid RequestedToCapacityRatioPriority arguments: %s", err.Error()) } - resourceToWeightMap := make(noderesources.ResourceToWeightMap, 0) + resourceToWeightMap := make(noderesources.ResourceToWeightMap) if len(arguments.Resources) == 0 { resourceToWeightMap = noderesources.DefaultRequestedRatioResources return shape, resourceToWeightMap diff --git a/pkg/scheduler/framework/plugins/legacy_registry_test.go b/pkg/scheduler/framework/plugins/legacy_registry_test.go index 71002f3896d..db01be40ecc 100644 --- a/pkg/scheduler/framework/plugins/legacy_registry_test.go +++ b/pkg/scheduler/framework/plugins/legacy_registry_test.go @@ -1,5 +1,5 @@ /* -Copyright 2015 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -81,22 +81,22 @@ func produceConfig(keys []string, producersMap map[string]ConfigProducer, args C var plugins config.Plugins var pluginConfig []config.PluginConfig for _, k := range keys { - producer, exist := producersMap[k] + p, exist := producersMap[k] if !exist { return nil, nil, fmt.Errorf("finding key %q", k) } - p, pc := producer(args) - plugins.Append(&p) - pluginConfig = append(pluginConfig, pc...) + pl, plc := p(args) + plugins.Append(&pl) + pluginConfig = append(pluginConfig, plc...) } return &plugins, pluginConfig, nil } func TestRegisterConfigProducers(t *testing.T) { - registry := newConfigProducerRegistry() + registry := NewLegacyRegistry() testPredicateName1 := "testPredicate1" testFilterName1 := "testFilter1" - registry.RegisterPredicate(testPredicateName1, + registry.registerPredicateConfigProducer(testPredicateName1, func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, testFilterName1, nil) return @@ -104,7 +104,7 @@ func TestRegisterConfigProducers(t *testing.T) { testPredicateName2 := "testPredicate2" testFilterName2 := "testFilter2" - registry.RegisterPredicate(testPredicateName2, + registry.registerPredicateConfigProducer(testPredicateName2, func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, testFilterName2, nil) return @@ -112,7 +112,7 @@ func TestRegisterConfigProducers(t *testing.T) { testPriorityName1 := "testPriority1" testScoreName1 := "testScore1" - registry.RegisterPriority(testPriorityName1, + registry.registerPriorityConfigProducer(testPriorityName1, func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Score = appendToPluginSet(plugins.Score, testScoreName1, &args.Weight) return @@ -120,7 +120,7 @@ func TestRegisterConfigProducers(t *testing.T) { testPriorityName2 := "testPriority2" testScoreName2 := "testScore2" - registry.RegisterPriority(testPriorityName2, + registry.registerPriorityConfigProducer(testPriorityName2, func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Score = appendToPluginSet(plugins.Score, testScoreName2, &args.Weight) return diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index e64c36fbf3b..22666b1a276 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -120,10 +120,6 @@ type Scheduler struct { SchedulingQueue internalqueue.SchedulingQueue scheduledPodsHasSynced func() bool - - // The final configuration of the framework. - Plugins *schedulerapi.Plugins - PluginConfig []schedulerapi.PluginConfig } // Cache returns the cache in scheduler for test to check the data in scheduler. @@ -140,12 +136,11 @@ type schedulerOptions struct { bindTimeoutSeconds int64 podInitialBackoffSeconds int64 podMaxBackoffSeconds int64 - // Contains all in-tree plugins. - frameworkInTreeRegistry framework.Registry - // This registry contains out of tree plugins to be merged with the in-tree registry. + // Contains out-of-tree plugins to be merged with the in-tree registry. frameworkOutOfTreeRegistry framework.Registry - frameworkPlugins *schedulerapi.Plugins - frameworkPluginConfig []schedulerapi.PluginConfig + // Plugins and PluginConfig set from ComponentConfig. + frameworkPlugins *schedulerapi.Plugins + frameworkPluginConfig []schedulerapi.PluginConfig } // Option configures a Scheduler @@ -193,13 +188,6 @@ func WithBindTimeoutSeconds(bindTimeoutSeconds int64) Option { } } -// WithFrameworkInTreeRegistry sets the framework's in-tree registry. This is only used in integration tests. -func WithFrameworkInTreeRegistry(registry framework.Registry) Option { - return func(o *schedulerOptions) { - o.frameworkInTreeRegistry = registry - } -} - // WithFrameworkOutOfTreeRegistry sets the registry for out-of-tree plugins. Those plugins // will be appended to the default registry. func WithFrameworkOutOfTreeRegistry(registry framework.Registry) Option { @@ -247,8 +235,6 @@ var defaultSchedulerOptions = schedulerOptions{ bindTimeoutSeconds: BindTimeoutSeconds, podInitialBackoffSeconds: int64(internalqueue.DefaultPodInitialBackoffDuration.Seconds()), podMaxBackoffSeconds: int64(internalqueue.DefaultPodMaxBackoffDuration.Seconds()), - frameworkPlugins: nil, - frameworkPluginConfig: nil, } // New returns a Scheduler @@ -280,12 +266,9 @@ func New(client clientset.Interface, time.Duration(options.bindTimeoutSeconds)*time.Second, ) - registry := options.frameworkInTreeRegistry - if registry == nil { - registry = frameworkplugins.NewInTreeRegistry(&frameworkplugins.RegistryArgs{ - VolumeBinder: volumeBinder, - }) - } + registry := frameworkplugins.NewInTreeRegistry(&frameworkplugins.RegistryArgs{ + VolumeBinder: volumeBinder, + }) if err := registry.Merge(options.frameworkOutOfTreeRegistry); err != nil { return nil, err } @@ -317,7 +300,7 @@ func New(client clientset.Interface, switch { case source.Provider != nil: // Create the config from a named algorithm provider. - sc, err := configurator.CreateFromProvider(*source.Provider) + sc, err := configurator.createFromProvider(*source.Provider) if err != nil { return nil, fmt.Errorf("couldn't create scheduler using provider %q: %v", *source.Provider, err) } @@ -335,7 +318,7 @@ func New(client clientset.Interface, return nil, err } } - sc, err := configurator.CreateFromConfig(*policy) + sc, err := configurator.createFromConfig(*policy) if err != nil { return nil, fmt.Errorf("couldn't create scheduler from policy: %v", err) } diff --git a/test/integration/scheduler/predicates_test.go b/test/integration/scheduler/predicates_test.go index 0d57e533840..cecbc20fa6c 100644 --- a/test/integration/scheduler/predicates_test.go +++ b/test/integration/scheduler/predicates_test.go @@ -28,7 +28,6 @@ import ( utilfeature "k8s.io/apiserver/pkg/util/feature" featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/features" - "k8s.io/kubernetes/pkg/scheduler/algorithmprovider" st "k8s.io/kubernetes/pkg/scheduler/testing" testutils "k8s.io/kubernetes/test/utils" imageutils "k8s.io/kubernetes/test/utils/image" @@ -879,8 +878,6 @@ func TestInterPodAffinity(t *testing.T) { // TestEvenPodsSpreadPredicate verifies that EvenPodsSpread predicate functions well. func TestEvenPodsSpreadPredicate(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.EvenPodsSpread, true)() - // Apply feature gates to enable EvenPodsSpread - defer algorithmprovider.ApplyFeatureGates()() context := initTest(t, "eps-predicate") cs := context.clientSet diff --git a/test/integration/scheduler/priorities_test.go b/test/integration/scheduler/priorities_test.go index 05d4e6705ee..aa5b8711a11 100644 --- a/test/integration/scheduler/priorities_test.go +++ b/test/integration/scheduler/priorities_test.go @@ -28,7 +28,6 @@ import ( utilfeature "k8s.io/apiserver/pkg/util/feature" featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/features" - "k8s.io/kubernetes/pkg/scheduler/algorithmprovider" st "k8s.io/kubernetes/pkg/scheduler/testing" testutils "k8s.io/kubernetes/test/utils" imageutils "k8s.io/kubernetes/test/utils/image" @@ -247,8 +246,6 @@ func makeContainersWithImages(images []string) []v1.Container { // TestEvenPodsSpreadPriority verifies that EvenPodsSpread priority functions well. func TestEvenPodsSpreadPriority(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.EvenPodsSpread, true)() - // Apply feature gates to enable EvenPodsSpread - defer algorithmprovider.ApplyFeatureGates()() context := initTest(t, "eps-priority") cs := context.clientSet diff --git a/test/integration/scheduler/taint_test.go b/test/integration/scheduler/taint_test.go index 535b051baad..f960541d9d5 100644 --- a/test/integration/scheduler/taint_test.go +++ b/test/integration/scheduler/taint_test.go @@ -37,7 +37,6 @@ import ( featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/controller/nodelifecycle" "k8s.io/kubernetes/pkg/features" - "k8s.io/kubernetes/pkg/scheduler/algorithmprovider" "k8s.io/kubernetes/plugin/pkg/admission/defaulttolerationseconds" "k8s.io/kubernetes/plugin/pkg/admission/podtolerationrestriction" pluginapi "k8s.io/kubernetes/plugin/pkg/admission/podtolerationrestriction/apis/podtolerationrestriction" @@ -83,9 +82,6 @@ func TestTaintNodeByCondition(t *testing.T) { admission.SetExternalKubeClientSet(externalClientset) admission.SetExternalKubeInformerFactory(externalInformers) - // Apply feature gates to enable TaintNodesByCondition - defer algorithmprovider.ApplyFeatureGates()() - context = initTestScheduler(t, context, false, nil) defer cleanupTest(t, context) @@ -648,8 +644,6 @@ func TestTaintBasedEvictions(t *testing.T) { // Enable TaintBasedEvictions defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.TaintBasedEvictions, true)() - // ApplyFeatureGates() is called to ensure TaintNodesByCondition related logic is applied/restored properly. - defer algorithmprovider.ApplyFeatureGates()() // Build admission chain handler. podTolerations := podtolerationrestriction.NewPodTolerationsPlugin(&pluginapi.Configuration{})