diff --git a/pkg/scheduler/api/compatibility/compatibility_test.go b/pkg/scheduler/api/compatibility/compatibility_test.go index 2ce24e4a752..e126eaa387b 100644 --- a/pkg/scheduler/api/compatibility/compatibility_test.go +++ b/pkg/scheduler/api/compatibility/compatibility_test.go @@ -36,6 +36,9 @@ import ( ) func TestCompatibility_v1_Scheduler(t *testing.T) { + snapshot := scheduler.RegisteredPredicatesAndPrioritiesSnapshot() + defer scheduler.ApplyPredicatesAndPriorities(snapshot) + // Add serialized versions of scheduler config that exercise available options to ensure compatibility between releases schedulerFiles := map[string]struct { JSON string @@ -44,6 +47,27 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { wantPlugins map[string][]kubeschedulerconfig.Plugin wantExtenders []schedulerapi.ExtenderConfig }{ + // This is a special test for the "composite" predicate "GeneralPredicate". GeneralPredicate is a combination + // of predicates, and here we test that if given, it is mapped to the set of plugins that should be executed. + "GeneralPredicate": { + JSON: `{ + "kind": "Policy", + "apiVersion": "v1", + "predicates": [ + {"name": "GeneralPredicates"} + ], + "priorities": [ + ] + }`, + wantPlugins: map[string][]kubeschedulerconfig.Plugin{ + "FilterPlugin": { + {Name: "NodeResources"}, + {Name: "NodeName"}, + {Name: "NodePorts"}, + {Name: "NodeAffinity"}, + }, + }, + }, // Do not change this JSON after the corresponding release has been tagged. // A failure indicates backwards compatibility with the specified release was broken. "1.0": { @@ -209,7 +233,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {"name": "MaxGCEPDVolumeCount"}, {"name": "MaxAzureDiskVolumeCount"}, {"name": "MatchInterPodAffinity"}, - {"name": "GeneralPredicates"}, {"name": "TestServiceAffinity", "argument": {"serviceAffinity" : {"labels" : ["region"]}}}, {"name": "TestLabelsPresence", "argument": {"labelsPresence" : {"labels" : ["foo"], "presence":true}}} ],"priorities": [ @@ -229,7 +252,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "MaxGCEPDVolumeCount", "MaxAzureDiskVolumeCount", "MatchInterPodAffinity", - "GeneralPredicates", "TestServiceAffinity", "TestLabelsPresence", ), @@ -278,7 +300,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {"name": "MaxGCEPDVolumeCount"}, {"name": "MaxAzureDiskVolumeCount"}, {"name": "MatchInterPodAffinity"}, - {"name": "GeneralPredicates"}, {"name": "TestServiceAffinity", "argument": {"serviceAffinity" : {"labels" : ["region"]}}}, {"name": "TestLabelsPresence", "argument": {"labelsPresence" : {"labels" : ["foo"], "presence":true}}} ],"priorities": [ @@ -301,7 +322,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "MaxGCEPDVolumeCount", "MaxAzureDiskVolumeCount", "MatchInterPodAffinity", - "GeneralPredicates", "TestServiceAffinity", "TestLabelsPresence", ), @@ -351,7 +371,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {"name": "MaxGCEPDVolumeCount"}, {"name": "MaxAzureDiskVolumeCount"}, {"name": "MatchInterPodAffinity"}, - {"name": "GeneralPredicates"}, {"name": "TestServiceAffinity", "argument": {"serviceAffinity" : {"labels" : ["region"]}}}, {"name": "TestLabelsPresence", "argument": {"labelsPresence" : {"labels" : ["foo"], "presence":true}}} ],"priorities": [ @@ -384,7 +403,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "MaxGCEPDVolumeCount", "MaxAzureDiskVolumeCount", "MatchInterPodAffinity", - "GeneralPredicates", "TestServiceAffinity", "TestLabelsPresence", ), @@ -446,7 +464,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {"name": "MaxGCEPDVolumeCount"}, {"name": "MaxAzureDiskVolumeCount"}, {"name": "MatchInterPodAffinity"}, - {"name": "GeneralPredicates"}, {"name": "TestServiceAffinity", "argument": {"serviceAffinity" : {"labels" : ["region"]}}}, {"name": "TestLabelsPresence", "argument": {"labelsPresence" : {"labels" : ["foo"], "presence":true}}} ],"priorities": [ @@ -480,7 +497,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "MaxGCEPDVolumeCount", "MaxAzureDiskVolumeCount", "MatchInterPodAffinity", - "GeneralPredicates", "TestServiceAffinity", "TestLabelsPresence", ), @@ -542,7 +558,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {"name": "MaxGCEPDVolumeCount"}, {"name": "MaxAzureDiskVolumeCount"}, {"name": "MatchInterPodAffinity"}, - {"name": "GeneralPredicates"}, {"name": "CheckVolumeBinding"}, {"name": "TestServiceAffinity", "argument": {"serviceAffinity" : {"labels" : ["region"]}}}, {"name": "TestLabelsPresence", "argument": {"labelsPresence" : {"labels" : ["foo"], "presence":true}}} @@ -577,7 +592,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "MaxGCEPDVolumeCount", "MaxAzureDiskVolumeCount", "MatchInterPodAffinity", - "GeneralPredicates", "TestServiceAffinity", "TestLabelsPresence", ), @@ -642,7 +656,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {"name": "MaxGCEPDVolumeCount"}, {"name": "MaxAzureDiskVolumeCount"}, {"name": "MatchInterPodAffinity"}, - {"name": "GeneralPredicates"}, {"name": "CheckVolumeBinding"}, {"name": "TestServiceAffinity", "argument": {"serviceAffinity" : {"labels" : ["region"]}}}, {"name": "TestLabelsPresence", "argument": {"labelsPresence" : {"labels" : ["foo"], "presence":true}}} @@ -680,7 +693,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "MaxGCEPDVolumeCount", "MaxAzureDiskVolumeCount", "MatchInterPodAffinity", - "GeneralPredicates", "TestServiceAffinity", "TestLabelsPresence", ), @@ -746,7 +758,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {"name": "MaxGCEPDVolumeCount"}, {"name": "MaxAzureDiskVolumeCount"}, {"name": "MatchInterPodAffinity"}, - {"name": "GeneralPredicates"}, {"name": "CheckVolumeBinding"}, {"name": "TestServiceAffinity", "argument": {"serviceAffinity" : {"labels" : ["region"]}}}, {"name": "TestLabelsPresence", "argument": {"labelsPresence" : {"labels" : ["foo"], "presence":true}}} @@ -795,7 +806,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "MaxGCEPDVolumeCount", "MaxAzureDiskVolumeCount", "MatchInterPodAffinity", - "GeneralPredicates", "TestServiceAffinity", "TestLabelsPresence", ), @@ -863,7 +873,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {"name": "MaxAzureDiskVolumeCount"}, {"name": "MaxCSIVolumeCountPred"}, {"name": "MatchInterPodAffinity"}, - {"name": "GeneralPredicates"}, {"name": "CheckVolumeBinding"}, {"name": "TestServiceAffinity", "argument": {"serviceAffinity" : {"labels" : ["region"]}}}, {"name": "TestLabelsPresence", "argument": {"labelsPresence" : {"labels" : ["foo"], "presence":true}}} @@ -912,7 +921,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "MaxGCEPDVolumeCount", "MaxAzureDiskVolumeCount", "MatchInterPodAffinity", - "GeneralPredicates", "TestServiceAffinity", "TestLabelsPresence", ), @@ -980,7 +988,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {"name": "MaxCSIVolumeCountPred"}, {"name": "MaxCinderVolumeCount"}, {"name": "MatchInterPodAffinity"}, - {"name": "GeneralPredicates"}, {"name": "CheckVolumeBinding"}, {"name": "TestServiceAffinity", "argument": {"serviceAffinity" : {"labels" : ["region"]}}}, {"name": "TestLabelsPresence", "argument": {"labelsPresence" : {"labels" : ["foo"], "presence":true}}} @@ -1030,7 +1037,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "MaxAzureDiskVolumeCount", "MaxCinderVolumeCount", "MatchInterPodAffinity", - "GeneralPredicates", "TestServiceAffinity", "TestLabelsPresence", ), @@ -1098,7 +1104,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {"name": "MaxCSIVolumeCountPred"}, {"name": "MaxCinderVolumeCount"}, {"name": "MatchInterPodAffinity"}, - {"name": "GeneralPredicates"}, {"name": "CheckVolumeBinding"}, {"name": "TestServiceAffinity", "argument": {"serviceAffinity" : {"labels" : ["region"]}}}, {"name": "TestLabelsPresence", "argument": {"labelsPresence" : {"labels" : ["foo"], "presence":true}}} @@ -1152,7 +1157,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "MaxAzureDiskVolumeCount", "MaxCinderVolumeCount", "MatchInterPodAffinity", - "GeneralPredicates", "TestServiceAffinity", "TestLabelsPresence", ), @@ -1204,6 +1208,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { seenPredicates := sets.NewString() seenPriorities := sets.NewString() mandatoryPredicates := sets.NewString("CheckNodeCondition") + generalPredicateFilters := []string{"NodeResources", "NodeName", "NodePorts", "NodeAffinity"} filterToPredicateMap := map[string]string{ "TaintToleration": "PodToleratesNodeTaints", "NodeName": "HostName", @@ -1273,6 +1278,9 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { seenPredicates.Insert(filterToPredicateMap[p.Name]) } + if pluginsToStringSet(gotPlugins["FilterPlugin"]).HasAll(generalPredicateFilters...) { + seenPredicates.Insert("GeneralPredicates") + } for _, p := range gotPlugins["ScorePlugin"] { seenPriorities.Insert(scoreToPriorityMap[p.Name]) @@ -1308,3 +1316,11 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { t.Errorf("Registered priorities are missing from compatibility test (add to test stanza for version currently in development): %#v", registeredPriorities.Difference(seenPriorities).List()) } } + +func pluginsToStringSet(plugins []kubeschedulerconfig.Plugin) sets.String { + s := sets.NewString() + for _, p := range plugins { + s.Insert(p.Name) + } + return s +} diff --git a/pkg/scheduler/core/generic_scheduler.go b/pkg/scheduler/core/generic_scheduler.go index c769a461352..a9ca2b49fec 100644 --- a/pkg/scheduler/core/generic_scheduler.go +++ b/pkg/scheduler/core/generic_scheduler.go @@ -646,7 +646,7 @@ func (g *genericScheduler) podFitsOnNode( if err != nil { return false, []predicates.PredicateFailureReason{}, nil, err } - } else if !podsAdded || len(failedPredicates) != 0 { + } else if !podsAdded || len(failedPredicates) != 0 || !status.IsSuccess() { break } diff --git a/pkg/scheduler/framework/plugins/default_registry.go b/pkg/scheduler/framework/plugins/default_registry.go index 177ba7ed231..6a97c463799 100644 --- a/pkg/scheduler/framework/plugins/default_registry.go +++ b/pkg/scheduler/framework/plugins/default_registry.go @@ -109,6 +109,15 @@ func NewDefaultConfigProducerRegistry() *ConfigProducerRegistry { PredicateToConfigProducer: make(map[string]ConfigProducer), PriorityToConfigProducer: make(map[string]ConfigProducer), } + registry.RegisterPredicate(predicates.GeneralPred, + func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + // GeneralPredicate is a combination of predicates. + plugins.Filter = appendToPluginSet(plugins.Filter, noderesources.Name, nil) + plugins.Filter = appendToPluginSet(plugins.Filter, nodename.Name, nil) + plugins.Filter = appendToPluginSet(plugins.Filter, nodeports.Name, nil) + plugins.Filter = appendToPluginSet(plugins.Filter, nodeaffinity.Name, nil) + return + }) registry.RegisterPredicate(predicates.PodToleratesNodeTaintsPred, func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Filter = appendToPluginSet(plugins.Filter, tainttoleration.Name, nil) diff --git a/test/integration/scheduler/scheduler_test.go b/test/integration/scheduler/scheduler_test.go index 6bba62d1b87..7b116b0f159 100644 --- a/test/integration/scheduler/scheduler_test.go +++ b/test/integration/scheduler/scheduler_test.go @@ -129,7 +129,6 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) { "CheckNodeDiskPressure", "CheckNodeMemoryPressure", "CheckNodePIDPressure", - "GeneralPredicates", "MatchInterPodAffinity", "MaxAzureDiskVolumeCount", "MaxEBSVolumeCount", @@ -143,6 +142,10 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) { ), expectedPlugins: map[string][]kubeschedulerconfig.Plugin{ "FilterPlugin": { + {Name: "NodeResources"}, + {Name: "NodeName"}, + {Name: "NodePorts"}, + {Name: "NodeAffinity"}, {Name: "VolumeRestrictions"}, {Name: "TaintToleration"}, {Name: "NodeVolumeLimits"}, @@ -200,7 +203,6 @@ kind: Policy "CheckNodeDiskPressure", "CheckNodeMemoryPressure", "CheckNodePIDPressure", - "GeneralPredicates", "MatchInterPodAffinity", "MaxAzureDiskVolumeCount", "MaxEBSVolumeCount", @@ -214,6 +216,10 @@ kind: Policy ), expectedPlugins: map[string][]kubeschedulerconfig.Plugin{ "FilterPlugin": { + {Name: "NodeResources"}, + {Name: "NodeName"}, + {Name: "NodePorts"}, + {Name: "NodeAffinity"}, {Name: "VolumeRestrictions"}, {Name: "TaintToleration"}, {Name: "NodeVolumeLimits"},