From 2b3df3587e3466eb2734ff0c022ec5e6b8c28c9a Mon Sep 17 00:00:00 2001 From: Abdullah Gharaibeh Date: Wed, 25 Dec 2019 13:36:46 -0500 Subject: [PATCH] Deprecate scheduler predicate and priority factory registration --- pkg/scheduler/BUILD | 1 - pkg/scheduler/algorithm_factory.go | 463 ++++++------------ .../algorithmprovider/defaults/BUILD | 2 - .../algorithmprovider/defaults/defaults.go | 17 +- .../defaults/register_predicates.go | 144 ------ .../defaults/register_priorities.go | 59 --- .../algorithmprovider/plugins_test.go | 46 +- pkg/scheduler/apis/config/testing/BUILD | 1 - .../apis/config/testing/compatibility_test.go | 106 +--- pkg/scheduler/factory.go | 79 ++- pkg/scheduler/factory_test.go | 250 +--------- pkg/scheduler/framework/v1alpha1/framework.go | 6 + pkg/scheduler/scheduler_test.go | 11 - test/integration/scheduler/BUILD | 3 - test/integration/scheduler/framework_test.go | 38 +- test/integration/scheduler/scheduler_test.go | 80 +-- 16 files changed, 247 insertions(+), 1059 deletions(-) delete mode 100644 pkg/scheduler/algorithmprovider/defaults/register_predicates.go diff --git a/pkg/scheduler/BUILD b/pkg/scheduler/BUILD index c2538ecbacd..70453ebe943 100644 --- a/pkg/scheduler/BUILD +++ b/pkg/scheduler/BUILD @@ -110,7 +110,6 @@ go_test( "//staging/src/k8s.io/client-go/testing:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/events:go_default_library", - "//vendor/github.com/google/go-cmp/cmp:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", ], ) diff --git a/pkg/scheduler/algorithm_factory.go b/pkg/scheduler/algorithm_factory.go index ac6a112f684..2a71de38638 100644 --- a/pkg/scheduler/algorithm_factory.go +++ b/pkg/scheduler/algorithm_factory.go @@ -51,196 +51,160 @@ type AlgorithmFactoryArgs struct { // PriorityMetadataProducerFactory produces MetadataProducer from the given args. type PriorityMetadataProducerFactory func(AlgorithmFactoryArgs) priorities.MetadataProducer -// FitPredicateFactory produces a FitPredicate from the given args. -type FitPredicateFactory func(AlgorithmFactoryArgs) predicates.FitPredicate - -// PriorityFunctionFactory produces map & reduce priority functions -// from a given args. -type PriorityFunctionFactory func(AlgorithmFactoryArgs) (priorities.PriorityMapFunction, priorities.PriorityReduceFunction) - -// PriorityConfigFactory produces a PriorityConfig from the given function and weight -type PriorityConfigFactory struct { - MapReduceFunction PriorityFunctionFactory - Weight int64 -} - var ( - schedulerFactoryMutex sync.RWMutex + algorithmRegistry = &AlgorithmRegistry{ + // predicate keys supported for backward compatibility with v1.Policy. + predicateKeys: sets.NewString( + "PodFitsPorts", // This exists for compatibility reasons. + predicates.PodFitsHostPortsPred, + predicates.PodFitsResourcesPred, + predicates.HostNamePred, + predicates.MatchNodeSelectorPred, + predicates.NoVolumeZoneConflictPred, + predicates.MaxEBSVolumeCountPred, + predicates.MaxGCEPDVolumeCountPred, + predicates.MaxAzureDiskVolumeCountPred, + predicates.MaxCSIVolumeCountPred, + predicates.MaxCinderVolumeCountPred, + predicates.MatchInterPodAffinityPred, + predicates.NoDiskConflictPred, + predicates.GeneralPred, + predicates.PodToleratesNodeTaintsPred, + predicates.CheckNodeUnschedulablePred, + predicates.CheckVolumeBindingPred, + ), - // maps that hold registered algorithm types - // TODO(Huang-Wei): remove this. - fitPredicateMap = make(map[string]FitPredicateFactory) - mandatoryFitPredicates = sets.NewString() - priorityFunctionMap = make(map[string]PriorityConfigFactory) - algorithmProviderMap = make(map[string]AlgorithmProviderConfig) + // priority keys to weights, this exist for backward compatibility with v1.Policy. + priorityKeys: map[string]int64{ + priorities.LeastRequestedPriority: 1, + priorities.BalancedResourceAllocation: 1, + priorities.MostRequestedPriority: 1, + priorities.ImageLocalityPriority: 1, + priorities.NodeAffinityPriority: 1, + priorities.SelectorSpreadPriority: 1, + priorities.ServiceSpreadingPriority: 1, + priorities.TaintTolerationPriority: 1, + priorities.InterPodAffinityPriority: 1, + priorities.NodePreferAvoidPodsPriority: 10000, + }, + // MandatoryPredicates the set of keys for predicates that the scheduler will + // be configured with all the time. + mandatoryPredicateKeys: sets.NewString( + predicates.PodToleratesNodeTaintsPred, + predicates.CheckNodeUnschedulablePred, + ), + + algorithmProviders: make(map[string]AlgorithmProviderConfig), + } // Registered metadata producers priorityMetadataProducerFactory PriorityMetadataProducerFactory + + schedulerFactoryMutex sync.RWMutex ) // AlgorithmProviderConfig is used to store the configuration of algorithm providers. type AlgorithmProviderConfig struct { - FitPredicateKeys sets.String - PriorityFunctionKeys sets.String + PredicateKeys sets.String + PriorityKeys sets.String } -// Snapshot is used to store current state of registered predicates and priorities. -type Snapshot struct { - fitPredicateMap map[string]FitPredicateFactory - mandatoryFitPredicates sets.String - priorityFunctionMap map[string]PriorityConfigFactory - algorithmProviderMap map[string]AlgorithmProviderConfig +// AlgorithmRegistry is used to store current state of registered predicates and priorities. +type AlgorithmRegistry struct { + predicateKeys sets.String + priorityKeys map[string]int64 + mandatoryPredicateKeys sets.String + algorithmProviders map[string]AlgorithmProviderConfig } // RegisteredPredicatesAndPrioritiesSnapshot returns a snapshot of current registered predicates and priorities. -func RegisteredPredicatesAndPrioritiesSnapshot() *Snapshot { +func RegisteredPredicatesAndPrioritiesSnapshot() *AlgorithmRegistry { schedulerFactoryMutex.RLock() defer schedulerFactoryMutex.RUnlock() - copy := Snapshot{ - fitPredicateMap: make(map[string]FitPredicateFactory), - mandatoryFitPredicates: sets.NewString(), - priorityFunctionMap: make(map[string]PriorityConfigFactory), - algorithmProviderMap: make(map[string]AlgorithmProviderConfig), + copy := AlgorithmRegistry{ + predicateKeys: sets.NewString(), + mandatoryPredicateKeys: sets.NewString(), + priorityKeys: make(map[string]int64), + algorithmProviders: make(map[string]AlgorithmProviderConfig), } - for k, v := range fitPredicateMap { - copy.fitPredicateMap[k] = v + for k := range algorithmRegistry.predicateKeys { + copy.predicateKeys.Insert(k) } - for k := range mandatoryFitPredicates { - copy.mandatoryFitPredicates[k] = struct{}{} + for k := range algorithmRegistry.mandatoryPredicateKeys { + copy.mandatoryPredicateKeys.Insert(k) } - for k, v := range priorityFunctionMap { - copy.priorityFunctionMap[k] = v + for k, v := range algorithmRegistry.priorityKeys { + copy.priorityKeys[k] = v } - for provider, config := range algorithmProviderMap { + for provider, config := range algorithmRegistry.algorithmProviders { copyPredKeys, copyPrioKeys := sets.NewString(), sets.NewString() - for k := range config.FitPredicateKeys { - copyPredKeys[k] = struct{}{} + for k := range config.PredicateKeys { + copyPredKeys.Insert(k) } - for k := range config.PriorityFunctionKeys { - copyPrioKeys[k] = struct{}{} + for k := range config.PriorityKeys { + copyPrioKeys.Insert(k) } - copy.algorithmProviderMap[provider] = AlgorithmProviderConfig{ - FitPredicateKeys: copyPredKeys, - PriorityFunctionKeys: copyPrioKeys, + copy.algorithmProviders[provider] = AlgorithmProviderConfig{ + PredicateKeys: copyPredKeys, + PriorityKeys: copyPrioKeys, } } return © } // ApplyPredicatesAndPriorities sets state of predicates and priorities to `s`. -func ApplyPredicatesAndPriorities(s *Snapshot) { +func ApplyPredicatesAndPriorities(s *AlgorithmRegistry) { schedulerFactoryMutex.Lock() - fitPredicateMap = s.fitPredicateMap - mandatoryFitPredicates = s.mandatoryFitPredicates - priorityFunctionMap = s.priorityFunctionMap - algorithmProviderMap = s.algorithmProviderMap + algorithmRegistry = s schedulerFactoryMutex.Unlock() } -// RegisterFitPredicate registers a fit predicate with the algorithm +// RegisterPredicate registers a fit predicate with the algorithm // registry. Returns the name with which the predicate was registered. // TODO(Huang-Wei): remove this. -func RegisterFitPredicate(name string, predicate predicates.FitPredicate) string { - return RegisterFitPredicateFactory(name, func(AlgorithmFactoryArgs) predicates.FitPredicate { return predicate }) -} - -// RemoveFitPredicate removes a fit predicate from factory. -func RemoveFitPredicate(name string) { - schedulerFactoryMutex.Lock() - defer schedulerFactoryMutex.Unlock() - - validateAlgorithmNameOrDie(name) - delete(fitPredicateMap, name) - mandatoryFitPredicates.Delete(name) -} - -// RemovePredicateKeyFromAlgoProvider removes a fit predicate key from algorithmProvider. -func RemovePredicateKeyFromAlgoProvider(providerName, key string) error { - schedulerFactoryMutex.Lock() - defer schedulerFactoryMutex.Unlock() - - validateAlgorithmNameOrDie(providerName) - provider, ok := algorithmProviderMap[providerName] - if !ok { - return fmt.Errorf("provider %v is not registered", providerName) - } - provider.FitPredicateKeys.Delete(key) - return nil -} - -// RemovePredicateKeyFromAlgorithmProviderMap removes a fit predicate key from all algorithmProviders which in algorithmProviderMap. -func RemovePredicateKeyFromAlgorithmProviderMap(key string) { - schedulerFactoryMutex.Lock() - defer schedulerFactoryMutex.Unlock() - - for _, provider := range algorithmProviderMap { - provider.FitPredicateKeys.Delete(key) - } -} - -// InsertPredicateKeyToAlgoProvider insert a fit predicate key to algorithmProvider. -func InsertPredicateKeyToAlgoProvider(providerName, key string) error { - schedulerFactoryMutex.Lock() - defer schedulerFactoryMutex.Unlock() - - validateAlgorithmNameOrDie(providerName) - provider, ok := algorithmProviderMap[providerName] - if !ok { - return fmt.Errorf("provider %v is not registered", providerName) - } - provider.FitPredicateKeys.Insert(key) - return nil -} - -// InsertPredicateKeyToAlgorithmProviderMap insert a fit predicate key to all algorithmProviders which in algorithmProviderMap. -func InsertPredicateKeyToAlgorithmProviderMap(key string) { - schedulerFactoryMutex.Lock() - defer schedulerFactoryMutex.Unlock() - - for _, provider := range algorithmProviderMap { - provider.FitPredicateKeys.Insert(key) - } -} - -// InsertPriorityKeyToAlgorithmProviderMap inserts a priority function to all algorithmProviders which are in algorithmProviderMap. -func InsertPriorityKeyToAlgorithmProviderMap(key string) { - schedulerFactoryMutex.Lock() - defer schedulerFactoryMutex.Unlock() - - for _, provider := range algorithmProviderMap { - provider.PriorityFunctionKeys.Insert(key) - } -} - -// RegisterMandatoryFitPredicate registers a fit predicate with the algorithm registry, the predicate is used by -// kubelet, DaemonSet; it is always included in configuration. Returns the name with which the predicate was -// registered. -func RegisterMandatoryFitPredicate(name string, predicate predicates.FitPredicate) string { +func RegisterPredicate(name string) string { schedulerFactoryMutex.Lock() defer schedulerFactoryMutex.Unlock() validateAlgorithmNameOrDie(name) - fitPredicateMap[name] = func(AlgorithmFactoryArgs) predicates.FitPredicate { return predicate } - mandatoryFitPredicates.Insert(name) + algorithmRegistry.predicateKeys.Insert(name) return name } -// RegisterFitPredicateFactory registers a fit predicate factory with the -// algorithm registry. Returns the name with which the predicate was registered. -func RegisterFitPredicateFactory(name string, predicateFactory FitPredicateFactory) string { +// RegisterMandatoryPredicate registers a mandatory predicate. +func RegisterMandatoryPredicate(name string) string { schedulerFactoryMutex.Lock() defer schedulerFactoryMutex.Unlock() validateAlgorithmNameOrDie(name) - fitPredicateMap[name] = predicateFactory + algorithmRegistry.predicateKeys.Insert(name) + algorithmRegistry.mandatoryPredicateKeys.Insert(name) return name } -// RegisterCustomFitPredicate registers a custom fit predicate with the algorithm registry. +// AddPredicateToAlgorithmProviders adds a predicate key to all algorithm providers. +func AddPredicateToAlgorithmProviders(key string) { + schedulerFactoryMutex.Lock() + defer schedulerFactoryMutex.Unlock() + + for _, provider := range algorithmRegistry.algorithmProviders { + provider.PredicateKeys.Insert(key) + } +} + +// AddPriorityToAlgorithmProviders adds a priority key to all algorithm providers. +func AddPriorityToAlgorithmProviders(key string) { + schedulerFactoryMutex.Lock() + defer schedulerFactoryMutex.Unlock() + for _, provider := range algorithmRegistry.algorithmProviders { + provider.PriorityKeys.Insert(key) + } +} + +// RegisterCustomPredicate registers a custom fit predicate with the algorithm registry. // Returns the name, with which the predicate was registered. -func RegisterCustomFitPredicate(policy schedulerapi.PredicatePolicy, pluginArgs *plugins.ConfigProducerArgs) string { - var predicateFactory FitPredicateFactory +func RegisterCustomPredicate(policy schedulerapi.PredicatePolicy, pluginArgs *plugins.ConfigProducerArgs) string { var ok bool - policyName := policy.Name + var predicate string validatePredicateOrDie(policy) @@ -250,7 +214,7 @@ func RegisterCustomFitPredicate(policy schedulerapi.PredicatePolicy, pluginArgs // We use the ServiceAffinity predicate name for all ServiceAffinity custom predicates. // It may get called multiple times but we essentially only register one instance of ServiceAffinity predicate. // This name is then used to find the registered plugin and run the plugin instead of the predicate. - policyName = predicates.CheckServiceAffinityPred + predicate = predicates.CheckServiceAffinityPred // map LabelsPresence policy to ConfigProducerArgs that's used to configure the ServiceAffinity plugin. if pluginArgs.ServiceAffinityArgs == nil { @@ -258,15 +222,11 @@ func RegisterCustomFitPredicate(policy schedulerapi.PredicatePolicy, pluginArgs } pluginArgs.ServiceAffinityArgs.AffinityLabels = append(pluginArgs.ServiceAffinityArgs.AffinityLabels, policy.Argument.ServiceAffinity.Labels...) - - predicateFactory = func(args AlgorithmFactoryArgs) predicates.FitPredicate { - return nil - } } else if policy.Argument.LabelsPresence != nil { // We use the CheckNodeLabelPresencePred predicate name for all kNodeLabel custom predicates. // It may get called multiple times but we essentially only register one instance of NodeLabel predicate. // This name is then used to find the registered plugin and run the plugin instead of the predicate. - policyName = predicates.CheckNodeLabelPresencePred + predicate = predicates.CheckNodeLabelPresencePred // Map LabelPresence policy to ConfigProducerArgs that's used to configure the NodeLabel plugin. if pluginArgs.NodeLabelArgs == nil { @@ -277,29 +237,18 @@ func RegisterCustomFitPredicate(policy schedulerapi.PredicatePolicy, pluginArgs } else { pluginArgs.NodeLabelArgs.AbsentLabels = append(pluginArgs.NodeLabelArgs.AbsentLabels, policy.Argument.LabelsPresence.Labels...) } - predicateFactory = func(_ AlgorithmFactoryArgs) predicates.FitPredicate { - return nil - } } - } else if predicateFactory, ok = fitPredicateMap[policyName]; ok { + } else if _, ok = algorithmRegistry.predicateKeys[policy.Name]; ok { // checking to see if a pre-defined predicate is requested - klog.V(2).Infof("Predicate type %s already registered, reusing.", policyName) - return policyName + klog.V(2).Infof("Predicate type %s already registered, reusing.", policy.Name) + return policy.Name } - if predicateFactory == nil { - klog.Fatalf("Invalid configuration: Predicate type not found for %s", policyName) + if len(predicate) == 0 { + klog.Fatalf("Invalid configuration: Predicate type not found for %s", policy.Name) } - return RegisterFitPredicateFactory(policyName, predicateFactory) -} - -// IsFitPredicateRegistered is useful for testing providers. -func IsFitPredicateRegistered(name string) bool { - schedulerFactoryMutex.RLock() - defer schedulerFactoryMutex.RUnlock() - _, ok := fitPredicateMap[name] - return ok + return predicate } // RegisterPriorityMetadataProducerFactory registers a PriorityMetadataProducerFactory. @@ -309,35 +258,21 @@ func RegisterPriorityMetadataProducerFactory(f PriorityMetadataProducerFactory) priorityMetadataProducerFactory = f } -// RegisterPriorityMapReduceFunction registers a priority function with the algorithm registry. Returns the name, +// RegisterPriority registers a priority function with the algorithm registry. Returns the name, // with which the function was registered. -func RegisterPriorityMapReduceFunction( - name string, - mapFunction priorities.PriorityMapFunction, - reduceFunction priorities.PriorityReduceFunction, - weight int) string { - return RegisterPriorityConfigFactory(name, PriorityConfigFactory{ - MapReduceFunction: func(AlgorithmFactoryArgs) (priorities.PriorityMapFunction, priorities.PriorityReduceFunction) { - return mapFunction, reduceFunction - }, - Weight: int64(weight), - }) -} - -// RegisterPriorityConfigFactory registers a priority config factory with its name. -func RegisterPriorityConfigFactory(name string, pcf PriorityConfigFactory) string { +func RegisterPriority(name string, weight int64) string { schedulerFactoryMutex.Lock() defer schedulerFactoryMutex.Unlock() validateAlgorithmNameOrDie(name) - priorityFunctionMap[name] = pcf + algorithmRegistry.priorityKeys[name] = weight return name } -// RegisterCustomPriorityFunction registers a custom priority function with the algorithm registry. +// RegisterCustomPriority registers a custom priority with the algorithm registry. // Returns the name, with which the priority function was registered. -func RegisterCustomPriorityFunction(policy schedulerapi.PriorityPolicy, configProducerArgs *plugins.ConfigProducerArgs) string { - var pcf *PriorityConfigFactory - name := policy.Name +func RegisterCustomPriority(policy schedulerapi.PriorityPolicy, configProducerArgs *plugins.ConfigProducerArgs) string { + var priority string + var weight int64 validatePriorityOrDie(policy) @@ -348,36 +283,26 @@ func RegisterCustomPriorityFunction(policy schedulerapi.PriorityPolicy, configPr // It may get called multiple times but we essentially only register one instance of // ServiceAffinity priority. // This name is then used to find the registered plugin and run the plugin instead of the priority. - name = serviceaffinity.Name + priority = serviceaffinity.Name if configProducerArgs.ServiceAffinityArgs == nil { configProducerArgs.ServiceAffinityArgs = &serviceaffinity.Args{} } configProducerArgs.ServiceAffinityArgs.AntiAffinityLabelsPreference = append(configProducerArgs.ServiceAffinityArgs.AntiAffinityLabelsPreference, policy.Argument.ServiceAntiAffinity.Label) - weight := policy.Weight + weight = policy.Weight schedulerFactoryMutex.RLock() - if existing, ok := priorityFunctionMap[name]; ok { + if existingWeight, ok := algorithmRegistry.priorityKeys[priority]; ok { // If there are n ServiceAffinity priorities in the policy, the weight for the corresponding // score plugin is n*(weight of each priority). - weight += existing.Weight + weight += existingWeight } schedulerFactoryMutex.RUnlock() - pcf = &PriorityConfigFactory{ - MapReduceFunction: func(args AlgorithmFactoryArgs) (priorities.PriorityMapFunction, priorities.PriorityReduceFunction) { - return priorities.NewServiceAntiAffinityPriority( - args.SharedLister.Pods(), - args.InformerFactory.Core().V1().Services().Lister(), - configProducerArgs.ServiceAffinityArgs.AntiAffinityLabelsPreference, - ) - }, - Weight: weight, - } } else if policy.Argument.LabelPreference != nil { // We use the NodeLabel plugin name for all NodeLabel custom priorities. // It may get called multiple times but we essentially only register one instance of NodeLabel priority. // This name is then used to find the registered plugin and run the plugin instead of the priority. - name = nodelabel.Name + priority = nodelabel.Name if configProducerArgs.NodeLabelArgs == nil { configProducerArgs.NodeLabelArgs = &nodelabel.Args{} } @@ -386,50 +311,36 @@ func RegisterCustomPriorityFunction(policy schedulerapi.PriorityPolicy, configPr } else { configProducerArgs.NodeLabelArgs.AbsentLabelsPreference = append(configProducerArgs.NodeLabelArgs.AbsentLabelsPreference, policy.Argument.LabelPreference.Label) } - weight := policy.Weight + weight = policy.Weight schedulerFactoryMutex.RLock() - if existing, ok := priorityFunctionMap[name]; ok { + if existingWeight, ok := algorithmRegistry.priorityKeys[priority]; ok { // If there are n NodeLabel priority configured in the policy, the weight for the corresponding // priority is n*(weight of each priority in policy). - weight += existing.Weight + weight += existingWeight } schedulerFactoryMutex.RUnlock() - pcf = &PriorityConfigFactory{ - MapReduceFunction: func(_ AlgorithmFactoryArgs) (priorities.PriorityMapFunction, priorities.PriorityReduceFunction) { - return nil, nil - }, - Weight: weight, - } } else if policy.Argument.RequestedToCapacityRatioArguments != nil { scoringFunctionShape, resources := buildScoringFunctionShapeFromRequestedToCapacityRatioArguments(policy.Argument.RequestedToCapacityRatioArguments) configProducerArgs.RequestedToCapacityRatioArgs = &noderesources.RequestedToCapacityRatioArgs{ FunctionShape: scoringFunctionShape, ResourceToWeightMap: resources, } - pcf = &PriorityConfigFactory{ - MapReduceFunction: func(args AlgorithmFactoryArgs) (priorities.PriorityMapFunction, priorities.PriorityReduceFunction) { - p := priorities.RequestedToCapacityRatioResourceAllocationPriority(scoringFunctionShape, resources) - return p.PriorityMap, nil - }, - Weight: policy.Weight, - } // We do not allow specifying the name for custom plugins, see #83472 - name = noderesources.RequestedToCapacityRatioName + priority = noderesources.RequestedToCapacityRatioName + weight = policy.Weight } - } else if existingPcf, ok := priorityFunctionMap[name]; ok { - klog.V(2).Infof("Priority type %s already registered, reusing.", name) + } else if _, ok := algorithmRegistry.priorityKeys[policy.Name]; ok { + klog.V(2).Infof("Priority type %s already registered, reusing.", policy.Name) // set/update the weight based on the policy - pcf = &PriorityConfigFactory{ - MapReduceFunction: existingPcf.MapReduceFunction, - Weight: policy.Weight, - } + priority = policy.Name + weight = policy.Weight } - if pcf == nil { - klog.Fatalf("Invalid configuration: Priority type not found for %s", name) + if len(priority) == 0 { + klog.Fatalf("Invalid configuration: Priority type not found for %s", policy.Name) } - return RegisterPriorityConfigFactory(name, *pcf) + return RegisterPriority(priority, weight) } func buildScoringFunctionShapeFromRequestedToCapacityRatioArguments(arguments *schedulerapi.RequestedToCapacityRatioArguments) (priorities.FunctionShape, priorities.ResourceToWeightMap) { @@ -462,22 +373,14 @@ func buildScoringFunctionShapeFromRequestedToCapacityRatioArguments(arguments *s return shape, resourceToWeightMap } -// IsPriorityFunctionRegistered is useful for testing providers. -func IsPriorityFunctionRegistered(name string) bool { - schedulerFactoryMutex.RLock() - defer schedulerFactoryMutex.RUnlock() - _, ok := priorityFunctionMap[name] - return ok -} - // RegisterAlgorithmProvider registers a new algorithm provider with the algorithm registry. func RegisterAlgorithmProvider(name string, predicateKeys, priorityKeys sets.String) string { schedulerFactoryMutex.Lock() defer schedulerFactoryMutex.Unlock() validateAlgorithmNameOrDie(name) - algorithmProviderMap[name] = AlgorithmProviderConfig{ - FitPredicateKeys: predicateKeys, - PriorityFunctionKeys: priorityKeys, + algorithmRegistry.algorithmProviders[name] = AlgorithmProviderConfig{ + PredicateKeys: predicateKeys, + PriorityKeys: priorityKeys, } return name } @@ -487,7 +390,7 @@ func GetAlgorithmProvider(name string) (*AlgorithmProviderConfig, error) { schedulerFactoryMutex.RLock() defer schedulerFactoryMutex.RUnlock() - provider, ok := algorithmProviderMap[name] + provider, ok := algorithmRegistry.algorithmProviders[name] if !ok { return nil, fmt.Errorf("provider %q is not registered", name) } @@ -495,29 +398,6 @@ func GetAlgorithmProvider(name string) (*AlgorithmProviderConfig, error) { return &provider, nil } -func getFitPredicateFunctions(names sets.String, args AlgorithmFactoryArgs) (map[string]predicates.FitPredicate, error) { - schedulerFactoryMutex.RLock() - defer schedulerFactoryMutex.RUnlock() - - fitPredicates := map[string]predicates.FitPredicate{} - for _, name := range names.List() { - factory, ok := fitPredicateMap[name] - if !ok { - return nil, fmt.Errorf("invalid predicate name %q specified - no corresponding function found", name) - } - fitPredicates[name] = factory(args) - } - - // Always include mandatory fit predicates. - for name := range mandatoryFitPredicates { - if factory, found := fitPredicateMap[name]; found { - fitPredicates[name] = factory(args) - } - } - - return fitPredicates, nil -} - func getPriorityMetadataProducer(args AlgorithmFactoryArgs) (priorities.MetadataProducer, error) { schedulerFactoryMutex.Lock() defer schedulerFactoryMutex.Unlock() @@ -528,43 +408,6 @@ func getPriorityMetadataProducer(args AlgorithmFactoryArgs) (priorities.Metadata return priorityMetadataProducerFactory(args), nil } -func getPriorityFunctionConfigs(names sets.String, args AlgorithmFactoryArgs) ([]priorities.PriorityConfig, error) { - schedulerFactoryMutex.RLock() - defer schedulerFactoryMutex.RUnlock() - - var configs []priorities.PriorityConfig - for _, name := range names.List() { - factory, ok := priorityFunctionMap[name] - if !ok { - return nil, fmt.Errorf("invalid priority name %s specified - no corresponding function found", name) - } - mapFunction, reduceFunction := factory.MapReduceFunction(args) - configs = append(configs, priorities.PriorityConfig{ - Name: name, - Map: mapFunction, - Reduce: reduceFunction, - Weight: factory.Weight, - }) - } - if err := validateSelectedConfigs(configs); err != nil { - return nil, err - } - return configs, nil -} - -// validateSelectedConfigs validates the config weights to avoid the overflow. -func validateSelectedConfigs(configs []priorities.PriorityConfig) error { - var totalPriority int64 - for _, config := range configs { - // Checks totalPriority against MaxTotalScore to avoid overflow - if config.Weight*framework.MaxNodeScore > framework.MaxTotalScore-totalPriority { - return fmt.Errorf("total priority of priority functions has overflown") - } - totalPriority += config.Weight * framework.MaxNodeScore - } - return nil -} - var validName = regexp.MustCompile("^[a-zA-Z0-9]([-a-zA-Z0-9]*[a-zA-Z0-9])$") func validateAlgorithmNameOrDie(name string) { @@ -606,34 +449,10 @@ func validatePriorityOrDie(priority schedulerapi.PriorityPolicy) { } } -// ListRegisteredFitPredicates returns the registered fit predicates. -func ListRegisteredFitPredicates() []string { - schedulerFactoryMutex.RLock() - defer schedulerFactoryMutex.RUnlock() - - var names []string - for name := range fitPredicateMap { - names = append(names, name) - } - return names -} - -// ListRegisteredPriorityFunctions returns the registered priority functions. -func ListRegisteredPriorityFunctions() []string { - schedulerFactoryMutex.RLock() - defer schedulerFactoryMutex.RUnlock() - - var names []string - for name := range priorityFunctionMap { - names = append(names, name) - } - return names -} - // ListAlgorithmProviders is called when listing all available algorithm providers in `kube-scheduler --help` func ListAlgorithmProviders() string { var availableAlgorithmProviders []string - for name := range algorithmProviderMap { + for name := range algorithmRegistry.algorithmProviders { availableAlgorithmProviders = append(availableAlgorithmProviders, name) } sort.Strings(availableAlgorithmProviders) diff --git a/pkg/scheduler/algorithmprovider/defaults/BUILD b/pkg/scheduler/algorithmprovider/defaults/BUILD index 8d2f49a1555..39a577a7142 100644 --- a/pkg/scheduler/algorithmprovider/defaults/BUILD +++ b/pkg/scheduler/algorithmprovider/defaults/BUILD @@ -10,14 +10,12 @@ go_library( name = "go_default_library", srcs = [ "defaults.go", - "register_predicates.go", "register_priorities.go", ], importpath = "k8s.io/kubernetes/pkg/scheduler/algorithmprovider/defaults", deps = [ "//pkg/features:go_default_library", "//pkg/scheduler:go_default_library", - "//pkg/scheduler/algorithm:go_default_library", "//pkg/scheduler/algorithm/predicates:go_default_library", "//pkg/scheduler/algorithm/priorities:go_default_library", "//pkg/scheduler/apis/config:go_default_library", diff --git a/pkg/scheduler/algorithmprovider/defaults/defaults.go b/pkg/scheduler/algorithmprovider/defaults/defaults.go index 8f6d0f665f9..002c256deec 100644 --- a/pkg/scheduler/algorithmprovider/defaults/defaults.go +++ b/pkg/scheduler/algorithmprovider/defaults/defaults.go @@ -66,24 +66,19 @@ func ApplyFeatureGates() (restore func()) { if utilfeature.DefaultFeatureGate.Enabled(features.EvenPodsSpread) { klog.Infof("Registering EvenPodsSpread predicate and priority function") // register predicate - scheduler.InsertPredicateKeyToAlgorithmProviderMap(predicates.EvenPodsSpreadPred) - scheduler.RegisterFitPredicate(predicates.EvenPodsSpreadPred, predicates.EvenPodsSpreadPredicate) + scheduler.AddPredicateToAlgorithmProviders(predicates.EvenPodsSpreadPred) + scheduler.RegisterPredicate(predicates.EvenPodsSpreadPred) // register priority - scheduler.InsertPriorityKeyToAlgorithmProviderMap(priorities.EvenPodsSpreadPriority) - scheduler.RegisterPriorityMapReduceFunction( - priorities.EvenPodsSpreadPriority, - priorities.CalculateEvenPodsSpreadPriorityMap, - priorities.CalculateEvenPodsSpreadPriorityReduce, - 1, - ) + scheduler.AddPriorityToAlgorithmProviders(priorities.EvenPodsSpreadPriority) + scheduler.RegisterPriority(priorities.EvenPodsSpreadPriority, 1) } // Prioritizes nodes that satisfy pod's resource limits if utilfeature.DefaultFeatureGate.Enabled(features.ResourceLimitsPriorityFunction) { klog.Infof("Registering resourcelimits priority function") - scheduler.RegisterPriorityMapReduceFunction(priorities.ResourceLimitsPriority, nil, nil, 1) + scheduler.RegisterPriority(priorities.ResourceLimitsPriority, 1) // Register the priority function to specific provider too. - scheduler.InsertPriorityKeyToAlgorithmProviderMap(scheduler.RegisterPriorityMapReduceFunction(priorities.ResourceLimitsPriority, nil, nil, 1)) + scheduler.AddPriorityToAlgorithmProviders(priorities.ResourceLimitsPriority) } restore = func() { diff --git a/pkg/scheduler/algorithmprovider/defaults/register_predicates.go b/pkg/scheduler/algorithmprovider/defaults/register_predicates.go deleted file mode 100644 index 02012e4fc0a..00000000000 --- a/pkg/scheduler/algorithmprovider/defaults/register_predicates.go +++ /dev/null @@ -1,144 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package defaults - -import ( - "k8s.io/kubernetes/pkg/scheduler" - "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" -) - -func init() { - // IMPORTANT NOTES for predicate developers: - // Registers predicates and priorities that are not enabled by default, but user can pick when creating their - // own set of priorities/predicates. - - // PodFitsPorts has been replaced by PodFitsHostPorts for better user understanding. - // For backwards compatibility with 1.0, PodFitsPorts is registered as well. - scheduler.RegisterFitPredicate("PodFitsPorts", predicates.PodFitsHostPorts) - // Fit is defined based on the absence of port conflicts. - // This predicate is actually a default predicate, because it is invoked from - // predicates.GeneralPredicates() - scheduler.RegisterFitPredicate(predicates.PodFitsHostPortsPred, predicates.PodFitsHostPorts) - // Fit is determined by resource availability. - // This predicate is actually a default predicate, because it is invoked from - // predicates.GeneralPredicates() - scheduler.RegisterFitPredicate(predicates.PodFitsResourcesPred, predicates.PodFitsResources) - // Fit is determined by the presence of the Host parameter and a string match - // This predicate is actually a default predicate, because it is invoked from - // predicates.GeneralPredicates() - scheduler.RegisterFitPredicate(predicates.HostNamePred, predicates.PodFitsHost) - // Fit is determined by node selector query. - scheduler.RegisterFitPredicate(predicates.MatchNodeSelectorPred, predicates.PodMatchNodeSelector) - - // Fit is determined by volume zone requirements. - scheduler.RegisterFitPredicateFactory( - predicates.NoVolumeZoneConflictPred, - func(args scheduler.AlgorithmFactoryArgs) predicates.FitPredicate { - pvLister := args.InformerFactory.Core().V1().PersistentVolumes().Lister() - pvcLister := args.InformerFactory.Core().V1().PersistentVolumeClaims().Lister() - storageClassLister := args.InformerFactory.Storage().V1().StorageClasses().Lister() - return predicates.NewVolumeZonePredicate(pvLister, pvcLister, storageClassLister) - }, - ) - // Fit is determined by whether or not there would be too many AWS EBS volumes attached to the node - scheduler.RegisterFitPredicateFactory( - predicates.MaxEBSVolumeCountPred, - func(args scheduler.AlgorithmFactoryArgs) predicates.FitPredicate { - csiNodeLister := scheduler.GetCSINodeLister(args.InformerFactory) - pvLister := args.InformerFactory.Core().V1().PersistentVolumes().Lister() - pvcLister := args.InformerFactory.Core().V1().PersistentVolumeClaims().Lister() - storageClassLister := args.InformerFactory.Storage().V1().StorageClasses().Lister() - return predicates.NewMaxPDVolumeCountPredicate(predicates.EBSVolumeFilterType, csiNodeLister, storageClassLister, pvLister, pvcLister) - }, - ) - // Fit is determined by whether or not there would be too many GCE PD volumes attached to the node - scheduler.RegisterFitPredicateFactory( - predicates.MaxGCEPDVolumeCountPred, - func(args scheduler.AlgorithmFactoryArgs) predicates.FitPredicate { - csiNodeLister := scheduler.GetCSINodeLister(args.InformerFactory) - pvLister := args.InformerFactory.Core().V1().PersistentVolumes().Lister() - pvcLister := args.InformerFactory.Core().V1().PersistentVolumeClaims().Lister() - storageClassLister := args.InformerFactory.Storage().V1().StorageClasses().Lister() - return predicates.NewMaxPDVolumeCountPredicate(predicates.GCEPDVolumeFilterType, csiNodeLister, storageClassLister, pvLister, pvcLister) - }, - ) - // Fit is determined by whether or not there would be too many Azure Disk volumes attached to the node - scheduler.RegisterFitPredicateFactory( - predicates.MaxAzureDiskVolumeCountPred, - func(args scheduler.AlgorithmFactoryArgs) predicates.FitPredicate { - csiNodeLister := scheduler.GetCSINodeLister(args.InformerFactory) - pvLister := args.InformerFactory.Core().V1().PersistentVolumes().Lister() - pvcLister := args.InformerFactory.Core().V1().PersistentVolumeClaims().Lister() - storageClassLister := args.InformerFactory.Storage().V1().StorageClasses().Lister() - return predicates.NewMaxPDVolumeCountPredicate(predicates.AzureDiskVolumeFilterType, csiNodeLister, storageClassLister, pvLister, pvcLister) - }, - ) - scheduler.RegisterFitPredicateFactory( - predicates.MaxCSIVolumeCountPred, - func(args scheduler.AlgorithmFactoryArgs) predicates.FitPredicate { - csiNodeLister := scheduler.GetCSINodeLister(args.InformerFactory) - pvLister := args.InformerFactory.Core().V1().PersistentVolumes().Lister() - pvcLister := args.InformerFactory.Core().V1().PersistentVolumeClaims().Lister() - storageClassLister := args.InformerFactory.Storage().V1().StorageClasses().Lister() - return predicates.NewCSIMaxVolumeLimitPredicate(csiNodeLister, pvLister, pvcLister, storageClassLister) - }, - ) - scheduler.RegisterFitPredicateFactory( - predicates.MaxCinderVolumeCountPred, - func(args scheduler.AlgorithmFactoryArgs) predicates.FitPredicate { - csiNodeLister := scheduler.GetCSINodeLister(args.InformerFactory) - pvLister := args.InformerFactory.Core().V1().PersistentVolumes().Lister() - pvcLister := args.InformerFactory.Core().V1().PersistentVolumeClaims().Lister() - storageClassLister := args.InformerFactory.Storage().V1().StorageClasses().Lister() - return predicates.NewMaxPDVolumeCountPredicate(predicates.CinderVolumeFilterType, csiNodeLister, storageClassLister, pvLister, pvcLister) - }, - ) - - // Fit is determined by inter-pod affinity. - scheduler.RegisterFitPredicateFactory( - predicates.MatchInterPodAffinityPred, - func(args scheduler.AlgorithmFactoryArgs) predicates.FitPredicate { - return nil - }, - ) - - // Fit is determined by non-conflicting disk volumes. - scheduler.RegisterFitPredicateFactory( - predicates.NoDiskConflictPred, - func(args scheduler.AlgorithmFactoryArgs) predicates.FitPredicate { - return nil - }, - ) - - // GeneralPredicates are the predicates that are enforced by all Kubernetes components - // (e.g. kubelet and all schedulers) - scheduler.RegisterFitPredicate(predicates.GeneralPred, predicates.GeneralPredicates) - - // Fit is determined based on whether a pod can tolerate all of the node's taints - scheduler.RegisterMandatoryFitPredicate(predicates.PodToleratesNodeTaintsPred, predicates.PodToleratesNodeTaints) - - // Fit is determined based on whether a pod can tolerate unschedulable of node - scheduler.RegisterMandatoryFitPredicate(predicates.CheckNodeUnschedulablePred, predicates.CheckNodeUnschedulablePredicate) - - // Fit is determined by volume topology requirements. - scheduler.RegisterFitPredicateFactory( - predicates.CheckVolumeBindingPred, - func(args scheduler.AlgorithmFactoryArgs) predicates.FitPredicate { - return nil - }, - ) -} diff --git a/pkg/scheduler/algorithmprovider/defaults/register_priorities.go b/pkg/scheduler/algorithmprovider/defaults/register_priorities.go index 6e9888282cb..811a252075a 100644 --- a/pkg/scheduler/algorithmprovider/defaults/register_priorities.go +++ b/pkg/scheduler/algorithmprovider/defaults/register_priorities.go @@ -18,7 +18,6 @@ package defaults import ( "k8s.io/kubernetes/pkg/scheduler" - "k8s.io/kubernetes/pkg/scheduler/algorithm" "k8s.io/kubernetes/pkg/scheduler/algorithm/priorities" ) @@ -32,62 +31,4 @@ func init() { statefulSetLister := args.InformerFactory.Apps().V1().StatefulSets().Lister() return priorities.NewMetadataFactory(serviceLister, controllerLister, replicaSetLister, statefulSetLister, args.HardPodAffinitySymmetricWeight) }) - - // ServiceSpreadingPriority is a priority config factory that spreads pods by minimizing - // the number of pods (belonging to the same service) on the same node. - // Register the factory so that it's available, but do not include it as part of the default priorities - // Largely replaced by "SelectorSpreadPriority", but registered for backward compatibility with 1.0 - scheduler.RegisterPriorityConfigFactory( - priorities.ServiceSpreadingPriority, - scheduler.PriorityConfigFactory{ - MapReduceFunction: func(args scheduler.AlgorithmFactoryArgs) (priorities.PriorityMapFunction, priorities.PriorityReduceFunction) { - serviceLister := args.InformerFactory.Core().V1().Services().Lister() - return priorities.NewSelectorSpreadPriority(serviceLister, algorithm.EmptyControllerLister{}, algorithm.EmptyReplicaSetLister{}, algorithm.EmptyStatefulSetLister{}) - }, - Weight: 1, - }, - ) - // Optional, cluster-autoscaler friendly priority function - give used nodes higher priority. - scheduler.RegisterPriorityMapReduceFunction(priorities.MostRequestedPriority, priorities.MostRequestedPriorityMap, nil, 1) - scheduler.RegisterPriorityMapReduceFunction( - priorities.RequestedToCapacityRatioPriority, - priorities.RequestedToCapacityRatioResourceAllocationPriorityDefault().PriorityMap, - nil, - 1) - // spreads pods by minimizing the number of pods (belonging to the same service or replication controller) on the same node. - scheduler.RegisterPriorityConfigFactory( - priorities.SelectorSpreadPriority, - scheduler.PriorityConfigFactory{ - MapReduceFunction: func(args scheduler.AlgorithmFactoryArgs) (priorities.PriorityMapFunction, priorities.PriorityReduceFunction) { - serviceLister := args.InformerFactory.Core().V1().Services().Lister() - controllerLister := args.InformerFactory.Core().V1().ReplicationControllers().Lister() - replicaSetLister := args.InformerFactory.Apps().V1().ReplicaSets().Lister() - statefulSetLister := args.InformerFactory.Apps().V1().StatefulSets().Lister() - return priorities.NewSelectorSpreadPriority(serviceLister, controllerLister, replicaSetLister, statefulSetLister) - }, - Weight: 1, - }, - ) - // pods should be placed in the same topological domain (e.g. same node, same rack, same zone, same power domain, etc.) - // as some other pods, or, conversely, should not be placed in the same topological domain as some other pods. - scheduler.RegisterPriorityMapReduceFunction(priorities.InterPodAffinityPriority, nil, nil, 1) - - // Prioritize nodes by least requested utilization. - scheduler.RegisterPriorityMapReduceFunction(priorities.LeastRequestedPriority, priorities.LeastRequestedPriorityMap, nil, 1) - - // Prioritizes nodes to help achieve balanced resource usage - scheduler.RegisterPriorityMapReduceFunction(priorities.BalancedResourceAllocation, priorities.BalancedResourceAllocationMap, nil, 1) - - // Set this weight large enough to override all other priority functions. - // TODO: Figure out a better way to do this, maybe at same time as fixing #24720. - scheduler.RegisterPriorityMapReduceFunction(priorities.NodePreferAvoidPodsPriority, nil, nil, 10000) - - // Prioritizes nodes that have labels matching NodeAffinity - scheduler.RegisterPriorityMapReduceFunction(priorities.NodeAffinityPriority, nil, nil, 1) - - // Prioritizes nodes that marked with taint which pod can tolerate. - scheduler.RegisterPriorityMapReduceFunction(priorities.TaintTolerationPriority, nil, nil, 1) - - // ImageLocalityPriority prioritizes nodes that have images requested by the pod present. - scheduler.RegisterPriorityMapReduceFunction(priorities.ImageLocalityPriority, nil, nil, 1) } diff --git a/pkg/scheduler/algorithmprovider/plugins_test.go b/pkg/scheduler/algorithmprovider/plugins_test.go index 6579607bd94..039107fe0a0 100644 --- a/pkg/scheduler/algorithmprovider/plugins_test.go +++ b/pkg/scheduler/algorithmprovider/plugins_test.go @@ -17,7 +17,6 @@ limitations under the License. package algorithmprovider import ( - "fmt" "testing" "k8s.io/kubernetes/pkg/scheduler" @@ -30,47 +29,6 @@ var ( } ) -func TestDefaultConfigExists(t *testing.T) { - p, err := scheduler.GetAlgorithmProvider(schedulerapi.SchedulerDefaultProviderName) - if err != nil { - t.Errorf("error retrieving default provider: %v", err) - } - if p == nil { - t.Error("algorithm provider config should not be nil") - } - if len(p.FitPredicateKeys) == 0 { - t.Error("default algorithm provider shouldn't have 0 fit predicates") - } -} - -func TestAlgorithmProviders(t *testing.T) { - for _, pn := range algorithmProviderNames { - t.Run(pn, func(t *testing.T) { - p, err := scheduler.GetAlgorithmProvider(pn) - if err != nil { - t.Fatalf("error retrieving provider: %v", err) - } - if len(p.PriorityFunctionKeys) == 0 { - t.Errorf("algorithm provider shouldn't have 0 priority functions") - } - for _, pf := range p.PriorityFunctionKeys.List() { - t.Run(fmt.Sprintf("priorityfunction/%s", pf), func(t *testing.T) { - if !scheduler.IsPriorityFunctionRegistered(pf) { - t.Errorf("priority function is not registered but is used in the algorithm provider") - } - }) - } - for _, fp := range p.FitPredicateKeys.List() { - t.Run(fmt.Sprintf("fitpredicate/%s", fp), func(t *testing.T) { - if !scheduler.IsFitPredicateRegistered(fp) { - t.Errorf("fit predicate is not registered but is used in the algorithm provider") - } - }) - } - }) - } -} - func TestApplyFeatureGates(t *testing.T) { for _, pn := range algorithmProviderNames { t.Run(pn, func(t *testing.T) { @@ -79,7 +37,7 @@ func TestApplyFeatureGates(t *testing.T) { t.Fatalf("Error retrieving provider: %v", err) } - if !p.FitPredicateKeys.Has("PodToleratesNodeTaints") { + if !p.PredicateKeys.Has("PodToleratesNodeTaints") { t.Fatalf("Failed to find predicate: 'PodToleratesNodeTaints'") } }) @@ -94,7 +52,7 @@ func TestApplyFeatureGates(t *testing.T) { t.Fatalf("Error retrieving '%s' provider: %v", pn, err) } - if !p.FitPredicateKeys.Has("PodToleratesNodeTaints") { + if !p.PredicateKeys.Has("PodToleratesNodeTaints") { t.Fatalf("Failed to find predicate: 'PodToleratesNodeTaints'") } }) diff --git a/pkg/scheduler/apis/config/testing/BUILD b/pkg/scheduler/apis/config/testing/BUILD index 3b75554372c..4e70a631935 100644 --- a/pkg/scheduler/apis/config/testing/BUILD +++ b/pkg/scheduler/apis/config/testing/BUILD @@ -18,7 +18,6 @@ go_test( "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", diff --git a/pkg/scheduler/apis/config/testing/compatibility_test.go b/pkg/scheduler/apis/config/testing/compatibility_test.go index a9f9ac0f737..b7b8f962c11 100644 --- a/pkg/scheduler/apis/config/testing/compatibility_test.go +++ b/pkg/scheduler/apis/config/testing/compatibility_test.go @@ -23,7 +23,6 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" @@ -39,12 +38,11 @@ import ( ) type testCase struct { - name string - JSON string - featureGates map[featuregate.Feature]bool - wantPredicates sets.String - wantPlugins map[string][]config.Plugin - wantExtenders []config.Extender + name string + JSON string + featureGates map[featuregate.Feature]bool + wantPlugins map[string][]config.Plugin + wantExtenders []config.Extender } func TestCompatibility_v1_Scheduler(t *testing.T) { @@ -117,16 +115,15 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {"name": "TestLabelPreference", "weight": 4, "argument": {"labelPreference": {"label": "bar", "presence":true}}} ] }`, - wantPredicates: sets.NewString( - "PodFitsPorts", - ), wantPlugins: map[string][]config.Plugin{ "PreFilterPlugin": { + {Name: "NodePorts"}, {Name: "NodeResourcesFit"}, {Name: "ServiceAffinity"}, }, "FilterPlugin": { {Name: "NodeUnschedulable"}, + {Name: "NodePorts"}, {Name: "NodeAffinity"}, {Name: "NodeResourcesFit"}, {Name: "VolumeRestrictions"}, @@ -166,7 +163,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {"name": "TestLabelPreference", "weight": 4, "argument": {"labelPreference": {"label": "bar", "presence":true}}} ] }`, - wantPredicates: sets.NewString(), wantPlugins: map[string][]config.Plugin{ "PreFilterPlugin": { {Name: "NodePorts"}, @@ -223,7 +219,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {"name": "TestLabelPreference", "weight": 4, "argument": {"labelPreference": {"label": "bar", "presence":true}}} ] }`, - wantPredicates: sets.NewString(), wantPlugins: map[string][]config.Plugin{ "PreFilterPlugin": { {Name: "NodePorts"}, @@ -289,7 +284,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {"name": "InterPodAffinityPriority", "weight": 2} ] }`, - wantPredicates: sets.NewString(), wantPlugins: map[string][]config.Plugin{ "PreFilterPlugin": { {Name: "NodePorts"}, @@ -363,7 +357,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {"name": "MostRequestedPriority", "weight": 2} ] }`, - wantPredicates: sets.NewString(), wantPlugins: map[string][]config.Plugin{ "PreFilterPlugin": { {Name: "NodePorts"}, @@ -448,7 +441,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "nodeCacheCapable": true }] }`, - wantPredicates: sets.NewString(), wantPlugins: map[string][]config.Plugin{ "PreFilterPlugin": { {Name: "NodePorts"}, @@ -544,7 +536,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "nodeCacheCapable": true }] }`, - wantPredicates: sets.NewString(), wantPlugins: map[string][]config.Plugin{ "PreFilterPlugin": { {Name: "NodePorts"}, @@ -641,7 +632,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "nodeCacheCapable": true }] }`, - wantPredicates: sets.NewString(), wantPlugins: map[string][]config.Plugin{ "PreFilterPlugin": { {Name: "NodePorts"}, @@ -742,7 +732,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "ignorable":true }] }`, - wantPredicates: sets.NewString(), wantPlugins: map[string][]config.Plugin{ "PreFilterPlugin": { {Name: "NodePorts"}, @@ -855,7 +844,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "ignorable":true }] }`, - wantPredicates: sets.NewString(), wantPlugins: map[string][]config.Plugin{ "PreFilterPlugin": { {Name: "NodePorts"}, @@ -970,7 +958,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "ignorable":true }] }`, - wantPredicates: sets.NewString(), wantPlugins: map[string][]config.Plugin{ "PreFilterPlugin": { {Name: "NodePorts"}, @@ -1085,7 +1072,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "ignorable":true }] }`, - wantPredicates: sets.NewString(), wantPlugins: map[string][]config.Plugin{ "PreFilterPlugin": { {Name: "NodePorts"}, @@ -1205,7 +1191,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "ignorable":true }] }`, - wantPredicates: sets.NewString(), wantPlugins: map[string][]config.Plugin{ "PreFilterPlugin": { {Name: "NodePorts"}, @@ -1319,55 +1304,12 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { }, }, } - registeredPredicates := sets.NewString(scheduler.ListRegisteredFitPredicates()...) - seenPredicates := sets.NewString() - seenPriorities := sets.NewString() - mandatoryPredicates := sets.NewString() - generalPredicateFilters := []string{"NodeResourcesFit", "NodeName", "NodePorts", "NodeAffinity"} - filterToPredicateMap := map[string]string{ - "NodeUnschedulable": "CheckNodeUnschedulable", - "TaintToleration": "PodToleratesNodeTaints", - "NodeName": "HostName", - "NodePorts": "PodFitsHostPorts", - "NodeResourcesFit": "PodFitsResources", - "NodeAffinity": "MatchNodeSelector", - "VolumeBinding": "CheckVolumeBinding", - "VolumeRestrictions": "NoDiskConflict", - "VolumeZone": "NoVolumeZoneConflict", - "NodeVolumeLimits": "MaxCSIVolumeCountPred", - "EBSLimits": "MaxEBSVolumeCount", - "GCEPDLimits": "MaxGCEPDVolumeCount", - "AzureDiskLimits": "MaxAzureDiskVolumeCount", - "CinderLimits": "MaxCinderVolumeCount", - "InterPodAffinity": "MatchInterPodAffinity", - "PodTopologySpread": "EvenPodsSpread", - } - scoreToPriorityMap := map[string]string{ - "DefaultPodTopologySpread": "SelectorSpreadPriority", - "ImageLocality": "ImageLocalityPriority", - "InterPodAffinity": "InterPodAffinityPriority", - "NodeAffinity": "NodeAffinityPriority", - "NodePreferAvoidPods": "NodePreferAvoidPodsPriority", - "TaintToleration": "TaintTolerationPriority", - "NodeResourcesLeastAllocated": "LeastRequestedPriority", - "NodeResourcesBalancedAllocation": "BalancedResourceAllocation", - "NodeResourcesMostAllocated": "MostRequestedPriority", - "RequestedToCapacityRatio": "RequestedToCapacityRatioPriority", - "NodeLabel": "TestLabelPreference", - "ServiceAffinity": "TestServiceAntiAffinity", - "ResourceLimitsPriority": "NodeResourceLimits", - } - for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { for feature, value := range tc.featureGates { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, feature, value)() } defer algorithmprovider.ApplyFeatureGates()() - if len(tc.featureGates) > 0 { - // The enabled featuregate can register more predicates - registeredPredicates = registeredPredicates.Union(sets.NewString(scheduler.ListRegisteredFitPredicates()...)) - } policyConfigMap := v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "scheduler-custom-policy-config"}, Data: map[string]string{config.SchedulerPolicyConfigMapKey: tc.JSON}, @@ -1395,28 +1337,8 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { if err != nil { t.Fatalf("Error constructing: %v", err) } - gotPredicates := sets.NewString() - for p := range sched.Algorithm.Predicates() { - gotPredicates.Insert(p) - } - wantPredicates := tc.wantPredicates.Union(mandatoryPredicates) - if !gotPredicates.Equal(wantPredicates) { - t.Errorf("Got predicates %v, want %v", gotPredicates, wantPredicates) - } gotPlugins := sched.Framework.ListPlugins() - for _, p := range gotPlugins["FilterPlugin"] { - seenPredicates.Insert(filterToPredicateMap[p.Name]) - - } - if pluginsToStringSet(gotPlugins["FilterPlugin"]).HasAll(generalPredicateFilters...) { - seenPredicates.Insert("GeneralPredicates") - } - for _, p := range gotPlugins["ScorePlugin"] { - seenPriorities.Insert(scoreToPriorityMap[p.Name]) - - } - if diff := cmp.Diff(tc.wantPlugins, gotPlugins); diff != "" { t.Errorf("unexpected plugins diff (-want, +got): %s", diff) } @@ -1435,20 +1357,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { t.Errorf("Got extender #%d %+v, want %+v", i, gotExtenders[i], wantExtenders[i]) } } - - seenPredicates = seenPredicates.Union(gotPredicates) }) } - - if !seenPredicates.HasAll(registeredPredicates.List()...) { - t.Errorf("Registered predicates are missing from compatibility test (add to test stanza for version currently in development): %#v", registeredPredicates.Difference(seenPredicates).List()) - } -} - -func pluginsToStringSet(plugins []config.Plugin) sets.String { - s := sets.NewString() - for _, p := range plugins { - s.Insert(p.Name) - } - return s } diff --git a/pkg/scheduler/factory.go b/pkg/scheduler/factory.go index 33a0a98b288..b97c63c5c29 100644 --- a/pkg/scheduler/factory.go +++ b/pkg/scheduler/factory.go @@ -126,7 +126,7 @@ func (c *Configurator) CreateFromProvider(providerName string) (*Scheduler, erro if err != nil { return nil, err } - return c.CreateFromKeys(provider.FitPredicateKeys, provider.PriorityFunctionKeys, []algorithm.SchedulerExtender{}) + return c.CreateFromKeys(provider.PredicateKeys, provider.PriorityKeys, []algorithm.SchedulerExtender{}) } // CreateFromConfig creates a scheduler from the configuration file @@ -145,11 +145,11 @@ func (c *Configurator) CreateFromConfig(policy schedulerapi.Policy) (*Scheduler, if err != nil { return nil, err } - predicateKeys = provider.FitPredicateKeys + predicateKeys = provider.PredicateKeys } else { for _, predicate := range policy.Predicates { klog.V(2).Infof("Registering predicate: %s", predicate.Name) - predicateKeys.Insert(RegisterCustomFitPredicate(predicate, c.configProducerArgs)) + predicateKeys.Insert(RegisterCustomPredicate(predicate, c.configProducerArgs)) } } @@ -160,7 +160,7 @@ func (c *Configurator) CreateFromConfig(policy schedulerapi.Policy) (*Scheduler, if err != nil { return nil, err } - priorityKeys = provider.PriorityFunctionKeys + priorityKeys = provider.PriorityKeys } else { for _, priority := range policy.Priorities { if priority.Name == priorities.EqualPriority { @@ -168,7 +168,7 @@ func (c *Configurator) CreateFromConfig(policy schedulerapi.Policy) (*Scheduler, continue } klog.V(2).Infof("Registering priority: %s", priority.Name) - priorityKeys.Insert(RegisterCustomPriorityFunction(priority, c.configProducerArgs)) + priorityKeys.Insert(RegisterCustomPriority(priority, c.configProducerArgs)) } } @@ -225,7 +225,7 @@ func (c *Configurator) CreateFromKeys(predicateKeys, priorityKeys sets.String, e HardPodAffinityWeight: c.hardPodAffinitySymmetricWeight, } - predicateFuncs, pluginsForPredicates, pluginConfigForPredicates, err := c.getPredicateConfigs(predicateKeys) + pluginsForPredicates, pluginConfigForPredicates, err := c.getPredicateConfigs(predicateKeys) if err != nil { return nil, err } @@ -282,7 +282,7 @@ func (c *Configurator) CreateFromKeys(predicateKeys, priorityKeys sets.String, e algo := core.NewGenericScheduler( c.schedulerCache, podQueue, - predicateFuncs, + nil, priorityMetaProducer, c.nodeInfoSnapshot, framework, @@ -327,12 +327,7 @@ func getBinderFunc(client clientset.Interface, extenders []algorithm.SchedulerEx // getPriorityConfigs returns priorities configuration: ones that will run as priorities and ones that will run // as framework plugins. Specifically, a priority will run as a framework plugin if a plugin config producer was // registered for that priority. -func (c *Configurator) getPriorityConfigs(priorityKeys sets.String) (*schedulerapi.Plugins, []schedulerapi.PluginConfig, error) { - allPriorityConfigs, err := getPriorityFunctionConfigs(priorityKeys, c.algorithmFactoryArgs) - if err != nil { - return nil, nil, err - } - +func (c *Configurator) getPriorityConfigs(keys sets.String) (*schedulerapi.Plugins, []schedulerapi.PluginConfig, error) { if c.pluginConfigProducerRegistry == nil { return nil, nil, nil } @@ -340,10 +335,14 @@ func (c *Configurator) getPriorityConfigs(priorityKeys sets.String) (*schedulera var plugins schedulerapi.Plugins var pluginConfig []schedulerapi.PluginConfig frameworkConfigProducers := c.pluginConfigProducerRegistry.PriorityToConfigProducer - for _, p := range allPriorityConfigs { - if producer, exist := frameworkConfigProducers[p.Name]; exist { + for _, p := range keys.List() { + weight, exist := algorithmRegistry.priorityKeys[p] + if !exist { + return nil, nil, fmt.Errorf("priority key %q is not registered", p) + } + if producer, exist := frameworkConfigProducers[p]; exist { args := *c.configProducerArgs - args.Weight = int32(p.Weight) + args.Weight = int32(weight) pl, pc := producer(args) plugins.Append(&pl) pluginConfig = append(pluginConfig, pc...) @@ -357,54 +356,50 @@ func (c *Configurator) getPriorityConfigs(priorityKeys sets.String) (*schedulera // registered for that predicate. // Note that the framework executes plugins according to their order in the Plugins list, and so predicates run as plugins // are added to the Plugins list according to the order specified in predicates.Ordering(). -func (c *Configurator) getPredicateConfigs(predicateKeys sets.String) (map[string]predicates.FitPredicate, *schedulerapi.Plugins, []schedulerapi.PluginConfig, error) { - allFitPredicates, err := getFitPredicateFunctions(predicateKeys, c.algorithmFactoryArgs) - if err != nil { - return nil, nil, nil, err - } - +func (c *Configurator) getPredicateConfigs(keys sets.String) (*schedulerapi.Plugins, []schedulerapi.PluginConfig, error) { if c.pluginConfigProducerRegistry == nil { - return allFitPredicates, nil, nil, nil + // No config producer registry available, so predicates can't be translated to plugins. + return nil, nil, fmt.Errorf("No config producer registry available, can't producer plugins configs for provided predicate keys") } - asPlugins := sets.NewString() - asFitPredicates := make(map[string]predicates.FitPredicate) + allPredicates := keys.Union(algorithmRegistry.mandatoryPredicateKeys) + if allPredicates.Has("PodFitsPorts") { + // For compatibility reasons, "PodFitsPorts" as a key is still supported. + allPredicates.Delete("PodFitsPorts") + allPredicates.Insert(predicates.PodFitsHostPortsPred) + } frameworkConfigProducers := c.pluginConfigProducerRegistry.PredicateToConfigProducer - // First, identify the predicates that will run as actual fit predicates, and ones - // that will run as framework plugins. - for predicateKey := range allFitPredicates { - if _, exist := frameworkConfigProducers[predicateKey]; exist { - asPlugins.Insert(predicateKey) - } else { - asFitPredicates[predicateKey] = allFitPredicates[predicateKey] - } - } - - // Second, create the framework plugin configurations, and place them in the order + // Create the framework plugin configurations, and place them in the order // that the corresponding predicates were supposed to run. var plugins schedulerapi.Plugins var pluginConfig []schedulerapi.PluginConfig for _, predicateKey := range predicates.Ordering() { - if asPlugins.Has(predicateKey) { - producer := frameworkConfigProducers[predicateKey] + if allPredicates.Has(predicateKey) { + producer, exist := frameworkConfigProducers[predicateKey] + if !exist { + return nil, nil, fmt.Errorf("no framework config producer registered for %q", predicateKey) + } p, pc := producer(*c.configProducerArgs) plugins.Append(&p) pluginConfig = append(pluginConfig, pc...) - asPlugins.Delete(predicateKey) + allPredicates.Delete(predicateKey) } } // Third, add the rest in no specific order. - for predicateKey := range asPlugins { - producer := frameworkConfigProducers[predicateKey] + for predicateKey := range allPredicates { + producer, exist := frameworkConfigProducers[predicateKey] + if !exist { + return nil, nil, fmt.Errorf("no framework config producer registered for %q", predicateKey) + } p, pc := producer(*c.configProducerArgs) plugins.Append(&p) pluginConfig = append(pluginConfig, pc...) } - return asFitPredicates, &plugins, pluginConfig, nil + return &plugins, pluginConfig, nil } type podInformer struct { diff --git a/pkg/scheduler/factory_test.go b/pkg/scheduler/factory_test.go index e44d05dc37f..c37c65ffcd5 100644 --- a/pkg/scheduler/factory_test.go +++ b/pkg/scheduler/factory_test.go @@ -22,12 +22,9 @@ import ( "errors" "fmt" "reflect" - "sort" "testing" "time" - "github.com/google/go-cmp/cmp" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -43,7 +40,6 @@ import ( apitesting "k8s.io/kubernetes/pkg/api/testing" kubefeatures "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/algorithm" - "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/apis/config/scheme" extenderv1 "k8s.io/kubernetes/pkg/scheduler/apis/extender/v1" @@ -83,12 +79,6 @@ func TestCreateFromConfig(t *testing.T) { defer close(stopCh) factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight, stopCh) - // Pre-register some predicate and priority functions - RegisterFitPredicate("PredicateOne", PredicateFunc) - RegisterFitPredicate("PredicateTwo", PredicateFunc) - RegisterPriorityMapReduceFunction("PriorityOne", PriorityFunc, nil, 1) - RegisterPriorityMapReduceFunction("PriorityTwo", PriorityFunc, nil, 1) - configData = []byte(`{ "kind" : "Policy", "apiVersion" : "v1", @@ -97,16 +87,16 @@ func TestCreateFromConfig(t *testing.T) { {"name" : "TestZoneAffinity", "argument" : {"serviceAffinity" : {"labels" : ["foo"]}}}, {"name" : "TestRequireZone", "argument" : {"labelsPresence" : {"labels" : ["zone"], "presence" : true}}}, {"name" : "TestNoFooLabel", "argument" : {"labelsPresence" : {"labels" : ["foo"], "presence" : false}}}, - {"name" : "PredicateOne"}, - {"name" : "PredicateTwo"} + {"name" : "PodFitsResources"}, + {"name" : "PodFitsHostPorts"} ], "priorities" : [ {"name" : "RackSpread", "weight" : 3, "argument" : {"serviceAntiAffinity" : {"label" : "rack"}}}, {"name" : "ZoneSpread", "weight" : 3, "argument" : {"serviceAntiAffinity" : {"label" : "zone"}}}, {"name" : "LabelPreference1", "weight" : 3, "argument" : {"labelPreference" : {"label" : "l1", "presence": true}}}, {"name" : "LabelPreference2", "weight" : 3, "argument" : {"labelPreference" : {"label" : "l2", "presence": false}}}, - {"name" : "PriorityOne", "weight" : 2}, - {"name" : "PriorityTwo", "weight" : 1} ] + {"name" : "NodeAffinityPriority", "weight" : 2}, + {"name" : "ImageLocalityPriority", "weight" : 1} ] }`) if err := runtime.DecodeInto(scheme.Codecs.UniversalDecoder(), configData, &policy); err != nil { t.Errorf("Invalid configuration: %v", err) @@ -179,25 +169,19 @@ func TestCreateFromConfigWithHardPodAffinitySymmetricWeight(t *testing.T) { defer close(stopCh) factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight, stopCh) - // Pre-register some predicate and priority functions - RegisterFitPredicate("PredicateOne", PredicateFunc) - RegisterFitPredicate("PredicateTwo", PredicateFunc) - RegisterPriorityMapReduceFunction("PriorityOne", PriorityFunc, nil, 1) - RegisterPriorityMapReduceFunction("PriorityTwo", PriorityFunc, nil, 1) - configData = []byte(`{ "kind" : "Policy", "apiVersion" : "v1", "predicates" : [ {"name" : "TestZoneAffinity", "argument" : {"serviceAffinity" : {"labels" : ["zone"]}}}, {"name" : "TestRequireZone", "argument" : {"labelsPresence" : {"labels" : ["zone"], "presence" : true}}}, - {"name" : "PredicateOne"}, - {"name" : "PredicateTwo"} + {"name" : "PodFitsResources"}, + {"name" : "PodFitsHostPorts"} ], "priorities" : [ {"name" : "RackSpread", "weight" : 3, "argument" : {"serviceAntiAffinity" : {"label" : "rack"}}}, - {"name" : "PriorityOne", "weight" : 2}, - {"name" : "PriorityTwo", "weight" : 1} + {"name" : "NodeAffinityPriority", "weight" : 2}, + {"name" : "ImageLocalityPriority", "weight" : 1} ], "hardPodAffinitySymmetricWeight" : 10 }`) @@ -233,26 +217,12 @@ func TestCreateFromEmptyConfig(t *testing.T) { // The predicate/priority from DefaultProvider will be used. // TODO(Huang-Wei): refactor (or remove) this test along with eliminating 'RegisterFitPredicate()'. func TestCreateFromConfigWithUnspecifiedPredicatesOrPriorities(t *testing.T) { - predicateOne := "PredicateOne" client := fake.NewSimpleClientset() stopCh := make(chan struct{}) defer close(stopCh) factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight, stopCh) - factory.registry.Register(predicateOne, func(_ *runtime.Unknown, fh framework.FrameworkHandle) (framework.Plugin, error) { - return &TestPlugin{name: predicateOne}, nil - }) - factory.pluginConfigProducerRegistry.RegisterPredicate(predicateOne, func(_ frameworkplugins.ConfigProducerArgs) (schedulerapi.Plugins, []schedulerapi.PluginConfig) { - return schedulerapi.Plugins{ - Filter: &schedulerapi.PluginSet{ - Enabled: []schedulerapi.Plugin{{Name: predicateOne}}, - }, - }, nil - }) - RegisterFitPredicate(predicateOne, PredicateFunc) - RegisterPriorityMapReduceFunction("PriorityOne", PriorityFunc, nil, 1) - - RegisterAlgorithmProvider(schedulerapi.SchedulerDefaultProviderName, sets.NewString(predicateOne), sets.NewString("PriorityOne")) + RegisterAlgorithmProvider(schedulerapi.SchedulerDefaultProviderName, sets.NewString("PodFitsResources"), sets.NewString("NodeAffinityPriority")) configData := []byte(`{ "kind" : "Policy", @@ -267,8 +237,8 @@ func TestCreateFromConfigWithUnspecifiedPredicatesOrPriorities(t *testing.T) { if err != nil { t.Fatalf("Failed to create scheduler from configuration: %v", err) } - if !foundPlugin(c.Plugins.Filter.Enabled, predicateOne) { - t.Errorf("Expected predicate PredicateOne from %q", schedulerapi.SchedulerDefaultProviderName) + if !foundPlugin(c.Plugins.Filter.Enabled, "NodeResourcesFit") { + t.Errorf("Expected plugin NodeResourcesFit") } } @@ -281,48 +251,6 @@ func foundPlugin(plugins []schedulerapi.Plugin, name string) bool { return false } -// Test configures a scheduler from a policy that contains empty -// predicate/priority. -// Empty predicate/priority sets will be used. -func TestCreateFromConfigWithEmptyPredicatesOrPriorities(t *testing.T) { - client := fake.NewSimpleClientset() - stopCh := make(chan struct{}) - defer close(stopCh) - factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight, stopCh) - - RegisterFitPredicate("PredicateOne", PredicateFunc) - RegisterPriorityMapReduceFunction("PriorityOne", PriorityFunc, nil, 1) - - RegisterAlgorithmProvider(schedulerapi.SchedulerDefaultProviderName, sets.NewString("PredicateOne"), sets.NewString("PriorityOne")) - - configData := []byte(`{ - "kind" : "Policy", - "apiVersion" : "v1", - "predicates" : [], - "priorities" : [] - }`) - var policy schedulerapi.Policy - if err := runtime.DecodeInto(scheme.Codecs.UniversalDecoder(), configData, &policy); err != nil { - t.Fatalf("Invalid configuration: %v", err) - } - - c, err := factory.CreateFromConfig(policy) - if err != nil { - t.Fatalf("Failed to create scheduler from configuration: %v", err) - } - if len(c.Algorithm.Predicates()) != 0 { - t.Error("Expected empty predicate sets") - } -} - -func PredicateFunc(pod *v1.Pod, meta predicates.Metadata, nodeInfo *schedulernodeinfo.NodeInfo) (bool, []predicates.PredicateFailureReason, error) { - return true, nil, nil -} - -func PriorityFunc(pod *v1.Pod, meta interface{}, nodeInfo *schedulernodeinfo.NodeInfo) (framework.NodeScore, error) { - return framework.NodeScore{}, nil -} - func TestDefaultErrorFunc(t *testing.T) { testPod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "bar"}, @@ -727,159 +655,3 @@ func (t *TestPlugin) ScoreExtensions() framework.ScoreExtensions { func (t *TestPlugin) Filter(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeInfo *schedulernodeinfo.NodeInfo) *framework.Status { return nil } - -// Test configures a scheduler from a policies defined in a file -// It combines some configurable predicate/priorities with some pre-defined ones -func TestCreateWithFrameworkPlugins(t *testing.T) { - var configData []byte - var policy schedulerapi.Policy - - client := fake.NewSimpleClientset() - stopCh := make(chan struct{}) - defer close(stopCh) - - predicateOneName := "PredicateOne" - filterOneName := "FilterOne" - predicateTwoName := "PredicateTwo" - filterTwoName := "FilterTwo" - predicateThreeName := "PredicateThree" - predicateFourName := "PredicateFour" - priorityOneName := "PriorityOne" - scoreOneName := "ScoreOne" - priorityTwoName := "PriorityTwo" - scoreTwoName := "ScoreTwo" - priorityThreeName := "PriorityThree" - - configProducerRegistry := &frameworkplugins.ConfigProducerRegistry{ - PredicateToConfigProducer: make(map[string]frameworkplugins.ConfigProducer), - PriorityToConfigProducer: make(map[string]frameworkplugins.ConfigProducer), - } - configProducerRegistry.RegisterPredicate(predicateOneName, - func(_ frameworkplugins.ConfigProducerArgs) (schedulerapi.Plugins, []schedulerapi.PluginConfig) { - return schedulerapi.Plugins{ - Filter: &schedulerapi.PluginSet{ - Enabled: []schedulerapi.Plugin{ - {Name: filterOneName}}}}, nil - }) - - configProducerRegistry.RegisterPredicate(predicateTwoName, - func(_ frameworkplugins.ConfigProducerArgs) (schedulerapi.Plugins, []schedulerapi.PluginConfig) { - return schedulerapi.Plugins{ - Filter: &schedulerapi.PluginSet{ - Enabled: []schedulerapi.Plugin{ - {Name: filterTwoName}}}}, nil - }) - - configProducerRegistry.RegisterPriority(priorityOneName, - func(args frameworkplugins.ConfigProducerArgs) (schedulerapi.Plugins, []schedulerapi.PluginConfig) { - return schedulerapi.Plugins{ - Score: &schedulerapi.PluginSet{ - Enabled: []schedulerapi.Plugin{ - {Name: scoreOneName, Weight: args.Weight}}}}, nil - }) - - configProducerRegistry.RegisterPriority(priorityTwoName, - func(args frameworkplugins.ConfigProducerArgs) (schedulerapi.Plugins, []schedulerapi.PluginConfig) { - return schedulerapi.Plugins{ - Score: &schedulerapi.PluginSet{ - Enabled: []schedulerapi.Plugin{ - {Name: scoreTwoName, Weight: args.Weight}}}}, nil - }) - - registry := framework.Registry{ - filterOneName: func(_ *runtime.Unknown, fh framework.FrameworkHandle) (framework.Plugin, error) { - return &TestPlugin{name: filterOneName}, nil - }, - filterTwoName: func(_ *runtime.Unknown, fh framework.FrameworkHandle) (framework.Plugin, error) { - return &TestPlugin{name: filterTwoName}, nil - }, - scoreOneName: func(_ *runtime.Unknown, fh framework.FrameworkHandle) (framework.Plugin, error) { - return &TestPlugin{name: scoreOneName}, nil - }, - scoreTwoName: func(_ *runtime.Unknown, fh framework.FrameworkHandle) (framework.Plugin, error) { - return &TestPlugin{name: scoreTwoName}, nil - }, - } - - factory := newConfigFactoryWithFrameworkRegistry( - client, v1.DefaultHardPodAffinitySymmetricWeight, stopCh, registry, configProducerRegistry) - - // Pre-register some predicate and priority functions - RegisterMandatoryFitPredicate(predicateOneName, PredicateFunc) - RegisterFitPredicate(predicateTwoName, PredicateFunc) - RegisterFitPredicate(predicateThreeName, PredicateFunc) - RegisterMandatoryFitPredicate(predicateFourName, PredicateFunc) - RegisterPriorityMapReduceFunction(priorityOneName, PriorityFunc, nil, 1) - RegisterPriorityMapReduceFunction(priorityTwoName, PriorityFunc, nil, 1) - RegisterPriorityMapReduceFunction(priorityThreeName, PriorityFunc, nil, 1) - - configData = []byte(`{ - "kind" : "Policy", - "apiVersion" : "v1", - "predicates" : [ - {"name" : "PredicateOne"}, - {"name" : "PredicateTwo"}, - {"name" : "PredicateThree"}, - {"name" : "PredicateThree"} - ], - "priorities" : [ - {"name" : "PriorityOne", "weight" : 2}, - {"name" : "PriorityTwo", "weight" : 1}, - {"name" : "PriorityThree", "weight" : 1} ] - }`) - if err := runtime.DecodeInto(scheme.Codecs.UniversalDecoder(), configData, &policy); err != nil { - t.Errorf("Invalid configuration: %v", err) - } - - c, err := factory.CreateFromConfig(policy) - if err != nil { - t.Fatalf("creating config: %v", err) - } - - gotPredicates := sets.NewString() - for p := range c.Algorithm.Predicates() { - gotPredicates.Insert(p) - } - wantPredicates := sets.NewString( - predicateThreeName, - predicateFourName, - ) - if diff := cmp.Diff(wantPredicates, gotPredicates); diff != "" { - t.Errorf("unexpected predicates diff (-want, +got): %s", diff) - } - - // Verify the aggregated configuration. - wantPlugins := schedulerapi.Plugins{ - QueueSort: &schedulerapi.PluginSet{}, - PreFilter: &schedulerapi.PluginSet{}, - Filter: &schedulerapi.PluginSet{ - Enabled: []schedulerapi.Plugin{ - {Name: filterOneName}, - {Name: filterTwoName}, - }, - }, - PostFilter: &schedulerapi.PluginSet{}, - Score: &schedulerapi.PluginSet{ - Enabled: []schedulerapi.Plugin{ - {Name: scoreOneName, Weight: 2}, - {Name: scoreTwoName, Weight: 1}, - }, - }, - Reserve: &schedulerapi.PluginSet{}, - Permit: &schedulerapi.PluginSet{}, - PreBind: &schedulerapi.PluginSet{}, - Bind: &schedulerapi.PluginSet{}, - PostBind: &schedulerapi.PluginSet{}, - Unreserve: &schedulerapi.PluginSet{}, - } - - trans := cmp.Transformer("Sort", func(in []schedulerapi.Plugin) []schedulerapi.Plugin { - out := append([]schedulerapi.Plugin(nil), in...) // Copy input to avoid mutating it - sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) - return out - }) - - if diff := cmp.Diff(wantPlugins, c.Plugins, trans); diff != "" { - t.Errorf("unexpected plugin configuration (-want, +got): %s", diff) - } -} diff --git a/pkg/scheduler/framework/v1alpha1/framework.go b/pkg/scheduler/framework/v1alpha1/framework.go index 1c456874ebc..730d908528f 100644 --- a/pkg/scheduler/framework/v1alpha1/framework.go +++ b/pkg/scheduler/framework/v1alpha1/framework.go @@ -197,6 +197,7 @@ func NewFramework(r Registry, plugins *config.Plugins, args []config.PluginConfi } pluginsMap := make(map[string]Plugin) + var totalPriority int64 for name, factory := range r { // initialize only needed plugins. if _, ok := pg[name]; !ok { @@ -215,6 +216,11 @@ func NewFramework(r Registry, plugins *config.Plugins, args []config.PluginConfi if f.pluginNameToWeightMap[name] == 0 { f.pluginNameToWeightMap[name] = 1 } + // Checks totalPriority against MaxTotalScore to avoid overflow + if int64(f.pluginNameToWeightMap[name])*MaxNodeScore > MaxTotalScore-totalPriority { + return nil, fmt.Errorf("total score of Score plugins could overflow") + } + totalPriority += int64(f.pluginNameToWeightMap[name]) * MaxNodeScore } for _, e := range f.getExtensionPoints(plugins) { diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index e70481069d8..7005ac32a9f 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -56,7 +56,6 @@ import ( internalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache" fakecache "k8s.io/kubernetes/pkg/scheduler/internal/cache/fake" internalqueue "k8s.io/kubernetes/pkg/scheduler/internal/queue" - schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" nodeinfosnapshot "k8s.io/kubernetes/pkg/scheduler/nodeinfo/snapshot" st "k8s.io/kubernetes/pkg/scheduler/testing" "k8s.io/kubernetes/pkg/scheduler/volumebinder" @@ -143,10 +142,6 @@ func podWithResources(id, desiredHost string, limits v1.ResourceList, requests v return pod } -func PriorityOne(pod *v1.Pod, meta interface{}, nodeInfo *schedulernodeinfo.NodeInfo) (framework.NodeScore, error) { - return framework.NodeScore{}, nil -} - type mockScheduler struct { result core.ScheduleResult err error @@ -179,12 +174,8 @@ func TestSchedulerCreation(t *testing.T) { client := clientsetfake.NewSimpleClientset() informerFactory := informers.NewSharedInformerFactory(client, 0) - testSource := "testProvider" eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: client.EventsV1beta1().Events("")}) - RegisterPriorityMapReduceFunction("PriorityOne", PriorityOne, nil, 1) - RegisterAlgorithmProvider(testSource, sets.NewString("PredicateOne"), sets.NewString("PriorityOne")) - stopCh := make(chan struct{}) defer close(stopCh) _, err := New(client, @@ -192,7 +183,6 @@ func TestSchedulerCreation(t *testing.T) { NewPodInformer(client, 0), eventBroadcaster.NewRecorder(scheme.Scheme, "scheduler"), stopCh, - WithAlgorithmSource(schedulerapi.SchedulerAlgorithmSource{Provider: &testSource}), WithPodInitialBackoffSeconds(1), WithPodMaxBackoffSeconds(10), ) @@ -217,7 +207,6 @@ func TestSchedulerCreation(t *testing.T) { NewPodInformer(client, 0), eventBroadcaster.NewRecorder(scheme.Scheme, "scheduler"), stopCh, - WithAlgorithmSource(schedulerapi.SchedulerAlgorithmSource{Provider: &testSource}), WithPodInitialBackoffSeconds(1), WithPodMaxBackoffSeconds(10), WithFrameworkOutOfTreeRegistry(registryFake), diff --git a/test/integration/scheduler/BUILD b/test/integration/scheduler/BUILD index 74ed3df6e24..2ce8ee70629 100644 --- a/test/integration/scheduler/BUILD +++ b/test/integration/scheduler/BUILD @@ -28,11 +28,9 @@ go_test( "//pkg/controller/nodelifecycle:go_default_library", "//pkg/features:go_default_library", "//pkg/scheduler:go_default_library", - "//pkg/scheduler/algorithm/predicates:go_default_library", "//pkg/scheduler/algorithmprovider:go_default_library", "//pkg/scheduler/apis/config:go_default_library", "//pkg/scheduler/apis/extender/v1:go_default_library", - "//pkg/scheduler/framework/plugins/noderesources:go_default_library", "//pkg/scheduler/framework/v1alpha1:go_default_library", "//pkg/scheduler/nodeinfo:go_default_library", "//pkg/scheduler/testing:go_default_library", @@ -49,7 +47,6 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", diff --git a/test/integration/scheduler/framework_test.go b/test/integration/scheduler/framework_test.go index 7e3fa317b6b..c3d7f414e38 100644 --- a/test/integration/scheduler/framework_test.go +++ b/test/integration/scheduler/framework_test.go @@ -30,7 +30,6 @@ import ( clientset "k8s.io/client-go/kubernetes" scheduler "k8s.io/kubernetes/pkg/scheduler" schedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" - "k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources" framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" ) @@ -483,7 +482,7 @@ func TestPreFilterPlugin(t *testing.T) { // Create the master and the scheduler with the test plugin set. context := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "prefilter-plugin", nil), 2, scheduler.WithFrameworkPlugins(plugins), - scheduler.WithFrameworkInTreeRegistry(registry)) + scheduler.WithFrameworkOutOfTreeRegistry(registry)) defer cleanupTest(t, context) tests := []struct { @@ -554,7 +553,7 @@ func TestScorePlugin(t *testing.T) { context := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "score-plugin", nil), 10, scheduler.WithFrameworkPlugins(plugins), - scheduler.WithFrameworkInTreeRegistry(registry)) + scheduler.WithFrameworkOutOfTreeRegistry(registry)) defer cleanupTest(t, context) for i, fail := range []bool{false, true} { @@ -612,7 +611,7 @@ func TestNormalizeScorePlugin(t *testing.T) { } context := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "score-plugin", nil), 10, scheduler.WithFrameworkPlugins(plugins), - scheduler.WithFrameworkInTreeRegistry(registry)) + scheduler.WithFrameworkOutOfTreeRegistry(registry)) defer cleanupTest(t, context) @@ -657,7 +656,7 @@ func TestReservePlugin(t *testing.T) { // Create the master and the scheduler with the test plugin set. context := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "reserve-plugin", nil), 2, scheduler.WithFrameworkPlugins(plugins), - scheduler.WithFrameworkInTreeRegistry(registry)) + scheduler.WithFrameworkOutOfTreeRegistry(registry)) defer cleanupTest(t, context) for _, fail := range []bool{false, true} { @@ -709,7 +708,7 @@ func TestPrebindPlugin(t *testing.T) { // Create the master and the scheduler with the test plugin set. context := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "prebind-plugin", nil), 2, scheduler.WithFrameworkPlugins(plugins), - scheduler.WithFrameworkInTreeRegistry(registry)) + scheduler.WithFrameworkOutOfTreeRegistry(registry)) defer cleanupTest(t, context) tests := []struct { @@ -792,7 +791,7 @@ func TestUnreservePlugin(t *testing.T) { // Create the master and the scheduler with the test plugin set. context := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "unreserve-plugin", nil), 2, scheduler.WithFrameworkPlugins(plugins), - scheduler.WithFrameworkInTreeRegistry(registry)) + scheduler.WithFrameworkOutOfTreeRegistry(registry)) defer cleanupTest(t, context) tests := []struct { @@ -882,8 +881,7 @@ func TestBindPlugin(t *testing.T) { // Create the master and the scheduler with the test plugin set. context := initTestSchedulerWithOptions(t, testContext, false, nil, time.Second, scheduler.WithFrameworkPlugins(plugins), - scheduler.WithFrameworkInTreeRegistry(registry), - scheduler.WithFrameworkConfigProducerRegistry(nil)) + scheduler.WithFrameworkOutOfTreeRegistry(registry)) defer cleanupTest(t, context) // Add a few nodes. @@ -1043,7 +1041,7 @@ func TestPostBindPlugin(t *testing.T) { // Create the master and the scheduler with the test plugin set. context := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "postbind-plugin", nil), 2, scheduler.WithFrameworkPlugins(plugins), - scheduler.WithFrameworkInTreeRegistry(registry)) + scheduler.WithFrameworkOutOfTreeRegistry(registry)) defer cleanupTest(t, context) tests := []struct { @@ -1099,7 +1097,7 @@ func TestPermitPlugin(t *testing.T) { // Create the master and the scheduler with the test plugin set. context := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "permit-plugin", nil), 2, scheduler.WithFrameworkPlugins(plugins), - scheduler.WithFrameworkInTreeRegistry(registry)) + scheduler.WithFrameworkOutOfTreeRegistry(registry)) defer cleanupTest(t, context) tests := []struct { @@ -1187,7 +1185,7 @@ func TestMultiplePermitPlugins(t *testing.T) { // Create the master and the scheduler with the test plugin set. context := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "multi-permit-plugin", nil), 2, scheduler.WithFrameworkPlugins(plugins), - scheduler.WithFrameworkInTreeRegistry(registry)) + scheduler.WithFrameworkOutOfTreeRegistry(registry)) defer cleanupTest(t, context) // Both permit plugins will return Wait for permitting @@ -1242,7 +1240,7 @@ func TestPermitPluginsCancelled(t *testing.T) { // Create the master and the scheduler with the test plugin set. context := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "permit-plugins", nil), 2, scheduler.WithFrameworkPlugins(plugins), - scheduler.WithFrameworkInTreeRegistry(registry)) + scheduler.WithFrameworkOutOfTreeRegistry(registry)) defer cleanupTest(t, context) // Both permit plugins will return Wait for permitting @@ -1283,7 +1281,7 @@ func TestCoSchedulingWithPermitPlugin(t *testing.T) { // Create the master and the scheduler with the test plugin set. context := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "permit-plugin", nil), 2, scheduler.WithFrameworkPlugins(plugins), - scheduler.WithFrameworkInTreeRegistry(registry)) + scheduler.WithFrameworkOutOfTreeRegistry(registry)) defer cleanupTest(t, context) tests := []struct { @@ -1364,7 +1362,7 @@ func TestFilterPlugin(t *testing.T) { // Create the master and the scheduler with the test plugin set. context := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "filter-plugin", nil), 2, scheduler.WithFrameworkPlugins(plugins), - scheduler.WithFrameworkInTreeRegistry(registry)) + scheduler.WithFrameworkOutOfTreeRegistry(registry)) defer cleanupTest(t, context) for _, fail := range []bool{false, true} { @@ -1415,7 +1413,7 @@ func TestPostFilterPlugin(t *testing.T) { // Create the master and the scheduler with the test plugin set. context := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "post-filter-plugin", nil), 2, scheduler.WithFrameworkPlugins(plugins), - scheduler.WithFrameworkInTreeRegistry(registry)) + scheduler.WithFrameworkOutOfTreeRegistry(registry)) defer cleanupTest(t, context) for _, fail := range []bool{false, true} { @@ -1451,16 +1449,11 @@ func TestPreemptWithPermitPlugin(t *testing.T) { // Create a plugin registry for testing. Register only a permit plugin. permitPlugin := &PermitPlugin{} registry, plugins := initRegistryAndConfig(permitPlugin) - // Fit filter plugin must be registered. - registry.Register(noderesources.FitName, noderesources.NewFit) - plugins.Filter = &schedulerconfig.PluginSet{ - Enabled: []schedulerconfig.Plugin{{Name: noderesources.FitName}}, - } // Create the master and the scheduler with the test plugin set. context := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "preempt-with-permit-plugin", nil), 0, scheduler.WithFrameworkPlugins(plugins), - scheduler.WithFrameworkInTreeRegistry(registry)) + scheduler.WithFrameworkOutOfTreeRegistry(registry)) defer cleanupTest(t, context) // Add one node. @@ -1524,7 +1517,6 @@ func TestPreemptWithPermitPlugin(t *testing.T) { } func initTestSchedulerForFrameworkTest(t *testing.T, context *testContext, nodeCount int, opts ...scheduler.Option) *testContext { - opts = append(opts, scheduler.WithFrameworkConfigProducerRegistry(nil)) c := initTestSchedulerWithOptions(t, context, false, nil, time.Second, opts...) if nodeCount > 0 { _, err := createNodes(c.clientSet, "test-node", nil, nodeCount) diff --git a/test/integration/scheduler/scheduler_test.go b/test/integration/scheduler/scheduler_test.go index ce4beacb53b..1943c1ae2d5 100644 --- a/test/integration/scheduler/scheduler_test.go +++ b/test/integration/scheduler/scheduler_test.go @@ -29,7 +29,6 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" @@ -39,11 +38,8 @@ import ( "k8s.io/client-go/tools/events" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/scheduler" - "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" _ "k8s.io/kubernetes/pkg/scheduler/algorithmprovider" kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" - schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" - schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" "k8s.io/kubernetes/test/integration/framework" ) @@ -54,22 +50,6 @@ type nodeStateManager struct { makeUnSchedulable nodeMutationFunc } -func PredicateOne(pod *v1.Pod, meta predicates.Metadata, nodeInfo *schedulernodeinfo.NodeInfo) (bool, []predicates.PredicateFailureReason, error) { - return true, nil, nil -} - -func PredicateTwo(pod *v1.Pod, meta predicates.Metadata, nodeInfo *schedulernodeinfo.NodeInfo) (bool, []predicates.PredicateFailureReason, error) { - return true, nil, nil -} - -func PriorityOne(pod *v1.Pod, meta interface{}, nodeInfo *schedulernodeinfo.NodeInfo) (schedulerframework.NodeScore, error) { - return schedulerframework.NodeScore{}, nil -} - -func PriorityTwo(pod *v1.Pod, meta interface{}, nodeInfo *schedulernodeinfo.NodeInfo) (schedulerframework.NodeScore, error) { - return schedulerframework.NodeScore{}, nil -} - // TestSchedulerCreationFromConfigMap verifies that scheduler can be created // from configurations provided by a ConfigMap object and then verifies that the // configuration is applied correctly. @@ -84,39 +64,33 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) { defer clientSet.CoreV1().Nodes().DeleteCollection(nil, metav1.ListOptions{}) informerFactory := informers.NewSharedInformerFactory(clientSet, 0) - // Pre-register some predicate and priority functions - scheduler.RegisterFitPredicate("PredicateOne", PredicateOne) - scheduler.RegisterFitPredicate("PredicateTwo", PredicateTwo) - scheduler.RegisterPriorityMapReduceFunction("PriorityOne", PriorityOne, nil, 1) - scheduler.RegisterPriorityMapReduceFunction("PriorityTwo", PriorityTwo, nil, 1) - for i, test := range []struct { - policy string - expectedPredicates sets.String - expectedPlugins map[string][]kubeschedulerconfig.Plugin + policy string + expectedPlugins map[string][]kubeschedulerconfig.Plugin }{ { policy: `{ "kind" : "Policy", "apiVersion" : "v1", "predicates" : [ - {"name" : "PredicateOne"}, - {"name" : "PredicateTwo"} + {"name" : "PodFitsResources"} ], "priorities" : [ - {"name" : "PriorityOne", "weight" : 1}, - {"name" : "PriorityTwo", "weight" : 5} + {"name" : "ImageLocalityPriority", "weight" : 1} ] }`, - expectedPredicates: sets.NewString( - "PredicateOne", - "PredicateTwo", - ), expectedPlugins: map[string][]kubeschedulerconfig.Plugin{ + "PreFilterPlugin": { + {Name: "NodeResourcesFit"}, + }, "FilterPlugin": { {Name: "NodeUnschedulable"}, + {Name: "NodeResourcesFit"}, {Name: "TaintToleration"}, }, + "ScorePlugin": { + {Name: "ImageLocality", Weight: 1}, + }, }, }, { @@ -169,7 +143,6 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) { "predicates" : [], "priorities" : [] }`, - expectedPredicates: sets.NewString(), expectedPlugins: map[string][]kubeschedulerconfig.Plugin{ "FilterPlugin": { {Name: "NodeUnschedulable"}, @@ -181,23 +154,23 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) { policy: `apiVersion: v1 kind: Policy predicates: -- name: PredicateOne -- name: PredicateTwo +- name: PodFitsResources priorities: -- name: PriorityOne +- name: ImageLocalityPriority weight: 1 -- name: PriorityTwo - weight: 5 `, - expectedPredicates: sets.NewString( - "PredicateOne", - "PredicateTwo", - ), expectedPlugins: map[string][]kubeschedulerconfig.Plugin{ + "PreFilterPlugin": { + {Name: "NodeResourcesFit"}, + }, "FilterPlugin": { {Name: "NodeUnschedulable"}, + {Name: "NodeResourcesFit"}, {Name: "TaintToleration"}, }, + "ScorePlugin": { + {Name: "ImageLocality", Weight: 1}, + }, }, }, { @@ -248,7 +221,6 @@ kind: Policy predicates: [] priorities: [] `, - expectedPredicates: sets.NewString(), expectedPlugins: map[string][]kubeschedulerconfig.Plugin{ "FilterPlugin": { {Name: "NodeUnschedulable"}, @@ -291,20 +263,12 @@ priorities: [] scheduler.WithBindTimeoutSeconds(defaultBindTimeout), ) if err != nil { - t.Fatalf("couldn't make scheduler config: %v", err) + t.Fatalf("couldn't make scheduler config for test %d: %v", i, err) } - // Verify that the config is applied correctly. - schedPredicates := sets.NewString() - for k := range sched.Algorithm.Predicates() { - schedPredicates.Insert(k) - } - if !schedPredicates.Equal(test.expectedPredicates) { - t.Errorf("Expected predicates %v, got %v", test.expectedPredicates, schedPredicates) - } schedPlugins := sched.Framework.ListPlugins() if diff := cmp.Diff(test.expectedPlugins, schedPlugins); diff != "" { - t.Errorf("unexpected predicates diff (-want, +got): %s", diff) + t.Errorf("unexpected plugins diff (-want, +got): %s", diff) } } }