diff --git a/pkg/scheduler/generic_scheduler.go b/pkg/scheduler/generic_scheduler.go index 0aa679274ae..7a2542daa0a 100644 --- a/pkg/scheduler/generic_scheduler.go +++ b/pkg/scheduler/generic_scheduler.go @@ -135,6 +135,13 @@ func findNodesThatFit(pod api.Pod, podLister PodLister, predicates map[string]Fi // All scores are finally combined (added) to get the total weighted scores of all minions func prioritizeNodes(pod api.Pod, podLister PodLister, priorityConfigs []PriorityConfig, minionLister MinionLister) (HostPriorityList, error) { result := HostPriorityList{} + + // If no priority configs are provided, then the EqualPriority function is applied + // This is required to generate the priority list in the required format + if len(priorityConfigs) == 0 { + return EqualPriority(pod, podLister, minionLister) + } + combinedScores := map[string]int{} for _, priorityConfig := range priorityConfigs { weight := priorityConfig.Weight diff --git a/plugin/cmd/kube-scheduler/app/server.go b/plugin/cmd/kube-scheduler/app/server.go index f65f7100f3d..e255a256d92 100644 --- a/plugin/cmd/kube-scheduler/app/server.go +++ b/plugin/cmd/kube-scheduler/app/server.go @@ -18,8 +18,11 @@ limitations under the License. package app import ( + "fmt" + "io/ioutil" "net" "net/http" + "os" "strconv" "github.com/GoogleCloudPlatform/kubernetes/pkg/api" @@ -30,6 +33,8 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/util" "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler" _ "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/algorithmprovider" + schedulerapi "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/api" + latestschedulerapi "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/api/latest" "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/factory" "github.com/golang/glog" @@ -42,6 +47,7 @@ type SchedulerServer struct { Address util.IP ClientConfig client.Config AlgorithmProvider string + PolicyConfigFile string } // NewSchedulerServer creates a new SchedulerServer with default parameters @@ -60,6 +66,7 @@ func (s *SchedulerServer) AddFlags(fs *pflag.FlagSet) { fs.Var(&s.Address, "address", "The IP address to serve on (set to 0.0.0.0 for all interfaces)") client.BindClientConfigFlags(fs, &s.ClientConfig) fs.StringVar(&s.AlgorithmProvider, "algorithm_provider", s.AlgorithmProvider, "The scheduling algorithm provider to use") + fs.StringVar(&s.PolicyConfigFile, "policy_config_file", s.PolicyConfigFile, "File with scheduler policy configuration") } // Run runs the specified SchedulerServer. This should never exit. @@ -78,6 +85,7 @@ func (s *SchedulerServer) Run(_ []string) error { if err != nil { glog.Fatalf("Failed to create scheduler configuration: %v", err) } + sched := scheduler.New(config) sched.Run() @@ -85,10 +93,28 @@ func (s *SchedulerServer) Run(_ []string) error { } func (s *SchedulerServer) createConfig(configFactory *factory.ConfigFactory) (*scheduler.Config, error) { + var policy schedulerapi.Policy + var configData []byte + + if _, err := os.Stat(s.PolicyConfigFile); err == nil { + configData, err = ioutil.ReadFile(s.PolicyConfigFile) + if err != nil { + return nil, fmt.Errorf("Unable to read policy config: %v", err) + } + err = latestschedulerapi.Codec.DecodeInto(configData, &policy) + if err != nil { + return nil, fmt.Errorf("Invalid configuration: %v", err) + } + + return configFactory.CreateFromConfig(policy) + } + + // if the config file isn't provided, use the specified (or default) provider // check of algorithm provider is registered and fail fast _, err := factory.GetAlgorithmProvider(s.AlgorithmProvider) if err != nil { return nil, err } + return configFactory.CreateFromProvider(s.AlgorithmProvider) } diff --git a/plugin/pkg/scheduler/algorithmprovider/affinity/affinity.go b/plugin/pkg/scheduler/algorithmprovider/affinity/affinity.go deleted file mode 100644 index ee023355d47..00000000000 --- a/plugin/pkg/scheduler/algorithmprovider/affinity/affinity.go +++ /dev/null @@ -1,55 +0,0 @@ -/* -Copyright 2014 Google Inc. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// This algorithm provider has predicates and priorities related to affinity/anti-affinity for the scheduler. -package affinity - -import ( - algorithm "github.com/GoogleCloudPlatform/kubernetes/pkg/scheduler" - "github.com/GoogleCloudPlatform/kubernetes/pkg/util" - "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/factory" -) - -const AffinityProvider string = "AffinityProvider" - -func init() { - factory.RegisterAlgorithmProvider(AffinityProvider, affinityPredicates(), affinityPriorities()) -} - -func affinityPredicates() util.StringSet { - return util.NewStringSet( - "HostName", - "MatchNodeSelector", - "PodFitsPorts", - "PodFitsResources", - "NoDiskConflict", - // Ensures that all pods within the same service are hosted on minions within the same region as defined by the "region" label - factory.RegisterFitPredicate("ServiceAffinity", algorithm.NewServiceAffinityPredicate(factory.PodLister, factory.ServiceLister, factory.MinionLister, []string{"region"})), - // Fit is defined based on the presence of the "region" label on a minion, regardless of value. - factory.RegisterFitPredicate("NodeLabelPredicate", algorithm.NewNodeLabelPredicate(factory.MinionLister, []string{"region"}, true)), - ) -} - -func affinityPriorities() util.StringSet { - return util.NewStringSet( - "LeastRequestedPriority", - "ServiceSpreadingPriority", - // spreads pods belonging to the same service across minions in different zones - factory.RegisterPriorityFunction("ZoneSpreadingPriority", algorithm.NewServiceAntiAffinityPriority(factory.ServiceLister, "zone"), 2), - // Prioritize nodes based on the presence of the "zone" label on a minion, regardless of value. - factory.RegisterPriorityFunction("NodeLabelPriority", algorithm.NewNodeLabelPriority("zone", true), 1), - ) -} diff --git a/plugin/pkg/scheduler/algorithmprovider/plugins.go b/plugin/pkg/scheduler/algorithmprovider/plugins.go index ac7123efe26..d534b05c0bf 100644 --- a/plugin/pkg/scheduler/algorithmprovider/plugins.go +++ b/plugin/pkg/scheduler/algorithmprovider/plugins.go @@ -18,6 +18,5 @@ limitations under the License. package algorithmprovider import ( - _ "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/algorithmprovider/affinity" _ "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/algorithmprovider/defaults" ) diff --git a/plugin/pkg/scheduler/algorithmprovider/plugins_test.go b/plugin/pkg/scheduler/algorithmprovider/plugins_test.go index b70d55fc789..8f7dd3a1ffa 100644 --- a/plugin/pkg/scheduler/algorithmprovider/plugins_test.go +++ b/plugin/pkg/scheduler/algorithmprovider/plugins_test.go @@ -19,14 +19,12 @@ package algorithmprovider import ( "testing" - "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/algorithmprovider/affinity" "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/factory" ) var ( algorithmProviderNames = []string{ factory.DefaultProvider, - affinity.AffinityProvider, } ) diff --git a/plugin/pkg/scheduler/api/latest/latest.go b/plugin/pkg/scheduler/api/latest/latest.go new file mode 100644 index 00000000000..73551248e38 --- /dev/null +++ b/plugin/pkg/scheduler/api/latest/latest.go @@ -0,0 +1,38 @@ +/* +Copyright 2014 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package latest + +import ( + "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/api/v1" +) + +// Version is the string that represents the current external default version. +const Version = "v1" + +// OldestVersion is the string that represents the oldest server version supported. +const OldestVersion = "v1" + +// Versions is the list of versions that are recognized in code. The order provided +// may be assumed to be least feature rich to most feature rich, and clients may +// choose to prefer the latter items in the list over the former items when presented +// with a set of versions to choose. +var Versions = []string{"v1"} + +// Codec is the default codec for serializing input that should use +// the latest supported version. +// This codec can decode any object that Kubernetes is aware of. +var Codec = v1.Codec diff --git a/plugin/pkg/scheduler/api/register.go b/plugin/pkg/scheduler/api/register.go new file mode 100644 index 00000000000..ae3c8ae3d87 --- /dev/null +++ b/plugin/pkg/scheduler/api/register.go @@ -0,0 +1,32 @@ +/* +Copyright 2014 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/GoogleCloudPlatform/kubernetes/pkg/runtime" +) + +// Scheme is the default instance of runtime.Scheme to which types in the Kubernetes API are already registered. +var Scheme = runtime.NewScheme() + +func init() { + Scheme.AddKnownTypes("", + &Policy{}, + ) +} + +func (*Policy) IsAnAPIObject() {} diff --git a/plugin/pkg/scheduler/api/types.go b/plugin/pkg/scheduler/api/types.go new file mode 100644 index 00000000000..a1cac22bad4 --- /dev/null +++ b/plugin/pkg/scheduler/api/types.go @@ -0,0 +1,103 @@ +/* +Copyright 2014 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/GoogleCloudPlatform/kubernetes/pkg/api" +) + +type Policy struct { + api.TypeMeta `json:",inline"` + // Holds the information to configure the fit predicate functions + Predicates []PredicatePolicy `json:"predicates"` + // Holds the information to configure the priority functions + Priorities []PriorityPolicy `json:"priorities"` +} + +type PredicatePolicy struct { + // Identifier of the predicate policy + // For a custom predicate, the name can be user-defined + // For the Kubernetes provided predicates, the name is the identifier of the pre-defined predicate + Name string `json:"name"` + // Holds the parameters to configure the given predicate + Argument *PredicateArgument `json:"argument"` +} + +type PriorityPolicy struct { + // Identifier of the priority policy + // For a custom priority, the name can be user-defined + // For the Kubernetes provided priority functions, the name is the identifier of the pre-defined priority function + Name string `json:"name"` + // The numeric multiplier for the minion scores that the priority function generates + Weight int `json:"weight"` + // Holds the parameters to configure the given priority function + Argument *PriorityArgument `json:"argument"` +} + +// Represents the arguments that the different types of predicates take +// Only one of its members may be specified +type PredicateArgument struct { + // The predicate that provides affinity for pods belonging to a service + // It uses a label to identify minions that belong to the same "group" + ServiceAffinity *ServiceAffinity `json:"serviceAffinity"` + // The predicate that checks whether a particular minion has a certain label + // defined or not, regardless of value + LabelsPresence *LabelsPresence `json:"labelsPresence"` +} + +// Represents the arguments that the different types of priorities take. +// Only one of its members may be specified +type PriorityArgument struct { + // The priority function that ensures a good spread (anti-affinity) for pods belonging to a service + // It uses a label to identify minions that belong to the same "group" + ServiceAntiAffinity *ServiceAntiAffinity `json:"serviceAntiAffinity"` + // The priority function that checks whether a particular minion has a certain label + // defined or not, regardless of value + LabelPreference *LabelPreference `json:"labelPreference"` +} + +// Holds the parameters that are used to configure the corresponding predicate +type ServiceAffinity struct { + // The list of labels that identify minion "groups" + // All of the labels should match for the minion to be considered a fit for hosting the pod + Labels []string `json:"labels"` +} + +// Holds the parameters that are used to configure the corresponding predicate +type LabelsPresence struct { + // The list of labels that identify minion "groups" + // All of the labels should be either present (or absent) for the minion to be considered a fit for hosting the pod + Labels []string `json:"labels"` + // The boolean flag that indicates whether the labels should be present or absent from the minion + Presence bool `json:"presence"` +} + +// Holds the parameters that are used to configure the corresponding priority function +type ServiceAntiAffinity struct { + // Used to identify minion "groups" + Label string `json:"label"` +} + +// Holds the parameters that are used to configure the corresponding priority function +type LabelPreference struct { + // Used to identify minion "groups" + Label string `json:"label"` + // This is a boolean flag + // If true, higher priority is given to minions that have the label + // If false, higher priority is given to minions that do not have the label + Presence bool `json:"presence"` +} diff --git a/plugin/pkg/scheduler/api/v1/register.go b/plugin/pkg/scheduler/api/v1/register.go new file mode 100644 index 00000000000..b6718cff877 --- /dev/null +++ b/plugin/pkg/scheduler/api/v1/register.go @@ -0,0 +1,33 @@ +/* +Copyright 2014 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "github.com/GoogleCloudPlatform/kubernetes/pkg/runtime" + "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/api" +) + +// Codec encodes internal objects to the v1 scheme +var Codec = runtime.CodecFor(api.Scheme, "v1") + +func init() { + api.Scheme.AddKnownTypes("v1", + &Policy{}, + ) +} + +func (*Policy) IsAnAPIObject() {} diff --git a/plugin/pkg/scheduler/api/v1/types.go b/plugin/pkg/scheduler/api/v1/types.go new file mode 100644 index 00000000000..fefb278ad85 --- /dev/null +++ b/plugin/pkg/scheduler/api/v1/types.go @@ -0,0 +1,103 @@ +/* +Copyright 2014 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "github.com/GoogleCloudPlatform/kubernetes/pkg/api/v1beta3" +) + +type Policy struct { + v1beta3.TypeMeta `json:",inline"` + // Holds the information to configure the fit predicate functions + Predicates []PredicatePolicy `json:"predicates"` + // Holds the information to configure the priority functions + Priorities []PriorityPolicy `json:"priorities"` +} + +type PredicatePolicy struct { + // Identifier of the predicate policy + // For a custom predicate, the name can be user-defined + // For the Kubernetes provided predicates, the name is the identifier of the pre-defined predicate + Name string `json:"name"` + // Holds the parameters to configure the given predicate + Argument *PredicateArgument `json:"argument"` +} + +type PriorityPolicy struct { + // Identifier of the priority policy + // For a custom priority, the name can be user-defined + // For the Kubernetes provided priority functions, the name is the identifier of the pre-defined priority function + Name string `json:"name"` + // The numeric multiplier for the minion scores that the priority function generates + Weight int `json:"weight"` + // Holds the parameters to configure the given priority function + Argument *PriorityArgument `json:"argument"` +} + +// Represents the arguments that the different types of predicates take +// Only one of its members may be specified +type PredicateArgument struct { + // The predicate that provides affinity for pods belonging to a service + // It uses a label to identify minions that belong to the same "group" + ServiceAffinity *ServiceAffinity `json:"serviceAffinity"` + // The predicate that checks whether a particular minion has a certain label + // defined or not, regardless of value + LabelsPresence *LabelsPresence `json:"labelsPresence"` +} + +// Represents the arguments that the different types of priorities take. +// Only one of its members may be specified +type PriorityArgument struct { + // The priority function that ensures a good spread (anti-affinity) for pods belonging to a service + // It uses a label to identify minions that belong to the same "group" + ServiceAntiAffinity *ServiceAntiAffinity `json:"serviceAntiAffinity"` + // The priority function that checks whether a particular minion has a certain label + // defined or not, regardless of value + LabelPreference *LabelPreference `json:"labelPreference"` +} + +// Holds the parameters that are used to configure the corresponding predicate +type ServiceAffinity struct { + // The list of labels that identify minion "groups" + // All of the labels should match for the minion to be considered a fit for hosting the pod + Labels []string `json:"labels"` +} + +// Holds the parameters that are used to configure the corresponding predicate +type LabelsPresence struct { + // The list of labels that identify minion "groups" + // All of the labels should be either present (or absent) for the minion to be considered a fit for hosting the pod + Labels []string `json:"labels"` + // The boolean flag that indicates whether the labels should be present or absent from the minion + Presence bool `json:"presence"` +} + +// Holds the parameters that are used to configure the corresponding priority function +type ServiceAntiAffinity struct { + // Used to identify minion "groups" + Label string `json:"label"` +} + +// Holds the parameters that are used to configure the corresponding priority function +type LabelPreference struct { + // Used to identify minion "groups" + Label string `json:"label"` + // This is a boolean flag + // If true, higher priority is given to minions that have the label + // If false, higher priority is given to minions that do not have the label + Presence bool `json:"presence"` +} diff --git a/plugin/pkg/scheduler/factory/factory.go b/plugin/pkg/scheduler/factory/factory.go index e7cb6f12457..48c84142091 100644 --- a/plugin/pkg/scheduler/factory/factory.go +++ b/plugin/pkg/scheduler/factory/factory.go @@ -30,6 +30,7 @@ import ( algorithm "github.com/GoogleCloudPlatform/kubernetes/pkg/scheduler" "github.com/GoogleCloudPlatform/kubernetes/pkg/util" "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler" + schedulerapi "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/api" "github.com/golang/glog" ) @@ -53,7 +54,7 @@ type ConfigFactory struct { ServiceLister *cache.StoreToServiceLister } -// NewConfigFactory initializes the factory. +// Initializes the factory. func NewConfigFactory(client *client.Client) *ConfigFactory { return &ConfigFactory{ Client: client, @@ -69,7 +70,7 @@ func (f *ConfigFactory) Create() (*scheduler.Config, error) { return f.CreateFromProvider(DefaultProvider) } -// CreateFromProvider creates a scheduler from the name of a registered algorithm provider. +// Creates a scheduler from the name of a registered algorithm provider. func (f *ConfigFactory) CreateFromProvider(providerName string) (*scheduler.Config, error) { glog.V(2).Infof("creating scheduler from algorithm provider '%v'", providerName) provider, err := GetAlgorithmProvider(providerName) @@ -80,7 +81,26 @@ func (f *ConfigFactory) CreateFromProvider(providerName string) (*scheduler.Conf return f.CreateFromKeys(provider.FitPredicateKeys, provider.PriorityFunctionKeys) } -// CreateFromKeys creates a scheduler from a set of registered fit predicate keys and priority keys. +// Creates a scheduler from the configuration file +func (f *ConfigFactory) CreateFromConfig(policy schedulerapi.Policy) (*scheduler.Config, error) { + glog.V(2).Infof("creating scheduler from configuration: %v", policy) + + predicateKeys := util.NewStringSet() + for _, predicate := range policy.Predicates { + glog.V(2).Infof("Registering predicate: %s", predicate.Name) + predicateKeys.Insert(RegisterCustomFitPredicate(predicate)) + } + + priorityKeys := util.NewStringSet() + for _, priority := range policy.Priorities { + glog.V(2).Infof("Registering priority: %s", priority.Name) + priorityKeys.Insert(RegisterCustomPriorityFunction(priority)) + } + + return f.CreateFromKeys(predicateKeys, priorityKeys) +} + +// Creates a scheduler from a set of registered fit predicate keys and priority keys. func (f *ConfigFactory) CreateFromKeys(predicateKeys, priorityKeys util.StringSet) (*scheduler.Config, error) { glog.V(2).Infof("creating scheduler with fit predicates '%v' and priority functions '%v", predicateKeys, priorityKeys) predicateFuncs, err := getFitPredicateFunctions(predicateKeys) @@ -140,7 +160,7 @@ func (f *ConfigFactory) CreateFromKeys(predicateKeys, priorityKeys util.StringSe }, nil } -// createUnassignedPodLW returns a cache.ListWatch that finds all pods that need to be +// Returns a cache.ListWatch that finds all pods that need to be // scheduled. func (factory *ConfigFactory) createUnassignedPodLW() *cache.ListWatch { return cache.NewListWatchFromClient(factory.Client, "pods", api.NamespaceAll, labels.Set{"DesiredState.Host": ""}.AsSelector()) @@ -154,7 +174,7 @@ func parseSelectorOrDie(s string) labels.Selector { return selector } -// createAssignedPodLW returns a cache.ListWatch that finds all pods that are +// Returns a cache.ListWatch that finds all pods that are // already scheduled. // TODO: return a ListerWatcher interface instead? func (factory *ConfigFactory) createAssignedPodLW() *cache.ListWatch { @@ -166,7 +186,7 @@ func (factory *ConfigFactory) createMinionLW() *cache.ListWatch { return cache.NewListWatchFromClient(factory.Client, "minions", api.NamespaceAll, parseSelectorOrDie("")) } -// pollMinions lists all minions and filter out unhealthy ones, then returns +// Lists all minions and filter out unhealthy ones, then returns // an enumerator for cache.Poller. func (factory *ConfigFactory) pollMinions() (cache.Enumerator, error) { allNodes := &api.NodeList{} @@ -201,7 +221,7 @@ func (factory *ConfigFactory) pollMinions() (cache.Enumerator, error) { return &nodeEnumerator{nodes}, nil } -// createServiceLW returns a cache.ListWatch that gets all changes to services. +// Returns a cache.ListWatch that gets all changes to services. func (factory *ConfigFactory) createServiceLW() *cache.ListWatch { return cache.NewListWatchFromClient(factory.Client, "services", api.NamespaceAll, parseSelectorOrDie("")) } @@ -231,7 +251,7 @@ func (factory *ConfigFactory) makeDefaultErrorFunc(backoff *podBackoff, podQueue } } -// nodeEnumerator allows a cache.Poller to enumerate items in an api.NodeList +// Allows a cache.Poller to enumerate items in an api.NodeList type nodeEnumerator struct { *api.NodeList } diff --git a/plugin/pkg/scheduler/factory/factory_test.go b/plugin/pkg/scheduler/factory/factory_test.go index f07fe4daf68..9d3eff0a571 100644 --- a/plugin/pkg/scheduler/factory/factory_test.go +++ b/plugin/pkg/scheduler/factory/factory_test.go @@ -30,7 +30,10 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/client" "github.com/GoogleCloudPlatform/kubernetes/pkg/client/cache" "github.com/GoogleCloudPlatform/kubernetes/pkg/runtime" + algorithm "github.com/GoogleCloudPlatform/kubernetes/pkg/scheduler" "github.com/GoogleCloudPlatform/kubernetes/pkg/util" + schedulerapi "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/api" + latestschedulerapi "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/api/latest" ) func TestCreate(t *testing.T) { @@ -46,6 +49,89 @@ func TestCreate(t *testing.T) { factory.Create() } +// Test configures a scheduler from a policies defined in a file +// It combines some configurable predicate/priorities with some pre-defined ones +func TestCreateFromConfig(t *testing.T) { + var configData []byte + var policy schedulerapi.Policy + + handler := util.FakeHandler{ + StatusCode: 500, + ResponseBody: "", + T: t, + } + server := httptest.NewServer(&handler) + defer server.Close() + client := client.NewOrDie(&client.Config{Host: server.URL, Version: testapi.Version()}) + factory := NewConfigFactory(client) + + // Pre-register some predicate and priority functions + RegisterFitPredicate("PredicateOne", PredicateOne) + RegisterFitPredicate("PredicateTwo", PredicateTwo) + RegisterPriorityFunction("PriorityOne", PriorityOne, 1) + RegisterPriorityFunction("PriorityTwo", PriorityTwo, 1) + + configData = []byte(`{ + "kind" : "Policy", + "apiVersion" : "v1", + "predicates" : [ + {"name" : "TestZoneAffinity", "argument" : {"serviceAffinity" : {"labels" : ["zone"]}}}, + {"name" : "TestRequireZone", "argument" : {"labelsPresence" : {"labels" : ["zone"], "presence" : true}}}, + {"name" : "PredicateOne"}, + {"name" : "PredicateTwo"} + ], + "priorities" : [ + {"name" : "RackSpread", "weight" : 3, "argument" : {"serviceAntiAffinity" : {"label" : "rack"}}}, + {"name" : "PriorityOne", "weight" : 2}, + {"name" : "PriorityTwo", "weight" : 1} ] + }`) + err := latestschedulerapi.Codec.DecodeInto(configData, &policy) + if err != nil { + t.Errorf("Invalid configuration: %v", err) + } + + factory.CreateFromConfig(policy) +} + +func TestCreateFromEmptyConfig(t *testing.T) { + var configData []byte + var policy schedulerapi.Policy + + handler := util.FakeHandler{ + StatusCode: 500, + ResponseBody: "", + T: t, + } + server := httptest.NewServer(&handler) + defer server.Close() + client := client.NewOrDie(&client.Config{Host: server.URL, Version: testapi.Version()}) + factory := NewConfigFactory(client) + + configData = []byte(`{}`) + err := latestschedulerapi.Codec.DecodeInto(configData, &policy) + if err != nil { + t.Errorf("Invalid configuration: %v", err) + } + + factory.CreateFromConfig(policy) +} + +func PredicateOne(pod api.Pod, existingPods []api.Pod, node string) (bool, error) { + return true, nil +} + +func PredicateTwo(pod api.Pod, existingPods []api.Pod, node string) (bool, error) { + return true, nil +} + +func PriorityOne(pod api.Pod, podLister algorithm.PodLister, minionLister algorithm.MinionLister) (algorithm.HostPriorityList, error) { + return []algorithm.HostPriority{}, nil +} + +func PriorityTwo(pod api.Pod, podLister algorithm.PodLister, minionLister algorithm.MinionLister) (algorithm.HostPriorityList, error) { + return []algorithm.HostPriority{}, nil +} + func TestPollMinions(t *testing.T) { table := []struct { minions []api.Node diff --git a/plugin/pkg/scheduler/factory/plugins.go b/plugin/pkg/scheduler/factory/plugins.go index 4ec5c2c028a..e53c24f0b18 100644 --- a/plugin/pkg/scheduler/factory/plugins.go +++ b/plugin/pkg/scheduler/factory/plugins.go @@ -23,6 +23,7 @@ import ( algorithm "github.com/GoogleCloudPlatform/kubernetes/pkg/scheduler" "github.com/GoogleCloudPlatform/kubernetes/pkg/util" + schedulerapi "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/api" "github.com/golang/glog" ) @@ -45,7 +46,7 @@ type AlgorithmProviderConfig struct { PriorityFunctionKeys util.StringSet } -// RegisterFitPredicate registers a fit predicate with the algorithm registry. Returns the name, +// Registers a fit predicate with the algorithm registry. Returns the name, // with which the predicate was registered. func RegisterFitPredicate(name string, predicate algorithm.FitPredicate) string { schedulerFactoryMutex.Lock() @@ -55,7 +56,34 @@ func RegisterFitPredicate(name string, predicate algorithm.FitPredicate) string return name } -// IsFitPredicateRegistered check is useful for testing providers. +// Registers a custom fit predicate with the algorithm registry. +// Returns the name, with which the predicate was registered. +func RegisterCustomFitPredicate(policy schedulerapi.PredicatePolicy) string { + var predicate algorithm.FitPredicate + var ok bool + + validatePredicateOrDie(policy) + + // generate the predicate function, if a custom type is requested + if policy.Argument != nil { + if policy.Argument.ServiceAffinity != nil { + predicate = algorithm.NewServiceAffinityPredicate(PodLister, ServiceLister, MinionLister, policy.Argument.ServiceAffinity.Labels) + } else if policy.Argument.LabelsPresence != nil { + predicate = algorithm.NewNodeLabelPredicate(MinionLister, policy.Argument.LabelsPresence.Labels, policy.Argument.LabelsPresence.Presence) + } + } else if predicate, ok = fitPredicateMap[policy.Name]; ok { + // checking to see if a pre-defined predicate is requested + glog.V(2).Infof("Predicate type %s already registered, reusing.", policy.Name) + } + + if predicate == nil { + glog.Fatalf("Invalid configuration: Predicate type not found for %s", policy.Name) + } + + return RegisterFitPredicate(policy.Name, predicate) +} + +// This check is useful for testing providers. func IsFitPredicateRegistered(name string) bool { schedulerFactoryMutex.Lock() defer schedulerFactoryMutex.Unlock() @@ -63,7 +91,7 @@ func IsFitPredicateRegistered(name string) bool { return ok } -// RegisterFitPredicate registers a priority function with the algorithm registry. Returns the name, +// Registers a priority function with the algorithm registry. Returns the name, // with which the function was registered. func RegisterPriorityFunction(name string, function algorithm.PriorityFunction, weight int) string { schedulerFactoryMutex.Lock() @@ -73,7 +101,33 @@ func RegisterPriorityFunction(name string, function algorithm.PriorityFunction, return name } -// IsPriorityFunctionRegistered check is useful for testing providers. +// Registers a custom priority function with the algorithm registry. +// Returns the name, with which the priority function was registered. +func RegisterCustomPriorityFunction(policy schedulerapi.PriorityPolicy) string { + var priority algorithm.PriorityFunction + + validatePriorityOrDie(policy) + + // generate the priority function, if a custom priority is requested + if policy.Argument != nil { + if policy.Argument.ServiceAntiAffinity != nil { + priority = algorithm.NewServiceAntiAffinityPriority(ServiceLister, policy.Argument.ServiceAntiAffinity.Label) + } else if policy.Argument.LabelPreference != nil { + priority = algorithm.NewNodeLabelPriority(policy.Argument.LabelPreference.Label, policy.Argument.LabelPreference.Presence) + } + } else if priorityConfig, ok := priorityFunctionMap[policy.Name]; ok { + glog.V(2).Infof("Priority type %s already registered, reusing.", policy.Name) + priority = priorityConfig.Function + } + + if priority == nil { + glog.Fatalf("Invalid configuration: Priority type not found for %s", policy.Name) + } + + return RegisterPriorityFunction(policy.Name, priority, policy.Weight) +} + +// This check is useful for testing providers. func IsPriorityFunctionRegistered(name string) bool { schedulerFactoryMutex.Lock() defer schedulerFactoryMutex.Unlock() @@ -81,7 +135,7 @@ func IsPriorityFunctionRegistered(name string) bool { return ok } -// SetPriorityFunctionWeight sets the weight of an already registered priority function. +// Sets the weight of an already registered priority function. func SetPriorityFunctionWeight(name string, weight int) { schedulerFactoryMutex.Lock() defer schedulerFactoryMutex.Unlock() @@ -91,9 +145,10 @@ func SetPriorityFunctionWeight(name string, weight int) { return } config.Weight = weight + priorityFunctionMap[name] = config } -// RegisterAlgorithmProvider registers a new algorithm provider with the algorithm registry. This should +// Registers a new algorithm provider with the algorithm registry. This should // be called from the init function in a provider plugin. func RegisterAlgorithmProvider(name string, predicateKeys, priorityKeys util.StringSet) string { schedulerFactoryMutex.Lock() @@ -106,7 +161,7 @@ func RegisterAlgorithmProvider(name string, predicateKeys, priorityKeys util.Str return name } -// GetAlgorithmProvider should not be used to modify providers. It is publicly visible for testing. +// This function should not be used to modify providers. It is publicly visible for testing. func GetAlgorithmProvider(name string) (*AlgorithmProviderConfig, error) { schedulerFactoryMutex.Lock() defer schedulerFactoryMutex.Unlock() @@ -157,3 +212,33 @@ func validateAlgorithmNameOrDie(name string) { glog.Fatalf("algorithm name %v does not match the name validation regexp \"%v\".", name, validName) } } + +func validatePredicateOrDie(predicate schedulerapi.PredicatePolicy) { + if predicate.Argument != nil { + numArgs := 0 + if predicate.Argument.ServiceAffinity != nil { + numArgs++ + } + if predicate.Argument.LabelsPresence != nil { + numArgs++ + } + if numArgs != 1 { + glog.Fatalf("Exactly 1 predicate argument is required") + } + } +} + +func validatePriorityOrDie(priority schedulerapi.PriorityPolicy) { + if priority.Argument != nil { + numArgs := 0 + if priority.Argument.ServiceAntiAffinity != nil { + numArgs++ + } + if priority.Argument.LabelPreference != nil { + numArgs++ + } + if numArgs != 1 { + glog.Fatalf("Exactly 1 priority argument is required") + } + } +}