From 548e0da5671d37987ca9db3672141fb4075578c8 Mon Sep 17 00:00:00 2001 From: Abhishek Gupta Date: Thu, 19 Feb 2015 16:18:28 -0800 Subject: [PATCH] Configuring scheduler via json configuration file --- plugin/cmd/kube-scheduler/app/server.go | 27 +++++ .../algorithmprovider/affinity/affinity.go | 8 +- plugin/pkg/scheduler/api/latest/latest.go | 40 +++++++ plugin/pkg/scheduler/api/register.go | 32 ++++++ plugin/pkg/scheduler/api/types.go | 73 ++++++++++++ plugin/pkg/scheduler/api/v1/register.go | 33 ++++++ plugin/pkg/scheduler/api/v1/types.go | 106 ++++++++++++++++++ plugin/pkg/scheduler/factory/factory.go | 20 ++++ plugin/pkg/scheduler/factory/plugins.go | 87 +++++++++++++- 9 files changed, 421 insertions(+), 5 deletions(-) create mode 100644 plugin/pkg/scheduler/api/latest/latest.go create mode 100644 plugin/pkg/scheduler/api/register.go create mode 100644 plugin/pkg/scheduler/api/types.go create mode 100644 plugin/pkg/scheduler/api/v1/register.go create mode 100644 plugin/pkg/scheduler/api/v1/types.go diff --git a/plugin/cmd/kube-scheduler/app/server.go b/plugin/cmd/kube-scheduler/app/server.go index f65f7100f3d..4e643bf99d4 100644 --- a/plugin/cmd/kube-scheduler/app/server.go +++ b/plugin/cmd/kube-scheduler/app/server.go @@ -18,8 +18,11 @@ limitations under the License. package app import ( + "fmt" + "io/ioutil" "net" "net/http" + "os" "strconv" "github.com/GoogleCloudPlatform/kubernetes/pkg/api" @@ -30,6 +33,8 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/util" "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler" _ "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/algorithmprovider" + schedulerapi "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/api" + latestschedulerapi "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/api/latest" "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/factory" "github.com/golang/glog" @@ -42,6 +47,7 @@ type SchedulerServer struct { Address util.IP ClientConfig client.Config AlgorithmProvider string + PolicyConfigFile string } // NewSchedulerServer creates a new SchedulerServer with default parameters @@ -60,6 +66,7 @@ func (s *SchedulerServer) AddFlags(fs *pflag.FlagSet) { fs.Var(&s.Address, "address", "The IP address to serve on (set to 0.0.0.0 for all interfaces)") client.BindClientConfigFlags(fs, &s.ClientConfig) fs.StringVar(&s.AlgorithmProvider, "algorithm_provider", s.AlgorithmProvider, "The scheduling algorithm provider to use") + fs.StringVar(&s.PolicyConfigFile, "policy_config_file", s.PolicyConfigFile, "File with scheduler policy configuration") } // Run runs the specified SchedulerServer. This should never exit. @@ -78,6 +85,7 @@ func (s *SchedulerServer) Run(_ []string) error { if err != nil { glog.Fatalf("Failed to create scheduler configuration: %v", err) } + sched := scheduler.New(config) sched.Run() @@ -85,10 +93,29 @@ func (s *SchedulerServer) Run(_ []string) error { } func (s *SchedulerServer) createConfig(configFactory *factory.ConfigFactory) (*scheduler.Config, error) { + var policy schedulerapi.Policy + var configData []byte + + if _, err := os.Stat(s.PolicyConfigFile); err == nil { + configData, err = ioutil.ReadFile(s.PolicyConfigFile) + if err != nil { + return nil, fmt.Errorf("Unable to read policy config: %v", err) + } + //err = json.Unmarshal(configData, &policy) + err = latestschedulerapi.Codec.DecodeInto(configData, &policy) + if err != nil { + return nil, fmt.Errorf("Invalid configuration: %v", err) + } + + return configFactory.CreateFromConfig(policy) + } + + // if the config file isn't provided, use the specified (or default) provider // check of algorithm provider is registered and fail fast _, err := factory.GetAlgorithmProvider(s.AlgorithmProvider) if err != nil { return nil, err } + return configFactory.CreateFromProvider(s.AlgorithmProvider) } diff --git a/plugin/pkg/scheduler/algorithmprovider/affinity/affinity.go b/plugin/pkg/scheduler/algorithmprovider/affinity/affinity.go index ee023355d47..44f2bd26e6b 100644 --- a/plugin/pkg/scheduler/algorithmprovider/affinity/affinity.go +++ b/plugin/pkg/scheduler/algorithmprovider/affinity/affinity.go @@ -37,9 +37,9 @@ func affinityPredicates() util.StringSet { "PodFitsResources", "NoDiskConflict", // Ensures that all pods within the same service are hosted on minions within the same region as defined by the "region" label - factory.RegisterFitPredicate("ServiceAffinity", algorithm.NewServiceAffinityPredicate(factory.PodLister, factory.ServiceLister, factory.MinionLister, []string{"region"})), + factory.RegisterFitPredicate("RegionAffinity", algorithm.NewServiceAffinityPredicate(factory.PodLister, factory.ServiceLister, factory.MinionLister, []string{"region"})), // Fit is defined based on the presence of the "region" label on a minion, regardless of value. - factory.RegisterFitPredicate("NodeLabelPredicate", algorithm.NewNodeLabelPredicate(factory.MinionLister, []string{"region"}, true)), + factory.RegisterFitPredicate("RegionRequired", algorithm.NewNodeLabelPredicate(factory.MinionLister, []string{"region"}, true)), ) } @@ -48,8 +48,8 @@ func affinityPriorities() util.StringSet { "LeastRequestedPriority", "ServiceSpreadingPriority", // spreads pods belonging to the same service across minions in different zones - factory.RegisterPriorityFunction("ZoneSpreadingPriority", algorithm.NewServiceAntiAffinityPriority(factory.ServiceLister, "zone"), 2), + factory.RegisterPriorityFunction("ZoneSpread", algorithm.NewServiceAntiAffinityPriority(factory.ServiceLister, "zone"), 2), // Prioritize nodes based on the presence of the "zone" label on a minion, regardless of value. - factory.RegisterPriorityFunction("NodeLabelPriority", algorithm.NewNodeLabelPriority("zone", true), 1), + factory.RegisterPriorityFunction("ZonePreferred", algorithm.NewNodeLabelPriority("zone", true), 1), ) } diff --git a/plugin/pkg/scheduler/api/latest/latest.go b/plugin/pkg/scheduler/api/latest/latest.go new file mode 100644 index 00000000000..c4559130e41 --- /dev/null +++ b/plugin/pkg/scheduler/api/latest/latest.go @@ -0,0 +1,40 @@ +/* +Copyright 2014 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package latest + +import ( + "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/api/v1" +) + +// Version is the string that represents the current external default version. +const Version = "v1" + +// OldestVersion is the string that represents the oldest server version supported, +// for client code that wants to hardcode the lowest common denominator. +const OldestVersion = "v1" + +// Versions is the list of versions that are recognized in code. The order provided +// may be assumed to be least feature rich to most feature rich, and clients may +// choose to prefer the latter items in the list over the former items when presented +// with a set of versions to choose. +var Versions = []string{"v1"} + +// Codec is the default codec for serializing output that should use +// the latest supported version. Use this Codec when writing to +// disk, a data store that is not dynamically versioned, or in tests. +// This codec can decode any object that Kubernetes is aware of. +var Codec = v1.Codec diff --git a/plugin/pkg/scheduler/api/register.go b/plugin/pkg/scheduler/api/register.go new file mode 100644 index 00000000000..ae3c8ae3d87 --- /dev/null +++ b/plugin/pkg/scheduler/api/register.go @@ -0,0 +1,32 @@ +/* +Copyright 2014 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/GoogleCloudPlatform/kubernetes/pkg/runtime" +) + +// Scheme is the default instance of runtime.Scheme to which types in the Kubernetes API are already registered. +var Scheme = runtime.NewScheme() + +func init() { + Scheme.AddKnownTypes("", + &Policy{}, + ) +} + +func (*Policy) IsAnAPIObject() {} diff --git a/plugin/pkg/scheduler/api/types.go b/plugin/pkg/scheduler/api/types.go new file mode 100644 index 00000000000..7453eb199ab --- /dev/null +++ b/plugin/pkg/scheduler/api/types.go @@ -0,0 +1,73 @@ +/* +Copyright 2014 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/GoogleCloudPlatform/kubernetes/pkg/api" +) + +// Where possible, json tags match the cli argument names. +// Top level config objects and all values required for proper functioning are not "omitempty". Any truly optional piece of config is allowed to be omitted. + +type Policy struct { + api.TypeMeta `json:",inline"` + Predicates []PredicatePolicy `json:"predicates"` + Priorities []PriorityPolicy `json:"priorities"` +} + +type PredicatePolicy struct { + Name string `json:"name"` + Argument *PredicateArgument `json:"argument"` +} + +type PriorityPolicy struct { + Name string `json:"name"` + Weight int `json:"weight"` + Argument *PriorityArgument `json:"argument"` +} + +// PredicateArgument represents the arguments that the different types of predicates take. +// Only one of its members may be specified. +type PredicateArgument struct { + ServiceAffinity *ServiceAffinity `json:"serviceAffinity"` + LabelsPresence *LabelsPresence `json:"labelsPresence"` +} + +// PriorityArgument represents the arguments that the different types of priorities take. +// Only one of its members may be specified. +type PriorityArgument struct { + ServiceAntiAffinity *ServiceAntiAffinity `json:"serviceAntiAffinity"` + LabelPreference *LabelPreference `json:"labelPreference"` +} + +type ServiceAffinity struct { + Labels []string `json:"labels"` +} + +type LabelsPresence struct { + Labels []string `json:"labels"` + Presence bool `json:"presence"` +} + +type ServiceAntiAffinity struct { + Label string `json:"label"` +} + +type LabelPreference struct { + Label string `json:"label"` + Presence bool `json:"presence"` +} diff --git a/plugin/pkg/scheduler/api/v1/register.go b/plugin/pkg/scheduler/api/v1/register.go new file mode 100644 index 00000000000..b6718cff877 --- /dev/null +++ b/plugin/pkg/scheduler/api/v1/register.go @@ -0,0 +1,33 @@ +/* +Copyright 2014 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "github.com/GoogleCloudPlatform/kubernetes/pkg/runtime" + "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/api" +) + +// Codec encodes internal objects to the v1 scheme +var Codec = runtime.CodecFor(api.Scheme, "v1") + +func init() { + api.Scheme.AddKnownTypes("v1", + &Policy{}, + ) +} + +func (*Policy) IsAnAPIObject() {} diff --git a/plugin/pkg/scheduler/api/v1/types.go b/plugin/pkg/scheduler/api/v1/types.go new file mode 100644 index 00000000000..bbf5f0b8628 --- /dev/null +++ b/plugin/pkg/scheduler/api/v1/types.go @@ -0,0 +1,106 @@ +/* +Copyright 2014 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "github.com/GoogleCloudPlatform/kubernetes/pkg/api/v1beta3" +) + +// Where possible, json tags match the cli argument names. +// Top level config objects and all values required for proper functioning are not "omitempty". Any truly optional piece of config is allowed to be omitted. + +type Policy struct { + v1beta3.TypeMeta `json:",inline"` + // Predicates holds the information to configure the fit predicate functions + Predicates []PredicatePolicy `json:"predicates"` + // Priorities holds the information to configure the priority functions + Priorities []PriorityPolicy `json:"priorities"` +} + +type PredicatePolicy struct { + // Name is the identifier of the predicate policy + // For a custom predicate, the name can be user-defined + // For the Kubernetes provided predicates, the name is the identifier of the pre-defined predicate + Name string `json:"name"` + // Argument holds the parameters to configure the given predicate + Argument *PredicateArgument `json:"argument"` +} + +type PriorityPolicy struct { + // Name is the identifier of the priority policy + // For a custom priority, the name can be user-defined + // For the Kubernetes provided priority functions, the name is the identifier of the pre-defined priority function + Name string `json:"name"` + // Weight is the numeric multiplier for the minion scores that the priority function generates + Weight int `json:"weight"` + // Argument holds the parameters to configure the given priority function + Argument *PriorityArgument `json:"argument"` +} + +// PredicateArgument represents the arguments that the different types of predicates take +// Only one of its members may be specified +type PredicateArgument struct { + // ServiceAffinity is the predicate that provides affinity for pods belonging to a service + // It uses a label to identify minions that belong to the same "group" + ServiceAffinity *ServiceAffinity `json:"serviceAffinity"` + // LabelsPresence is the predicate that checks whether a particular minion has a certain label + // defined or not, regardless of value + LabelsPresence *LabelsPresence `json:"labelsPresence"` +} + +// PriorityArgument represents the arguments that the different types of priorities take. +// Only one of its members may be specified +type PriorityArgument struct { + // ServiceAntiAffinity is the priority function that ensures a good spread (anti-affinity) for pods belonging to a service + // It uses a label to identify minions that belong to the same "group" + ServiceAntiAffinity *ServiceAntiAffinity `json:"serviceAntiAffinity"` + // LabelPreference is the priority function that checks whether a particular minion has a certain label + // defined or not, regardless of value + LabelPreference *LabelPreference `json:"labelPreference"` +} + +// ServiceAffinity holds the parameters that are used to configure the corresponding predicate +type ServiceAffinity struct { + // Labels is the list of labels that identify minion "groups" + // All of the labels should match for the minion to be considered a fit for hosting the pod + Labels []string `json:"labels"` +} + +// LabelsPresence holds the parameters that are used to configure the corresponding predicate +type LabelsPresence struct { + // Labels is the list of labels that identify minion "groups" + // All of the labels should be either present (or absent) for the minion to be considered a fit for hosting the pod + Labels []string `json:"labels"` + // Presence is the boolean flag that indicates whether the labels should be present or absent from the minion + Presence bool `json:"presence"` +} + +// ServiceAntiAffinity holds the parameters that are used to configure the corresponding priority function +type ServiceAntiAffinity struct { + // Label is used to identify minion "groups" + Label string `json:"label"` +} + +// LabelPreference holds the parameters that are used to configure the corresponding priority function +type LabelPreference struct { + // Label is used to identify minion "groups" + Label string `json:"label"` + // Presence is a boolean flag + // If true, higher priority is given to minions that have the label + // If false, higher priority is given to minions that do not have the label + Presence bool `json:"presence"` +} diff --git a/plugin/pkg/scheduler/factory/factory.go b/plugin/pkg/scheduler/factory/factory.go index e7cb6f12457..9f4c4c6ec95 100644 --- a/plugin/pkg/scheduler/factory/factory.go +++ b/plugin/pkg/scheduler/factory/factory.go @@ -30,6 +30,7 @@ import ( algorithm "github.com/GoogleCloudPlatform/kubernetes/pkg/scheduler" "github.com/GoogleCloudPlatform/kubernetes/pkg/util" "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler" + schedulerapi "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/api" "github.com/golang/glog" ) @@ -80,6 +81,25 @@ func (f *ConfigFactory) CreateFromProvider(providerName string) (*scheduler.Conf return f.CreateFromKeys(provider.FitPredicateKeys, provider.PriorityFunctionKeys) } +// CreateFromConfig creates a scheduler from the configuration file +func (f *ConfigFactory) CreateFromConfig(policy schedulerapi.Policy) (*scheduler.Config, error) { + glog.V(2).Infof("creating scheduler from configuration: %v", policy) + + predicateKeys := util.NewStringSet() + for _, predicate := range policy.Predicates { + glog.V(2).Infof("Registering predicate: %s", predicate.Name) + predicateKeys.Insert(RegisterCustomPredicate(predicate)) + } + + priorityKeys := util.NewStringSet() + for _, priority := range policy.Priorities { + glog.V(2).Infof("Registering priority: %s", priority.Name) + priorityKeys.Insert(RegisterCustomPriorityFunction(priority)) + } + + return f.CreateFromKeys(predicateKeys, priorityKeys) +} + // CreateFromKeys creates a scheduler from a set of registered fit predicate keys and priority keys. func (f *ConfigFactory) CreateFromKeys(predicateKeys, priorityKeys util.StringSet) (*scheduler.Config, error) { glog.V(2).Infof("creating scheduler with fit predicates '%v' and priority functions '%v", predicateKeys, priorityKeys) diff --git a/plugin/pkg/scheduler/factory/plugins.go b/plugin/pkg/scheduler/factory/plugins.go index 4ec5c2c028a..a15ae65411b 100644 --- a/plugin/pkg/scheduler/factory/plugins.go +++ b/plugin/pkg/scheduler/factory/plugins.go @@ -23,6 +23,7 @@ import ( algorithm "github.com/GoogleCloudPlatform/kubernetes/pkg/scheduler" "github.com/GoogleCloudPlatform/kubernetes/pkg/util" + schedulerapi "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/api" "github.com/golang/glog" ) @@ -55,6 +56,33 @@ func RegisterFitPredicate(name string, predicate algorithm.FitPredicate) string return name } +// RegisterCustomPredicate registers a custom fit predicate with the algorithm registry. +// Returns the name, with which the predicate was registered. +func RegisterCustomPredicate(policy schedulerapi.PredicatePolicy) string { + var predicate algorithm.FitPredicate + var ok bool + + validatePredicateOrDie(policy) + + // generate the predicate function, if a custom type is requested + if policy.Argument != nil { + if policy.Argument.ServiceAffinity != nil { + predicate = algorithm.NewServiceAffinityPredicate(PodLister, ServiceLister, MinionLister, policy.Argument.ServiceAffinity.Labels) + } else if policy.Argument.LabelsPresence != nil { + predicate = algorithm.NewNodeLabelPredicate(MinionLister, policy.Argument.LabelsPresence.Labels, policy.Argument.LabelsPresence.Presence) + } + // check to see if a pre-defined predicate is requested + } else if predicate, ok = fitPredicateMap[policy.Name]; ok { + glog.V(2).Infof("Predicate type %s already registered, reusing.", policy.Name) + } + + if predicate == nil { + glog.Fatalf("Invalid configuration: Predicate type not found for %s", policy.Name) + } + + return RegisterFitPredicate(policy.Name, predicate) +} + // IsFitPredicateRegistered check is useful for testing providers. func IsFitPredicateRegistered(name string) bool { schedulerFactoryMutex.Lock() @@ -63,7 +91,7 @@ func IsFitPredicateRegistered(name string) bool { return ok } -// RegisterFitPredicate registers a priority function with the algorithm registry. Returns the name, +// RegisterPriorityFunction registers a priority function with the algorithm registry. Returns the name, // with which the function was registered. func RegisterPriorityFunction(name string, function algorithm.PriorityFunction, weight int) string { schedulerFactoryMutex.Lock() @@ -73,6 +101,32 @@ func RegisterPriorityFunction(name string, function algorithm.PriorityFunction, return name } +// RegisterCustomPriority registers a custom priority function with the algorithm registry. +// Returns the name, with which the priority function was registered. +func RegisterCustomPriorityFunction(policy schedulerapi.PriorityPolicy) string { + var priority algorithm.PriorityFunction + + validatePriorityOrDie(policy) + + // generate the priority function, if a custom priority is requested + if policy.Argument != nil { + if policy.Argument.ServiceAntiAffinity != nil { + priority = algorithm.NewServiceAntiAffinityPriority(ServiceLister, policy.Argument.ServiceAntiAffinity.Label) + } else if policy.Argument.LabelPreference != nil { + priority = algorithm.NewNodeLabelPriority(policy.Argument.LabelPreference.Label, policy.Argument.LabelPreference.Presence) + } + } else if priorityConfig, ok := priorityFunctionMap[policy.Name]; ok { + glog.V(2).Infof("Priority type %s already registered, reusing.", policy.Name) + priority = priorityConfig.Function + } + + if priority == nil { + glog.Fatalf("Invalid configuration: Priority type not found for %s", policy.Name) + } + + return RegisterPriorityFunction(policy.Name, priority, policy.Weight) +} + // IsPriorityFunctionRegistered check is useful for testing providers. func IsPriorityFunctionRegistered(name string) bool { schedulerFactoryMutex.Lock() @@ -91,6 +145,7 @@ func SetPriorityFunctionWeight(name string, weight int) { return } config.Weight = weight + priorityFunctionMap[name] = config } // RegisterAlgorithmProvider registers a new algorithm provider with the algorithm registry. This should @@ -157,3 +212,33 @@ func validateAlgorithmNameOrDie(name string) { glog.Fatalf("algorithm name %v does not match the name validation regexp \"%v\".", name, validName) } } + +func validatePredicateOrDie(predicate schedulerapi.PredicatePolicy) { + if predicate.Argument != nil { + numArgs := 0 + if predicate.Argument.ServiceAffinity != nil { + numArgs++ + } + if predicate.Argument.LabelsPresence != nil { + numArgs++ + } + if numArgs != 1 { + glog.Fatalf("Exactly 1 predicate argument is required") + } + } +} + +func validatePriorityOrDie(priority schedulerapi.PriorityPolicy) { + if priority.Argument != nil { + numArgs := 0 + if priority.Argument.ServiceAntiAffinity != nil { + numArgs++ + } + if priority.Argument.LabelPreference != nil { + numArgs++ + } + if numArgs != 1 { + glog.Fatalf("Exactly 1 priority argument is required") + } + } +}