From 8fc21710e7c47b42e5d92456e3dd18d4ab86266a Mon Sep 17 00:00:00 2001 From: Abdullah Gharaibeh Date: Tue, 24 Dec 2019 08:39:30 -0500 Subject: [PATCH] Added scheduler algorithm provider registery. --- pkg/scheduler/algorithmprovider/BUILD | 41 ++++- pkg/scheduler/algorithmprovider/registry.go | 160 ++++++++++++++++++ .../algorithmprovider/registry_test.go | 99 +++++++++++ 3 files changed, 297 insertions(+), 3 deletions(-) create mode 100644 pkg/scheduler/algorithmprovider/registry.go create mode 100644 pkg/scheduler/algorithmprovider/registry_test.go diff --git a/pkg/scheduler/algorithmprovider/BUILD b/pkg/scheduler/algorithmprovider/BUILD index 3359a83bd9f..6a1347c3724 100644 --- a/pkg/scheduler/algorithmprovider/BUILD +++ b/pkg/scheduler/algorithmprovider/BUILD @@ -8,18 +8,53 @@ load( go_library( name = "go_default_library", - srcs = ["plugins.go"], + srcs = [ + "plugins.go", + "registry.go", + ], importpath = "k8s.io/kubernetes/pkg/scheduler/algorithmprovider", - deps = ["//pkg/scheduler/algorithmprovider/defaults:go_default_library"], + deps = [ + "//pkg/features:go_default_library", + "//pkg/scheduler/algorithmprovider/defaults:go_default_library", + "//pkg/scheduler/apis/config:go_default_library", + "//pkg/scheduler/framework/plugins/defaultpodtopologyspread:go_default_library", + "//pkg/scheduler/framework/plugins/imagelocality:go_default_library", + "//pkg/scheduler/framework/plugins/interpodaffinity:go_default_library", + "//pkg/scheduler/framework/plugins/nodeaffinity:go_default_library", + "//pkg/scheduler/framework/plugins/nodename:go_default_library", + "//pkg/scheduler/framework/plugins/nodeports:go_default_library", + "//pkg/scheduler/framework/plugins/nodepreferavoidpods:go_default_library", + "//pkg/scheduler/framework/plugins/noderesources:go_default_library", + "//pkg/scheduler/framework/plugins/nodeunschedulable:go_default_library", + "//pkg/scheduler/framework/plugins/nodevolumelimits:go_default_library", + "//pkg/scheduler/framework/plugins/podtopologyspread:go_default_library", + "//pkg/scheduler/framework/plugins/tainttoleration:go_default_library", + "//pkg/scheduler/framework/plugins/volumebinding:go_default_library", + "//pkg/scheduler/framework/plugins/volumerestrictions:go_default_library", + "//pkg/scheduler/framework/plugins/volumezone:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], ) go_test( name = "go_default_test", - srcs = ["plugins_test.go"], + srcs = [ + "plugins_test.go", + "registry_test.go", + ], embed = [":go_default_library"], deps = [ "//pkg/scheduler:go_default_library", "//pkg/scheduler/apis/config:go_default_library", + "//pkg/scheduler/framework/plugins:go_default_library", + "//pkg/scheduler/framework/v1alpha1:go_default_library", + "//pkg/scheduler/nodeinfo/snapshot:go_default_library", + "//pkg/scheduler/volumebinder:go_default_library", + "//staging/src/k8s.io/client-go/informers:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", + "//vendor/github.com/google/go-cmp/cmp:go_default_library", ], ) diff --git a/pkg/scheduler/algorithmprovider/registry.go b/pkg/scheduler/algorithmprovider/registry.go new file mode 100644 index 00000000000..9f2f932fb15 --- /dev/null +++ b/pkg/scheduler/algorithmprovider/registry.go @@ -0,0 +1,160 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package algorithmprovider + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/runtime" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/klog" + "k8s.io/kubernetes/pkg/features" + schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultpodtopologyspread" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/imagelocality" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/interpodaffinity" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeaffinity" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodename" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeports" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodepreferavoidpods" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeunschedulable" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodevolumelimits" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/podtopologyspread" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/tainttoleration" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumebinding" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumerestrictions" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumezone" +) + +// ClusterAutoscalerProvider defines the default autoscaler provider +const ClusterAutoscalerProvider = "ClusterAutoscalerProvider" + +// Config the configuration of an algorithm provider. +type Config struct { + FrameworkPlugins *schedulerapi.Plugins + FrameworkPluginConfig []schedulerapi.PluginConfig +} + +// Registry is a collection of all available algorithm providers. +type Registry map[string]*Config + +// NewRegistry returns an algorithm provider registry instance. +func NewRegistry(hardPodAffinityWeight int64) Registry { + defaultConfig := getDefaultConfig(hardPodAffinityWeight) + applyFeatureGates(defaultConfig) + + caConfig := getClusterAutoscalerConfig(hardPodAffinityWeight) + applyFeatureGates(caConfig) + + return Registry{ + schedulerapi.SchedulerDefaultProviderName: defaultConfig, + ClusterAutoscalerProvider: caConfig, + } +} + +func getDefaultConfig(hardPodAffinityWeight int64) *Config { + return &Config{ + FrameworkPlugins: &schedulerapi.Plugins{ + PreFilter: &schedulerapi.PluginSet{ + Enabled: []schedulerapi.Plugin{ + {Name: noderesources.FitName}, + {Name: nodeports.Name}, + {Name: interpodaffinity.Name}, + }, + }, + Filter: &schedulerapi.PluginSet{ + Enabled: []schedulerapi.Plugin{ + {Name: nodeunschedulable.Name}, + {Name: noderesources.FitName}, + {Name: nodename.Name}, + {Name: nodeports.Name}, + {Name: nodeaffinity.Name}, + {Name: volumerestrictions.Name}, + {Name: tainttoleration.Name}, + {Name: nodevolumelimits.EBSName}, + {Name: nodevolumelimits.GCEPDName}, + {Name: nodevolumelimits.CSIName}, + {Name: nodevolumelimits.AzureDiskName}, + {Name: volumebinding.Name}, + {Name: volumezone.Name}, + {Name: interpodaffinity.Name}, + }, + }, + PostFilter: &schedulerapi.PluginSet{ + Enabled: []schedulerapi.Plugin{ + {Name: interpodaffinity.Name}, + {Name: tainttoleration.Name}, + }, + }, + Score: &schedulerapi.PluginSet{ + Enabled: []schedulerapi.Plugin{ + {Name: noderesources.BalancedAllocationName, Weight: 1}, + {Name: imagelocality.Name, Weight: 1}, + {Name: interpodaffinity.Name, Weight: 1}, + {Name: noderesources.LeastAllocatedName, Weight: 1}, + {Name: nodeaffinity.Name, Weight: 1}, + {Name: nodepreferavoidpods.Name, Weight: 10000}, + {Name: defaultpodtopologyspread.Name, Weight: 1}, + {Name: tainttoleration.Name, Weight: 1}, + }, + }, + }, + FrameworkPluginConfig: []schedulerapi.PluginConfig{ + { + Name: interpodaffinity.Name, + Args: runtime.Unknown{Raw: []byte(fmt.Sprintf(`{"hardPodAffinityWeight":%d}`, hardPodAffinityWeight))}, + }, + }, + } +} + +func getClusterAutoscalerConfig(hardPodAffinityWeight int64) *Config { + defaultConfig := getDefaultConfig(hardPodAffinityWeight) + caConfig := Config{ + FrameworkPlugins: &schedulerapi.Plugins{}, + } + defaultConfig.FrameworkPlugins.DeepCopyInto(caConfig.FrameworkPlugins) + caConfig.FrameworkPluginConfig = append([]schedulerapi.PluginConfig(nil), defaultConfig.FrameworkPluginConfig...) + + // Replace least with most requested. + for i := range caConfig.FrameworkPlugins.Score.Enabled { + if caConfig.FrameworkPlugins.Score.Enabled[i].Name == noderesources.LeastAllocatedName { + caConfig.FrameworkPlugins.Score.Enabled[i].Name = noderesources.MostAllocatedName + } + } + + return &caConfig +} + +func applyFeatureGates(config *Config) { + // Only add EvenPodsSpread if the feature is enabled. + if utilfeature.DefaultFeatureGate.Enabled(features.EvenPodsSpread) { + klog.Infof("Registering EvenPodsSpread predicate and priority function") + f := schedulerapi.Plugin{Name: podtopologyspread.Name} + config.FrameworkPlugins.PreFilter.Enabled = append(config.FrameworkPlugins.PreFilter.Enabled, f) + config.FrameworkPlugins.Filter.Enabled = append(config.FrameworkPlugins.Filter.Enabled, f) + s := schedulerapi.Plugin{Name: podtopologyspread.Name, Weight: 1} + config.FrameworkPlugins.Score.Enabled = append(config.FrameworkPlugins.Score.Enabled, s) + } + + // Prioritizes nodes that satisfy pod's resource limits + if utilfeature.DefaultFeatureGate.Enabled(features.ResourceLimitsPriorityFunction) { + klog.Infof("Registering resourcelimits priority function") + // TODO(ahg-g): append to config.FrameworkPlugins.Score.Enabled when available. + } +} diff --git a/pkg/scheduler/algorithmprovider/registry_test.go b/pkg/scheduler/algorithmprovider/registry_test.go new file mode 100644 index 00000000000..6d100eaac57 --- /dev/null +++ b/pkg/scheduler/algorithmprovider/registry_test.go @@ -0,0 +1,99 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package algorithmprovider + +import ( + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/kubernetes/pkg/scheduler" + "k8s.io/kubernetes/pkg/scheduler/apis/config" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins" + framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" + nodeinfosnapshot "k8s.io/kubernetes/pkg/scheduler/nodeinfo/snapshot" + "k8s.io/kubernetes/pkg/scheduler/volumebinder" +) + +func TestCompatibility(t *testing.T) { + testcases := []struct { + name string + provider string + wantPlugins map[string][]config.Plugin + }{ + { + name: "DefaultProvider", + provider: config.SchedulerDefaultProviderName, + }, + { + name: "ClusterAutoscalerProvider", + provider: ClusterAutoscalerProvider, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + client := fake.NewSimpleClientset() + informerFactory := informers.NewSharedInformerFactory(client, 0) + + sched, err := scheduler.New( + client, + informerFactory, + informerFactory.Core().V1().Pods(), + nil, + make(chan struct{}), + scheduler.WithAlgorithmSource(config.SchedulerAlgorithmSource{ + Provider: &tc.provider, + })) + if err != nil { + t.Fatalf("Error constructing: %v", err) + } + gotPlugins := sched.Framework.ListPlugins() + + volumeBinder := volumebinder.NewVolumeBinder( + client, + informerFactory.Core().V1().Nodes(), + informerFactory.Storage().V1().CSINodes(), + informerFactory.Core().V1().PersistentVolumeClaims(), + informerFactory.Core().V1().PersistentVolumes(), + informerFactory.Storage().V1().StorageClasses(), + time.Second, + ) + providerRegistry := NewRegistry(1) + config := providerRegistry[tc.provider] + fwk, err := framework.NewFramework( + plugins.NewInTreeRegistry(&plugins.RegistryArgs{ + VolumeBinder: volumeBinder, + }), + config.FrameworkPlugins, + config.FrameworkPluginConfig, + framework.WithClientSet(client), + framework.WithInformerFactory(informerFactory), + framework.WithSnapshotSharedLister(nodeinfosnapshot.NewEmptySnapshot()), + ) + if err != nil { + t.Fatalf("error initializing the scheduling framework: %v", err) + } + wantPlugins := fwk.ListPlugins() + + if diff := cmp.Diff(wantPlugins, gotPlugins); diff != "" { + t.Errorf("unexpected plugins diff (-want, +got): %s", diff) + } + }) + } +}