github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-30 06:54:01 +00:00
Code Issues Projects Releases Wiki Activity

Capture defaulted plugin configs from framework

Browse Source 
Some plugin configs are not produced until the framework is instantiated. Add a callback to capture them inside the framework constructor.

Change-Id: Id3f709b6461ccd0eafec7d21412cda093d9c4645
This commit is contained in:
Aldo Culquicondor 2020-11-10 19:41:18 -05:00
parent fe9fd060c9
commit d6ef6d6e43
5 changed files with 238 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
pkg/scheduler/BUILD
View File

@ -66,6 +66,7 @@ go_test(
"//pkg/scheduler/framework/plugins:go_default_library",
"//pkg/scheduler/framework/plugins/defaultbinder:go_default_library",
"//pkg/scheduler/framework/plugins/interpodaffinity:go_default_library",
"//pkg/scheduler/framework/plugins/nodeaffinity:go_default_library",
"//pkg/scheduler/framework/plugins/nodelabel:go_default_library",
"//pkg/scheduler/framework/plugins/nodeports:go_default_library",
"//pkg/scheduler/framework/plugins/noderesources:go_default_library",

169
pkg/scheduler/apis/config/testing/compatibility_test.go
View File

@ -1578,6 +1578,48 @@ func TestPluginsConfigurationCompatibility(t *testing.T) {
"PreBindPlugin": {{Name: "VolumeBinding"}},
"BindPlugin": {{Name: "DefaultBinder"}},
}
defaultPluginConfigs := []config.PluginConfig{
{
Name: "DefaultPreemption",
Args: &config.DefaultPreemptionArgs{
MinCandidateNodesPercentage: 10,
MinCandidateNodesAbsolute: 100,
},
},
{
Name: "InterPodAffinity",
Args: &config.InterPodAffinityArgs{
HardPodAffinityWeight: 1,
},
},
{
Name: "NodeAffinity",
Args: &config.NodeAffinityArgs{},
},
{
Name: "NodeResourcesFit",
Args: &config.NodeResourcesFitArgs{},
},
{
Name: "NodeResourcesLeastAllocated",
Args: &config.NodeResourcesLeastAllocatedArgs{
Resources: []config.ResourceSpec{
{Name: "cpu", Weight: 1},
{Name: "memory", Weight: 1},
},
},
},
{
Name: "PodTopologySpread",
Args: &config.PodTopologySpreadArgs{DefaultingType: config.SystemDefaulting},
},
{
Name: "VolumeBinding",
Args: &config.VolumeBindingArgs{
BindTimeoutSeconds: 600,
},
},
}
testcases := []struct {
name string
@ -1589,11 +1631,76 @@ func TestPluginsConfigurationCompatibility(t *testing.T) {
{
name: "default plugins",
wantPlugins: defaultPlugins,
wantPluginConfig: nil,
wantPluginConfig: defaultPluginConfigs,
},
{
name: "default plugins with customized plugin config",
wantPlugins: defaultPlugins,
name: "in-tree plugins with customized plugin config",
plugins: config.Plugins{
Filter: &config.PluginSet{
Enabled: []config.Plugin{
{Name: "NodeLabel"},
{Name: "ServiceAffinity"},
},
},
Score: &config.PluginSet{
Enabled: []config.Plugin{
{Name: "RequestedToCapacityRatio"},
},
},
},
wantPlugins: map[string][]config.Plugin{
"QueueSortPlugin": {
{Name: "PrioritySort"},
},
"PreFilterPlugin": {
{Name: "NodeResourcesFit"},
{Name: "NodePorts"},
{Name: "PodTopologySpread"},
{Name: "InterPodAffinity"},
{Name: "VolumeBinding"},
},
"FilterPlugin": {
{Name: "NodeUnschedulable"},
{Name: "NodeName"},
{Name: "TaintToleration"},
{Name: "NodeAffinity"},
{Name: "NodePorts"},
{Name: "NodeResourcesFit"},
{Name: "VolumeRestrictions"},
{Name: "EBSLimits"},
{Name: "GCEPDLimits"},
{Name: "NodeVolumeLimits"},
{Name: "AzureDiskLimits"},
{Name: "VolumeBinding"},
{Name: "VolumeZone"},
{Name: "PodTopologySpread"},
{Name: "InterPodAffinity"},
{Name: "NodeLabel"},
{Name: "ServiceAffinity"},
},
"PostFilterPlugin": {
{Name: "DefaultPreemption"},
},
"PreScorePlugin": {
{Name: "InterPodAffinity"},
{Name: "PodTopologySpread"},
{Name: "TaintToleration"},
},
"ScorePlugin": {
{Name: "NodeResourcesBalancedAllocation", Weight: 1},
{Name: "ImageLocality", Weight: 1},
{Name: "InterPodAffinity", Weight: 1},
{Name: "NodeResourcesLeastAllocated", Weight: 1},
{Name: "NodeAffinity", Weight: 1},
{Name: "NodePreferAvoidPods", Weight: 10000},
{Name: "PodTopologySpread", Weight: 2},
{Name: "TaintToleration", Weight: 1},
{Name: "RequestedToCapacityRatio", Weight: 1},
},
"ReservePlugin": {{Name: "VolumeBinding"}},
"PreBindPlugin": {{Name: "VolumeBinding"}},
"BindPlugin": {{Name: "DefaultBinder"}},
},
pluginConfig: []config.PluginConfig{
{
Name: "NodeResourcesFit",
@ -1660,12 +1767,47 @@ func TestPluginsConfigurationCompatibility(t *testing.T) {
},
},
wantPluginConfig: []config.PluginConfig{
{
Name: "DefaultPreemption",
Args: &config.DefaultPreemptionArgs{
MinCandidateNodesPercentage: 10,
MinCandidateNodesAbsolute: 100,
},
},
{
Name: "InterPodAffinity",
Args: &config.InterPodAffinityArgs{
HardPodAffinityWeight: 100,
},
},
{
Name: "NodeAffinity",
Args: &config.NodeAffinityArgs{},
},
{
Name: "NodeLabel",
Args: &config.NodeLabelArgs{
PresentLabels: []string{"foo", "bar"},
AbsentLabels: []string{"apple"},
PresentLabelsPreference: []string{"dog"},
AbsentLabelsPreference: []string{"cat"},
},
},
{
Name: "NodeResourcesFit",
Args: &config.NodeResourcesFitArgs{
IgnoredResources: []string{"foo", "bar"},
},
},
{
Name: "NodeResourcesLeastAllocated",
Args: &config.NodeResourcesLeastAllocatedArgs{
Resources: []config.ResourceSpec{
{Name: "cpu", Weight: 1},
{Name: "memory", Weight: 1},
},
},
},
{
Name: "PodTopologySpread",
Args: &config.PodTopologySpreadArgs{
@ -1695,21 +1837,6 @@ func TestPluginsConfigurationCompatibility(t *testing.T) {
},
},
},
{
Name: "InterPodAffinity",
Args: &config.InterPodAffinityArgs{
HardPodAffinityWeight: 100,
},
},
{
Name: "NodeLabel",
Args: &config.NodeLabelArgs{
PresentLabels: []string{"foo", "bar"},
AbsentLabels: []string{"apple"},
PresentLabelsPreference: []string{"dog"},
AbsentLabelsPreference: []string{"cat"},
},
},
{
Name: "ServiceAffinity",
Args: &config.ServiceAffinityArgs{
@ -1849,6 +1976,7 @@ func TestPluginsConfigurationCompatibility(t *testing.T) {
},
PreScore: &config.PluginSet{
Enabled: []config.Plugin{
{Name: "PodTopologySpread"},
{Name: "TaintToleration"},
{Name: "SelectorSpread"},
{Name: "InterPodAffinity"},
@ -1859,6 +1987,7 @@ func TestPluginsConfigurationCompatibility(t *testing.T) {
},
Score: &config.PluginSet{
Enabled: []config.Plugin{
{Name: "PodTopologySpread", Weight: 24},
{Name: "TaintToleration", Weight: 24},
{Name: "SelectorSpread", Weight: 24},
{Name: "NodePreferAvoidPods", Weight: 24},
@ -1906,11 +2035,13 @@ func TestPluginsConfigurationCompatibility(t *testing.T) {
{Name: "DefaultPreemption"},
},
"PreScorePlugin": {
{Name: "PodTopologySpread"},
{Name: "TaintToleration"},
{Name: "SelectorSpread"},
{Name: "InterPodAffinity"},
},
"ScorePlugin": {
{Name: "PodTopologySpread", Weight: 24},
{Name: "TaintToleration", Weight: 24},
{Name: "SelectorSpread", Weight: 24},
{Name: "NodePreferAvoidPods", Weight: 24},
@ -1924,7 +2055,7 @@ func TestPluginsConfigurationCompatibility(t *testing.T) {
"PreBindPlugin": {{Name: "VolumeBinding"}},
"BindPlugin": {{Name: "DefaultBinder"}},
},
wantPluginConfig: nil,
wantPluginConfig: defaultPluginConfigs,
},
}
for _, tc := range testcases {

6
pkg/scheduler/factory.go
View File

@ -137,12 +137,8 @@ func (c *Configurator) create() (*Scheduler, error) {
frameworkruntime.WithSnapshotSharedLister(c.nodeInfoSnapshot),
frameworkruntime.WithRunAllFilters(c.alwaysCheckAllPredicates),
frameworkruntime.WithPodNominator(nominator),
frameworkruntime.WithCaptureProfile(frameworkruntime.CaptureProfile(c.frameworkCapturer)),
)
for _, p := range c.profiles {
if c.frameworkCapturer != nil {
c.frameworkCapturer(p)
}
}
if err != nil {
return nil, fmt.Errorf("initializing profiles: %v", err)
}

83
pkg/scheduler/factory_test.go
View File

@ -39,10 +39,12 @@ import (
"k8s.io/kubernetes/pkg/scheduler/framework"
frameworkplugins "k8s.io/kubernetes/pkg/scheduler/framework/plugins"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/interpodaffinity"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeaffinity"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodelabel"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/podtopologyspread"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/serviceaffinity"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumebinding"
frameworkruntime "k8s.io/kubernetes/pkg/scheduler/framework/runtime"
internalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache"
internalqueue "k8s.io/kubernetes/pkg/scheduler/internal/queue"
@ -105,10 +107,39 @@ func TestCreateFromConfig(t *testing.T) {
"apiVersion" : "v1"
}`),
wantPluginConfig: []schedulerapi.PluginConfig{
{
Name: interpodaffinity.Name,
Args: &schedulerapi.InterPodAffinityArgs{
HardPodAffinityWeight: 1,
},
},
{
Name: nodeaffinity.Name,
Args: &schedulerapi.NodeAffinityArgs{},
},
{
Name: noderesources.FitName,
Args: &schedulerapi.NodeResourcesFitArgs{},
},
{
Name: noderesources.LeastAllocatedName,
Args: &schedulerapi.NodeResourcesLeastAllocatedArgs{
Resources: []schedulerapi.ResourceSpec{
{Name: "cpu", Weight: 1},
{Name: "memory", Weight: 1},
},
},
},
{
Name: podtopologyspread.Name,
Args: &schedulerapi.PodTopologySpreadArgs{DefaultingType: schedulerapi.SystemDefaulting},
},
{
Name: volumebinding.Name,
Args: &schedulerapi.VolumeBindingArgs{
BindTimeoutSeconds: 600,
},
},
},
wantPlugins: &schedulerapi.Plugins{
QueueSort: &schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "PrioritySort"}}},
@ -200,6 +231,16 @@ func TestCreateFromConfig(t *testing.T) {
]
}`),
wantPluginConfig: []schedulerapi.PluginConfig{
{
Name: interpodaffinity.Name,
Args: &schedulerapi.InterPodAffinityArgs{
HardPodAffinityWeight: 1,
},
},
{
Name: nodeaffinity.Name,
Args: &schedulerapi.NodeAffinityArgs{},
},
{
Name: nodelabel.Name,
Args: &schedulerapi.NodeLabelArgs{
@ -209,13 +250,6 @@ func TestCreateFromConfig(t *testing.T) {
AbsentLabelsPreference: []string{"l2"},
},
},
{
Name: serviceaffinity.Name,
Args: &schedulerapi.ServiceAffinityArgs{
AffinityLabels: []string{"zone", "foo"},
AntiAffinityLabelsPreference: []string{"rack", "zone"},
},
},
{
Name: noderesources.RequestedToCapacityRatioName,
Args: &schedulerapi.RequestedToCapacityRatioArgs{
@ -226,6 +260,13 @@ func TestCreateFromConfig(t *testing.T) {
Resources: []schedulerapi.ResourceSpec{},
},
},
{
Name: serviceaffinity.Name,
Args: &schedulerapi.ServiceAffinityArgs{
AffinityLabels: []string{"zone", "foo"},
AntiAffinityLabelsPreference: []string{"rack", "zone"},
},
},
},
wantPlugins: &schedulerapi.Plugins{
QueueSort: &schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "PrioritySort"}}},
@ -262,46 +303,31 @@ func TestCreateFromConfig(t *testing.T) {
"kind" : "Policy",
"apiVersion" : "v1",
"predicates" : [
{"name" : "TestZoneAffinity", "argument" : {"serviceAffinity" : {"labels" : ["zone"]}}},
{"name" : "TestRequireZone", "argument" : {"labelsPresence" : {"labels" : ["zone"], "presence" : true}}},
{"name" : "PodFitsResources"},
{"name" : "PodFitsHostPorts"}
],
"priorities" : [
{"name" : "RackSpread", "weight" : 3, "argument" : {"serviceAntiAffinity" : {"label" : "rack"}}},
{"name" : "NodeAffinityPriority", "weight" : 2},
{"name" : "ImageLocalityPriority", "weight" : 1},
{"name" : "InterPodAffinityPriority", "weight" : 1}
],
"hardPodAffinitySymmetricWeight" : 10
}`),
wantPluginConfig: []schedulerapi.PluginConfig{
{
Name: nodelabel.Name,
Args: &schedulerapi.NodeLabelArgs{
PresentLabels: []string{"zone"},
},
},
{
Name: serviceaffinity.Name,
Args: &schedulerapi.ServiceAffinityArgs{
AffinityLabels: []string{"zone"},
AntiAffinityLabelsPreference: []string{"rack"},
},
},
{
Name: interpodaffinity.Name,
Args: &schedulerapi.InterPodAffinityArgs{
HardPodAffinityWeight: 10,
},
},
{
Name: "NodeResourcesFit",
Args: &schedulerapi.NodeResourcesFitArgs{},
},
},
wantPlugins: &schedulerapi.Plugins{
QueueSort: &schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "PrioritySort"}}},
PreFilter: &schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{
{Name: "NodePorts"},
{Name: "NodeResourcesFit"},
{Name: "ServiceAffinity"},
}},
Filter: &schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{
@ -309,18 +335,13 @@ func TestCreateFromConfig(t *testing.T) {
{Name: "NodePorts"},
{Name: "NodeResourcesFit"},
{Name: "TaintToleration"},
{Name: "NodeLabel"},
{Name: "ServiceAffinity"},
},
},
PostFilter: &schedulerapi.PluginSet{},
PreScore: &schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "InterPodAffinity"}}},
Score: &schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{
{Name: "ImageLocality", Weight: 1},
{Name: "InterPodAffinity", Weight: 1},
{Name: "NodeAffinity", Weight: 2},
{Name: "ServiceAffinity", Weight: 3},
},
},
Reserve: &schedulerapi.PluginSet{},

34
pkg/scheduler/framework/runtime/framework.go
View File

@ -20,6 +20,7 @@ import (
"context"
"fmt"
"reflect"
"sort"
"time"
v1 "k8s.io/api/core/v1"
@ -130,6 +131,7 @@ type frameworkOptions struct {
podNominator framework.PodNominator
extenders []framework.Extender
runAllFilters bool
captureProfile CaptureProfile
}
// Option for the frameworkImpl.
@ -199,6 +201,16 @@ func WithExtenders(extenders []framework.Extender) Option {
}
}
// CaptureProfile is a callback to capture a finalized profile.
type CaptureProfile func(config.KubeSchedulerProfile)
// WithCaptureProfile sets a callback to capture the finalized profile.
func WithCaptureProfile(c CaptureProfile) Option {
return func(o *frameworkOptions) {
o.captureProfile = c
}
}
var defaultFrameworkOptions = frameworkOptions{
metricsRecorder: newMetricsRecorder(1000, time.Second),
}
@ -258,6 +270,11 @@ func NewFramework(r Registry, plugins *config.Plugins, args []config.PluginConfi
}
pluginConfig[name] = args[i].Args
}
outputProfile := config.KubeSchedulerProfile{
SchedulerName: f.profileName,
Plugins: plugins,
PluginConfig: make([]config.PluginConfig, 0, len(pg)),
}
pluginsMap := make(map[string]framework.Plugin)
var totalPriority int64
@ -271,6 +288,12 @@ func NewFramework(r Registry, plugins *config.Plugins, args []config.PluginConfi
if err != nil {
return nil, fmt.Errorf("getting args for Plugin %q: %w", name, err)
}
if args != nil {
outputProfile.PluginConfig = append(outputProfile.PluginConfig, config.PluginConfig{
Name: name,
Args: args,
})
}
p, err := factory(args, f)
if err != nil {
return nil, fmt.Errorf("initializing plugin %q: %w", name, err)
@ -314,6 +337,17 @@ func NewFramework(r Registry, plugins *config.Plugins, args []config.PluginConfi
return nil, fmt.Errorf("at least one bind plugin is needed")
}
if options.captureProfile != nil {
if len(outputProfile.PluginConfig) != 0 {
sort.Slice(outputProfile.PluginConfig, func(i, j int) bool {
return outputProfile.PluginConfig[i].Name < outputProfile.PluginConfig[j].Name
})
} else {
outputProfile.PluginConfig = nil
}
options.captureProfile(outputProfile)
}
return f, nil
}