github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-29 22:46:12 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #96426 from damemi/print-scheduler-config

Browse Source 
Log defaulted kube-scheduler component config at startup
This commit is contained in:
Kubernetes Prow Robot 2020-11-12 16:03:01 -08:00 committed by GitHub
commit ae95984e88
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 293 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
cmd/kube-scheduler/app/options/configfile.go
View File

@ -19,6 +19,7 @@ package options
import (
"fmt"
"io/ioutil"
"k8s.io/klog/v2"
"os"
"k8s.io/apimachinery/pkg/runtime"
@ -48,8 +49,13 @@ func loadConfig(data []byte) (*kubeschedulerconfig.KubeSchedulerConfiguration, e
return nil, fmt.Errorf("couldn't decode as KubeSchedulerConfiguration, got %s: ", gvk)
}
// WriteConfigFile writes the config into the given file name as YAML.
func WriteConfigFile(fileName string, cfg *kubeschedulerconfig.KubeSchedulerConfiguration) error {
// LogOrWriteConfig logs the completed component config and writes it into the given file name as YAML, if either is enabled
func LogOrWriteConfig(fileName string, cfg *kubeschedulerconfig.KubeSchedulerConfiguration, completedProfiles []kubeschedulerconfig.KubeSchedulerProfile) error {
if !(klog.V(2).Enabled() || len(fileName) > 0) {
return nil
}
cfg.Profiles = completedProfiles
const mediaType = runtime.ContentTypeYAML
info, ok := runtime.SerializerInfoForMediaType(kubeschedulerscheme.Codecs.SupportedMediaTypes(), mediaType)
if !ok {
@ -57,15 +63,26 @@ func WriteConfigFile(fileName string, cfg *kubeschedulerconfig.KubeSchedulerConf
}
encoder := kubeschedulerscheme.Codecs.EncoderForVersion(info.Serializer, kubeschedulerconfigv1beta1.SchemeGroupVersion)
configFile, err := os.Create(fileName)
if err != nil {
return err
}
defer configFile.Close()
if err := encoder.Encode(cfg, configFile); err != nil {
return err
if klog.V(2).Enabled() {
bytes, err := runtime.Encode(encoder, cfg)
if err != nil {
return err
}
configString := string(bytes)
klog.Infof("Using component config:\n%+v\n", configString)
}
if len(fileName) > 0 {
configFile, err := os.Create(fileName)
if err != nil {
return err
}
defer configFile.Close()
if err := encoder.Encode(cfg, configFile); err != nil {
return err
}
klog.Infof("Wrote configuration to: %s\n", fileName)
os.Exit(0)
}
return nil
}

16
cmd/kube-scheduler/app/server.go
View File

@ -128,14 +128,6 @@ func runCommand(cmd *cobra.Command, opts *options.Options, registryOptions ...Op
return err
}
if len(opts.WriteConfigTo) > 0 {
if err := options.WriteConfigFile(opts.WriteConfigTo, &cc.ComponentConfig); err != nil {
return err
}
klog.Infof("Wrote configuration to: %s\n", opts.WriteConfigTo)
return nil
}
return Run(ctx, cc, sched)
}
@ -310,6 +302,7 @@ func Setup(ctx context.Context, opts *options.Options, outOfTreeRegistryOptions
}
recorderFactory := getRecorderFactory(&cc)
completedProfiles := make([]kubeschedulerconfig.KubeSchedulerProfile, 0)
// Create the scheduler.
sched, err := scheduler.New(cc.Client,
cc.InformerFactory,
@ -323,10 +316,17 @@ func Setup(ctx context.Context, opts *options.Options, outOfTreeRegistryOptions
scheduler.WithPodInitialBackoffSeconds(cc.ComponentConfig.PodInitialBackoffSeconds),
scheduler.WithExtenders(cc.ComponentConfig.Extenders...),
scheduler.WithParallelism(cc.ComponentConfig.Parallelism),
scheduler.WithBuildFrameworkCapturer(func(profile kubeschedulerconfig.KubeSchedulerProfile) {
// Profiles are processed during Framework instantiation to set default plugins and configurations. Capturing them for logging
completedProfiles = append(completedProfiles, profile)
}),
)
if err != nil {
return nil, nil, err
}
if err := options.LogOrWriteConfig(opts.WriteConfigTo, &cc.ComponentConfig, completedProfiles); err != nil {
return nil, nil, err
}
return &cc, sched, nil
}

2
pkg/scheduler/BUILD
View File

@ -66,7 +66,9 @@ go_test(
"//pkg/scheduler/framework:go_default_library",
"//pkg/scheduler/framework/plugins:go_default_library",
"//pkg/scheduler/framework/plugins/defaultbinder:go_default_library",
"//pkg/scheduler/framework/plugins/defaultpreemption:go_default_library",
"//pkg/scheduler/framework/plugins/interpodaffinity:go_default_library",
"//pkg/scheduler/framework/plugins/nodeaffinity:go_default_library",
"//pkg/scheduler/framework/plugins/nodelabel:go_default_library",
"//pkg/scheduler/framework/plugins/nodeports:go_default_library",
"//pkg/scheduler/framework/plugins/noderesources:go_default_library",

169
pkg/scheduler/apis/config/testing/compatibility_test.go
View File

@ -1593,6 +1593,48 @@ func TestPluginsConfigurationCompatibility(t *testing.T) {
"PreBindPlugin": {{Name: "VolumeBinding"}},
"BindPlugin": {{Name: "DefaultBinder"}},
}
defaultPluginConfigs := []config.PluginConfig{
{
Name: "DefaultPreemption",
Args: &config.DefaultPreemptionArgs{
MinCandidateNodesPercentage: 10,
MinCandidateNodesAbsolute: 100,
},
},
{
Name: "InterPodAffinity",
Args: &config.InterPodAffinityArgs{
HardPodAffinityWeight: 1,
},
},
{
Name: "NodeAffinity",
Args: &config.NodeAffinityArgs{},
},
{
Name: "NodeResourcesFit",
Args: &config.NodeResourcesFitArgs{},
},
{
Name: "NodeResourcesLeastAllocated",
Args: &config.NodeResourcesLeastAllocatedArgs{
Resources: []config.ResourceSpec{
{Name: "cpu", Weight: 1},
{Name: "memory", Weight: 1},
},
},
},
{
Name: "PodTopologySpread",
Args: &config.PodTopologySpreadArgs{DefaultingType: config.SystemDefaulting},
},
{
Name: "VolumeBinding",
Args: &config.VolumeBindingArgs{
BindTimeoutSeconds: 600,
},
},
}
testcases := []struct {
name string
@ -1604,11 +1646,76 @@ func TestPluginsConfigurationCompatibility(t *testing.T) {
{
name: "default plugins",
wantPlugins: defaultPlugins,
wantPluginConfig: nil,
wantPluginConfig: defaultPluginConfigs,
},
{
name: "default plugins with customized plugin config",
wantPlugins: defaultPlugins,
name: "in-tree plugins with customized plugin config",
plugins: config.Plugins{
Filter: &config.PluginSet{
Enabled: []config.Plugin{
{Name: "NodeLabel"},
{Name: "ServiceAffinity"},
},
},
Score: &config.PluginSet{
Enabled: []config.Plugin{
{Name: "RequestedToCapacityRatio"},
},
},
},
wantPlugins: map[string][]config.Plugin{
"QueueSortPlugin": {
{Name: "PrioritySort"},
},
"PreFilterPlugin": {
{Name: "NodeResourcesFit"},
{Name: "NodePorts"},
{Name: "PodTopologySpread"},
{Name: "InterPodAffinity"},
{Name: "VolumeBinding"},
},
"FilterPlugin": {
{Name: "NodeUnschedulable"},
{Name: "NodeName"},
{Name: "TaintToleration"},
{Name: "NodeAffinity"},
{Name: "NodePorts"},
{Name: "NodeResourcesFit"},
{Name: "VolumeRestrictions"},
{Name: "EBSLimits"},
{Name: "GCEPDLimits"},
{Name: "NodeVolumeLimits"},
{Name: "AzureDiskLimits"},
{Name: "VolumeBinding"},
{Name: "VolumeZone"},
{Name: "PodTopologySpread"},
{Name: "InterPodAffinity"},
{Name: "NodeLabel"},
{Name: "ServiceAffinity"},
},
"PostFilterPlugin": {
{Name: "DefaultPreemption"},
},
"PreScorePlugin": {
{Name: "InterPodAffinity"},
{Name: "PodTopologySpread"},
{Name: "TaintToleration"},
},
"ScorePlugin": {
{Name: "NodeResourcesBalancedAllocation", Weight: 1},
{Name: "ImageLocality", Weight: 1},
{Name: "InterPodAffinity", Weight: 1},
{Name: "NodeResourcesLeastAllocated", Weight: 1},
{Name: "NodeAffinity", Weight: 1},
{Name: "NodePreferAvoidPods", Weight: 10000},
{Name: "PodTopologySpread", Weight: 2},
{Name: "TaintToleration", Weight: 1},
{Name: "RequestedToCapacityRatio", Weight: 1},
},
"ReservePlugin": {{Name: "VolumeBinding"}},
"PreBindPlugin": {{Name: "VolumeBinding"}},
"BindPlugin": {{Name: "DefaultBinder"}},
},
pluginConfig: []config.PluginConfig{
{
Name: "NodeResourcesFit",
@ -1675,12 +1782,47 @@ func TestPluginsConfigurationCompatibility(t *testing.T) {
},
},
wantPluginConfig: []config.PluginConfig{
{
Name: "DefaultPreemption",
Args: &config.DefaultPreemptionArgs{
MinCandidateNodesPercentage: 10,
MinCandidateNodesAbsolute: 100,
},
},
{
Name: "InterPodAffinity",
Args: &config.InterPodAffinityArgs{
HardPodAffinityWeight: 100,
},
},
{
Name: "NodeAffinity",
Args: &config.NodeAffinityArgs{},
},
{
Name: "NodeLabel",
Args: &config.NodeLabelArgs{
PresentLabels: []string{"foo", "bar"},
AbsentLabels: []string{"apple"},
PresentLabelsPreference: []string{"dog"},
AbsentLabelsPreference: []string{"cat"},
},
},
{
Name: "NodeResourcesFit",
Args: &config.NodeResourcesFitArgs{
IgnoredResources: []string{"foo", "bar"},
},
},
{
Name: "NodeResourcesLeastAllocated",
Args: &config.NodeResourcesLeastAllocatedArgs{
Resources: []config.ResourceSpec{
{Name: "cpu", Weight: 1},
{Name: "memory", Weight: 1},
},
},
},
{
Name: "PodTopologySpread",
Args: &config.PodTopologySpreadArgs{
@ -1710,21 +1852,6 @@ func TestPluginsConfigurationCompatibility(t *testing.T) {
},
},
},
{
Name: "InterPodAffinity",
Args: &config.InterPodAffinityArgs{
HardPodAffinityWeight: 100,
},
},
{
Name: "NodeLabel",
Args: &config.NodeLabelArgs{
PresentLabels: []string{"foo", "bar"},
AbsentLabels: []string{"apple"},
PresentLabelsPreference: []string{"dog"},
AbsentLabelsPreference: []string{"cat"},
},
},
{
Name: "ServiceAffinity",
Args: &config.ServiceAffinityArgs{
@ -1864,6 +1991,7 @@ func TestPluginsConfigurationCompatibility(t *testing.T) {
},
PreScore: &config.PluginSet{
Enabled: []config.Plugin{
{Name: "PodTopologySpread"},
{Name: "TaintToleration"},
{Name: "SelectorSpread"},
{Name: "InterPodAffinity"},
@ -1874,6 +2002,7 @@ func TestPluginsConfigurationCompatibility(t *testing.T) {
},
Score: &config.PluginSet{
Enabled: []config.Plugin{
{Name: "PodTopologySpread", Weight: 24},
{Name: "TaintToleration", Weight: 24},
{Name: "SelectorSpread", Weight: 24},
{Name: "NodePreferAvoidPods", Weight: 24},
@ -1921,11 +2050,13 @@ func TestPluginsConfigurationCompatibility(t *testing.T) {
{Name: "DefaultPreemption"},
},
"PreScorePlugin": {
{Name: "PodTopologySpread"},
{Name: "TaintToleration"},
{Name: "SelectorSpread"},
{Name: "InterPodAffinity"},
},
"ScorePlugin": {
{Name: "PodTopologySpread", Weight: 24},
{Name: "TaintToleration", Weight: 24},
{Name: "SelectorSpread", Weight: 24},
{Name: "NodePreferAvoidPods", Weight: 24},
@ -1939,7 +2070,7 @@ func TestPluginsConfigurationCompatibility(t *testing.T) {
"PreBindPlugin": {{Name: "VolumeBinding"}},
"BindPlugin": {{Name: "DefaultBinder"}},
},
wantPluginConfig: nil,
wantPluginConfig: defaultPluginConfigs,
},
}
for _, tc := range testcases {

6
pkg/scheduler/factory.go
View File

@ -138,12 +138,8 @@ func (c *Configurator) create() (*Scheduler, error) {
frameworkruntime.WithSnapshotSharedLister(c.nodeInfoSnapshot),
frameworkruntime.WithRunAllFilters(c.alwaysCheckAllPredicates),
frameworkruntime.WithPodNominator(nominator),
frameworkruntime.WithCaptureProfile(frameworkruntime.CaptureProfile(c.frameworkCapturer)),
)
for _, p := range c.profiles {
if c.frameworkCapturer != nil {
c.frameworkCapturer(p)
}
}
if err != nil {
return nil, fmt.Errorf("initializing profiles: %v", err)
}

99
pkg/scheduler/factory_test.go
View File

@ -38,11 +38,14 @@ import (
schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config"
"k8s.io/kubernetes/pkg/scheduler/framework"
frameworkplugins "k8s.io/kubernetes/pkg/scheduler/framework/plugins"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultpreemption"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/interpodaffinity"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeaffinity"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodelabel"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/podtopologyspread"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/serviceaffinity"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumebinding"
frameworkruntime "k8s.io/kubernetes/pkg/scheduler/framework/runtime"
internalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache"
internalqueue "k8s.io/kubernetes/pkg/scheduler/internal/queue"
@ -105,10 +108,46 @@ func TestCreateFromConfig(t *testing.T) {
"apiVersion" : "v1"
}`),
wantPluginConfig: []schedulerapi.PluginConfig{
{
Name: defaultpreemption.Name,
Args: &schedulerapi.DefaultPreemptionArgs{
MinCandidateNodesPercentage: 10,
MinCandidateNodesAbsolute: 100,
},
},
{
Name: interpodaffinity.Name,
Args: &schedulerapi.InterPodAffinityArgs{
HardPodAffinityWeight: 1,
},
},
{
Name: nodeaffinity.Name,
Args: &schedulerapi.NodeAffinityArgs{},
},
{
Name: noderesources.FitName,
Args: &schedulerapi.NodeResourcesFitArgs{},
},
{
Name: noderesources.LeastAllocatedName,
Args: &schedulerapi.NodeResourcesLeastAllocatedArgs{
Resources: []schedulerapi.ResourceSpec{
{Name: "cpu", Weight: 1},
{Name: "memory", Weight: 1},
},
},
},
{
Name: podtopologyspread.Name,
Args: &schedulerapi.PodTopologySpreadArgs{DefaultingType: schedulerapi.SystemDefaulting},
},
{
Name: volumebinding.Name,
Args: &schedulerapi.VolumeBindingArgs{
BindTimeoutSeconds: 600,
},
},
},
wantPlugins: &schedulerapi.Plugins{
QueueSort: &schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "PrioritySort"}}},
@ -200,6 +239,23 @@ func TestCreateFromConfig(t *testing.T) {
]
}`),
wantPluginConfig: []schedulerapi.PluginConfig{
{
Name: defaultpreemption.Name,
Args: &schedulerapi.DefaultPreemptionArgs{
MinCandidateNodesPercentage: 10,
MinCandidateNodesAbsolute: 100,
},
},
{
Name: interpodaffinity.Name,
Args: &schedulerapi.InterPodAffinityArgs{
HardPodAffinityWeight: 1,
},
},
{
Name: nodeaffinity.Name,
Args: &schedulerapi.NodeAffinityArgs{},
},
{
Name: nodelabel.Name,
Args: &schedulerapi.NodeLabelArgs{
@ -209,13 +265,6 @@ func TestCreateFromConfig(t *testing.T) {
AbsentLabelsPreference: []string{"l2"},
},
},
{
Name: serviceaffinity.Name,
Args: &schedulerapi.ServiceAffinityArgs{
AffinityLabels: []string{"zone", "foo"},
AntiAffinityLabelsPreference: []string{"rack", "zone"},
},
},
{
Name: noderesources.RequestedToCapacityRatioName,
Args: &schedulerapi.RequestedToCapacityRatioArgs{
@ -226,6 +275,13 @@ func TestCreateFromConfig(t *testing.T) {
Resources: []schedulerapi.ResourceSpec{},
},
},
{
Name: serviceaffinity.Name,
Args: &schedulerapi.ServiceAffinityArgs{
AffinityLabels: []string{"zone", "foo"},
AntiAffinityLabelsPreference: []string{"rack", "zone"},
},
},
},
wantPlugins: &schedulerapi.Plugins{
QueueSort: &schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "PrioritySort"}}},
@ -262,31 +318,20 @@ func TestCreateFromConfig(t *testing.T) {
"kind" : "Policy",
"apiVersion" : "v1",
"predicates" : [
{"name" : "TestZoneAffinity", "argument" : {"serviceAffinity" : {"labels" : ["zone"]}}},
{"name" : "TestRequireZone", "argument" : {"labelsPresence" : {"labels" : ["zone"], "presence" : true}}},
{"name" : "PodFitsResources"},
{"name" : "PodFitsHostPorts"}
],
"priorities" : [
{"name" : "RackSpread", "weight" : 3, "argument" : {"serviceAntiAffinity" : {"label" : "rack"}}},
{"name" : "NodeAffinityPriority", "weight" : 2},
{"name" : "ImageLocalityPriority", "weight" : 1},
{"name" : "InterPodAffinityPriority", "weight" : 1}
],
"hardPodAffinitySymmetricWeight" : 10
}`),
wantPluginConfig: []schedulerapi.PluginConfig{
{
Name: nodelabel.Name,
Args: &schedulerapi.NodeLabelArgs{
PresentLabels: []string{"zone"},
},
},
{
Name: serviceaffinity.Name,
Args: &schedulerapi.ServiceAffinityArgs{
AffinityLabels: []string{"zone"},
AntiAffinityLabelsPreference: []string{"rack"},
Name: defaultpreemption.Name,
Args: &schedulerapi.DefaultPreemptionArgs{
MinCandidateNodesPercentage: 10,
MinCandidateNodesAbsolute: 100,
},
},
{
@ -295,13 +340,16 @@ func TestCreateFromConfig(t *testing.T) {
HardPodAffinityWeight: 10,
},
},
{
Name: "NodeResourcesFit",
Args: &schedulerapi.NodeResourcesFitArgs{},
},
},
wantPlugins: &schedulerapi.Plugins{
QueueSort: &schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "PrioritySort"}}},
PreFilter: &schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{
{Name: "NodePorts"},
{Name: "NodeResourcesFit"},
{Name: "ServiceAffinity"},
}},
Filter: &schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{
@ -309,18 +357,13 @@ func TestCreateFromConfig(t *testing.T) {
{Name: "NodePorts"},
{Name: "NodeResourcesFit"},
{Name: "TaintToleration"},
{Name: "NodeLabel"},
{Name: "ServiceAffinity"},
},
},
PostFilter: &schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "DefaultPreemption"}}},
PreScore: &schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "InterPodAffinity"}}},
Score: &schedulerapi.PluginSet{
Enabled: []schedulerapi.Plugin{
{Name: "ImageLocality", Weight: 1},
{Name: "InterPodAffinity", Weight: 1},
{Name: "NodeAffinity", Weight: 2},
{Name: "ServiceAffinity", Weight: 3},
},
},
Reserve: &schedulerapi.PluginSet{},

34
pkg/scheduler/framework/runtime/framework.go
View File

@ -20,6 +20,7 @@ import (
"context"
"fmt"
"reflect"
"sort"
"time"
v1 "k8s.io/api/core/v1"
@ -130,6 +131,7 @@ type frameworkOptions struct {
podNominator framework.PodNominator
extenders []framework.Extender
runAllFilters bool
captureProfile CaptureProfile
}
// Option for the frameworkImpl.
@ -199,6 +201,16 @@ func WithExtenders(extenders []framework.Extender) Option {
}
}
// CaptureProfile is a callback to capture a finalized profile.
type CaptureProfile func(config.KubeSchedulerProfile)
// WithCaptureProfile sets a callback to capture the finalized profile.
func WithCaptureProfile(c CaptureProfile) Option {
return func(o *frameworkOptions) {
o.captureProfile = c
}
}
var defaultFrameworkOptions = frameworkOptions{
metricsRecorder: newMetricsRecorder(1000, time.Second),
}
@ -258,6 +270,11 @@ func NewFramework(r Registry, plugins *config.Plugins, args []config.PluginConfi
}
pluginConfig[name] = args[i].Args
}
outputProfile := config.KubeSchedulerProfile{
SchedulerName: f.profileName,
Plugins: plugins,
PluginConfig: make([]config.PluginConfig, 0, len(pg)),
}
pluginsMap := make(map[string]framework.Plugin)
var totalPriority int64
@ -271,6 +288,12 @@ func NewFramework(r Registry, plugins *config.Plugins, args []config.PluginConfi
if err != nil {
return nil, fmt.Errorf("getting args for Plugin %q: %w", name, err)
}
if args != nil {
outputProfile.PluginConfig = append(outputProfile.PluginConfig, config.PluginConfig{
Name: name,
Args: args,
})
}
p, err := factory(args, f)
if err != nil {
return nil, fmt.Errorf("initializing plugin %q: %w", name, err)
@ -314,6 +337,17 @@ func NewFramework(r Registry, plugins *config.Plugins, args []config.PluginConfi
return nil, fmt.Errorf("at least one bind plugin is needed")
}
if options.captureProfile != nil {
if len(outputProfile.PluginConfig) != 0 {
sort.Slice(outputProfile.PluginConfig, func(i, j int) bool {
return outputProfile.PluginConfig[i].Name < outputProfile.PluginConfig[j].Name
})
} else {
outputProfile.PluginConfig = nil
}
options.captureProfile(outputProfile)
}
return f, nil
}