/* Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package app import ( "context" "fmt" "net" "net/http" "net/http/httptest" "os" "path/filepath" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/spf13/pflag" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/util/feature" componentbaseconfig "k8s.io/component-base/config" "k8s.io/component-base/featuregate" featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kube-scheduler/config/v1beta3" "k8s.io/kubernetes/cmd/kube-scheduler/app/options" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/apis/config/testing/defaults" "k8s.io/kubernetes/pkg/scheduler/framework" ) func TestSetup(t *testing.T) { // temp dir tmpDir, err := os.MkdirTemp("", "scheduler-options") if err != nil { t.Fatal(err) } defer os.RemoveAll(tmpDir) // https server server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { w.WriteHeader(200) w.Write([]byte(`ok`)) })) defer server.Close() configKubeconfig := filepath.Join(tmpDir, "config.kubeconfig") if err := os.WriteFile(configKubeconfig, []byte(fmt.Sprintf(` apiVersion: v1 kind: Config clusters: - cluster: insecure-skip-tls-verify: true server: %s name: default contexts: - context: cluster: default user: default name: default current-context: default users: - name: default user: username: config `, server.URL)), os.FileMode(0600)); err != nil { t.Fatal(err) } // plugin config pluginConfigFilev1 := filepath.Join(tmpDir, "pluginv1.yaml") if err := os.WriteFile(pluginConfigFilev1, []byte(fmt.Sprintf(` apiVersion: kubescheduler.config.k8s.io/v1 kind: KubeSchedulerConfiguration clientConnection: kubeconfig: '%s' profiles: - plugins: multiPoint: enabled: - name: SchedulingGates - name: DefaultBinder - name: PrioritySort - name: DefaultPreemption - name: VolumeBinding - name: NodeResourcesFit - name: NodePorts - name: InterPodAffinity - name: TaintToleration disabled: - name: "*" preFilter: disabled: - name: VolumeBinding - name: InterPodAffinity filter: disabled: - name: VolumeBinding - name: InterPodAffinity - name: TaintToleration score: disabled: - name: VolumeBinding - name: NodeResourcesFit `, configKubeconfig)), os.FileMode(0600)); err != nil { t.Fatal(err) } // plugin config pluginConfigFilev1beta3 := filepath.Join(tmpDir, "pluginv1beta3.yaml") if err := os.WriteFile(pluginConfigFilev1beta3, []byte(fmt.Sprintf(` apiVersion: kubescheduler.config.k8s.io/v1beta3 kind: KubeSchedulerConfiguration clientConnection: kubeconfig: '%s' profiles: - plugins: multiPoint: enabled: - name: SchedulingGates - name: DefaultBinder - name: PrioritySort - name: DefaultPreemption - name: VolumeBinding - name: NodeResourcesFit - name: NodePorts - name: InterPodAffinity - name: TaintToleration disabled: - name: "*" preFilter: disabled: - name: VolumeBinding - name: InterPodAffinity filter: disabled: - name: VolumeBinding - name: InterPodAffinity - name: TaintToleration score: disabled: - name: VolumeBinding - name: NodeResourcesFit `, configKubeconfig)), os.FileMode(0600)); err != nil { t.Fatal(err) } // plugin config pluginConfigFilev1beta2 := filepath.Join(tmpDir, "pluginv1beta2.yaml") if err := os.WriteFile(pluginConfigFilev1beta2, []byte(fmt.Sprintf(` apiVersion: kubescheduler.config.k8s.io/v1beta2 kind: KubeSchedulerConfiguration clientConnection: kubeconfig: '%s' profiles: - plugins: preFilter: enabled: - name: NodeResourcesFit - name: NodePorts disabled: - name: "*" filter: enabled: - name: NodeResourcesFit - name: NodePorts disabled: - name: "*" preScore: enabled: - name: InterPodAffinity - name: TaintToleration disabled: - name: "*" score: enabled: - name: InterPodAffinity - name: TaintToleration disabled: - name: "*" `, configKubeconfig)), os.FileMode(0600)); err != nil { t.Fatal(err) } // out-of-tree plugin config v1 outOfTreePluginConfigFilev1 := filepath.Join(tmpDir, "outOfTreePluginv1.yaml") if err := os.WriteFile(outOfTreePluginConfigFilev1, []byte(fmt.Sprintf(` apiVersion: kubescheduler.config.k8s.io/v1 kind: KubeSchedulerConfiguration clientConnection: kubeconfig: '%s' profiles: - plugins: preFilter: enabled: - name: Foo filter: enabled: - name: Foo `, configKubeconfig)), os.FileMode(0600)); err != nil { t.Fatal(err) } // out-of-tree plugin config v1beta3 outOfTreePluginConfigFilev1beta3 := filepath.Join(tmpDir, "outOfTreePluginv1beta3.yaml") if err := os.WriteFile(outOfTreePluginConfigFilev1beta3, []byte(fmt.Sprintf(` apiVersion: kubescheduler.config.k8s.io/v1beta3 kind: KubeSchedulerConfiguration clientConnection: kubeconfig: '%s' profiles: - plugins: preFilter: enabled: - name: Foo filter: enabled: - name: Foo `, configKubeconfig)), os.FileMode(0600)); err != nil { t.Fatal(err) } // out-of-tree plugin config v1beta2 outOfTreePluginConfigFilev1beta2 := filepath.Join(tmpDir, "outOfTreePluginv1beta2.yaml") if err := os.WriteFile(outOfTreePluginConfigFilev1beta2, []byte(fmt.Sprintf(` apiVersion: kubescheduler.config.k8s.io/v1beta2 kind: KubeSchedulerConfiguration clientConnection: kubeconfig: '%s' profiles: - plugins: preFilter: enabled: - name: Foo filter: enabled: - name: Foo `, configKubeconfig)), os.FileMode(0600)); err != nil { t.Fatal(err) } // multiple profiles config multiProfilesConfig := filepath.Join(tmpDir, "multi-profiles.yaml") if err := os.WriteFile(multiProfilesConfig, []byte(fmt.Sprintf(` apiVersion: kubescheduler.config.k8s.io/v1 kind: KubeSchedulerConfiguration clientConnection: kubeconfig: '%s' profiles: - schedulerName: "profile-default-plugins" - schedulerName: "profile-disable-all-filter-and-score-plugins" plugins: preFilter: disabled: - name: "*" filter: disabled: - name: "*" postFilter: disabled: - name: "*" preScore: disabled: - name: "*" score: disabled: - name: "*" `, configKubeconfig)), os.FileMode(0600)); err != nil { t.Fatal(err) } // empty leader-election config emptyLeaderElectionConfig := filepath.Join(tmpDir, "empty-leader-election-config.yaml") if err := os.WriteFile(emptyLeaderElectionConfig, []byte(fmt.Sprintf(` apiVersion: kubescheduler.config.k8s.io/v1 kind: KubeSchedulerConfiguration clientConnection: kubeconfig: '%s' `, configKubeconfig)), os.FileMode(0600)); err != nil { t.Fatal(err) } // leader-election config leaderElectionConfig := filepath.Join(tmpDir, "leader-election-config.yaml") if err := os.WriteFile(leaderElectionConfig, []byte(fmt.Sprintf(` apiVersion: kubescheduler.config.k8s.io/v1 kind: KubeSchedulerConfiguration clientConnection: kubeconfig: '%s' leaderElection: leaseDuration: 1h `, configKubeconfig)), os.FileMode(0600)); err != nil { t.Fatal(err) } testcases := []struct { name string flags []string registryOptions []Option restoreFeatures map[featuregate.Feature]bool wantPlugins map[string]*config.Plugins wantLeaderElection *componentbaseconfig.LeaderElectionConfiguration }{ { name: "default config with an alpha feature enabled", flags: []string{ "--kubeconfig", configKubeconfig, "--feature-gates=VolumeCapacityPriority=true", }, wantPlugins: map[string]*config.Plugins{ "default-scheduler": defaults.ExpandedPluginsV1, }, restoreFeatures: map[featuregate.Feature]bool{ features.VolumeCapacityPriority: false, }, }, { name: "default config with a beta feature disabled", flags: []string{ "--kubeconfig", configKubeconfig, "--feature-gates=PodSchedulingReadiness=false", }, wantPlugins: map[string]*config.Plugins{ "default-scheduler": func() *config.Plugins { plugins := defaults.ExpandedPluginsV1.DeepCopy() plugins.PreEnqueue = config.PluginSet{} return plugins }(), }, restoreFeatures: map[featuregate.Feature]bool{ features.PodSchedulingReadiness: true, }, }, { name: "default config", flags: []string{ "--kubeconfig", configKubeconfig, }, wantPlugins: map[string]*config.Plugins{ "default-scheduler": defaults.ExpandedPluginsV1, }, }, { name: "component configuration v1beta2", flags: []string{ "--config", pluginConfigFilev1beta2, "--kubeconfig", configKubeconfig, }, wantPlugins: map[string]*config.Plugins{ "default-scheduler": func() *config.Plugins { plugins := defaults.PluginsV1beta2.DeepCopy() plugins.Filter.Enabled = []config.Plugin{ {Name: "NodeResourcesFit"}, {Name: "NodePorts"}, } plugins.PreFilter.Enabled = []config.Plugin{ {Name: "NodeResourcesFit"}, {Name: "NodePorts"}, } plugins.PreScore.Enabled = []config.Plugin{ {Name: "InterPodAffinity"}, {Name: "TaintToleration"}, } plugins.Score.Enabled = []config.Plugin{ {Name: "InterPodAffinity", Weight: 1}, {Name: "TaintToleration", Weight: 1}, } return plugins }(), }, }, { name: "component configuration v1beta3", flags: []string{ "--config", pluginConfigFilev1beta3, "--kubeconfig", configKubeconfig, }, wantPlugins: map[string]*config.Plugins{ "default-scheduler": func() *config.Plugins { plugins := defaults.ExpandedPluginsV1beta3.DeepCopy() plugins.Filter.Enabled = []config.Plugin{ {Name: "NodeResourcesFit"}, {Name: "NodePorts"}, } plugins.PreFilter.Enabled = []config.Plugin{ {Name: "NodeResourcesFit"}, {Name: "NodePorts"}, } plugins.PreScore.Enabled = []config.Plugin{ {Name: "NodeResourcesFit"}, {Name: "InterPodAffinity"}, {Name: "TaintToleration"}, } plugins.Score.Enabled = []config.Plugin{ {Name: "InterPodAffinity", Weight: 1}, {Name: "TaintToleration", Weight: 1}, } return plugins }(), }, }, { name: "component configuration v1", flags: []string{ "--config", pluginConfigFilev1, "--kubeconfig", configKubeconfig, }, wantPlugins: map[string]*config.Plugins{ "default-scheduler": func() *config.Plugins { plugins := defaults.ExpandedPluginsV1.DeepCopy() plugins.Filter.Enabled = []config.Plugin{ {Name: "NodeResourcesFit"}, {Name: "NodePorts"}, } plugins.PreFilter.Enabled = []config.Plugin{ {Name: "NodeResourcesFit"}, {Name: "NodePorts"}, } plugins.PreScore.Enabled = []config.Plugin{ {Name: "NodeResourcesFit"}, {Name: "InterPodAffinity"}, {Name: "TaintToleration"}, } plugins.Score.Enabled = []config.Plugin{ {Name: "InterPodAffinity", Weight: 1}, {Name: "TaintToleration", Weight: 1}, } return plugins }(), }, }, { name: "out-of-tree component configuration v1beta2", flags: []string{ "--config", outOfTreePluginConfigFilev1beta2, "--kubeconfig", configKubeconfig, }, registryOptions: []Option{WithPlugin("Foo", newFoo)}, wantPlugins: map[string]*config.Plugins{ "default-scheduler": func() *config.Plugins { plugins := defaults.PluginsV1beta2.DeepCopy() plugins.PreFilter.Enabled = append(plugins.PreFilter.Enabled, config.Plugin{Name: "Foo"}) plugins.Filter.Enabled = append(plugins.Filter.Enabled, config.Plugin{Name: "Foo"}) return plugins }(), }, }, { name: "out-of-tree component configuration v1beta3", flags: []string{ "--config", outOfTreePluginConfigFilev1beta3, "--kubeconfig", configKubeconfig, }, registryOptions: []Option{WithPlugin("Foo", newFoo)}, wantPlugins: map[string]*config.Plugins{ "default-scheduler": func() *config.Plugins { plugins := defaults.ExpandedPluginsV1beta3.DeepCopy() plugins.PreFilter.Enabled = append(plugins.PreFilter.Enabled, config.Plugin{Name: "Foo"}) plugins.Filter.Enabled = append(plugins.Filter.Enabled, config.Plugin{Name: "Foo"}) return plugins }(), }, }, { name: "out-of-tree component configuration v1", flags: []string{ "--config", outOfTreePluginConfigFilev1, "--kubeconfig", configKubeconfig, }, registryOptions: []Option{WithPlugin("Foo", newFoo)}, wantPlugins: map[string]*config.Plugins{ "default-scheduler": func() *config.Plugins { plugins := defaults.ExpandedPluginsV1.DeepCopy() plugins.PreFilter.Enabled = append(plugins.PreFilter.Enabled, config.Plugin{Name: "Foo"}) plugins.Filter.Enabled = append(plugins.Filter.Enabled, config.Plugin{Name: "Foo"}) return plugins }(), }, }, { name: "leader election CLI args, along with --config arg", flags: []string{ "--leader-elect=false", "--leader-elect-lease-duration=2h", // CLI args are favored over the fields in ComponentConfig "--lock-object-namespace=default", // deprecated CLI arg will be ignored if --config is specified "--config", emptyLeaderElectionConfig, }, wantLeaderElection: &componentbaseconfig.LeaderElectionConfiguration{ LeaderElect: false, // from CLI args LeaseDuration: metav1.Duration{Duration: 2 * time.Hour}, // from CLI args RenewDeadline: metav1.Duration{Duration: 10 * time.Second}, RetryPeriod: metav1.Duration{Duration: 2 * time.Second}, ResourceLock: "leases", ResourceName: v1beta3.SchedulerDefaultLockObjectName, ResourceNamespace: v1beta3.SchedulerDefaultLockObjectNamespace, }, }, { name: "leader election CLI args, without --config arg", flags: []string{ "--leader-elect=false", "--leader-elect-lease-duration=2h", "--lock-object-namespace=default", // deprecated CLI arg is honored if --config is not specified "--kubeconfig", configKubeconfig, }, wantLeaderElection: &componentbaseconfig.LeaderElectionConfiguration{ LeaderElect: false, // from CLI args LeaseDuration: metav1.Duration{Duration: 2 * time.Hour}, // from CLI args RenewDeadline: metav1.Duration{Duration: 10 * time.Second}, RetryPeriod: metav1.Duration{Duration: 2 * time.Second}, ResourceLock: "leases", ResourceName: v1beta3.SchedulerDefaultLockObjectName, ResourceNamespace: "default", // from deprecated CLI args }, }, { name: "leader election settings specified by ComponentConfig only", flags: []string{ "--config", leaderElectionConfig, }, wantLeaderElection: &componentbaseconfig.LeaderElectionConfiguration{ LeaderElect: true, LeaseDuration: metav1.Duration{Duration: 1 * time.Hour}, // from CC RenewDeadline: metav1.Duration{Duration: 10 * time.Second}, RetryPeriod: metav1.Duration{Duration: 2 * time.Second}, ResourceLock: "leases", ResourceName: v1beta3.SchedulerDefaultLockObjectName, ResourceNamespace: v1beta3.SchedulerDefaultLockObjectNamespace, }, }, { name: "leader election settings specified by CLI args and ComponentConfig", flags: []string{ "--leader-elect=true", "--leader-elect-renew-deadline=5s", "--leader-elect-retry-period=1s", "--config", leaderElectionConfig, }, wantLeaderElection: &componentbaseconfig.LeaderElectionConfiguration{ LeaderElect: true, LeaseDuration: metav1.Duration{Duration: 1 * time.Hour}, // from CC RenewDeadline: metav1.Duration{Duration: 5 * time.Second}, // from CLI args RetryPeriod: metav1.Duration{Duration: 1 * time.Second}, // from CLI args ResourceLock: "leases", ResourceName: v1beta3.SchedulerDefaultLockObjectName, ResourceNamespace: v1beta3.SchedulerDefaultLockObjectNamespace, }, }, } makeListener := func(t *testing.T) net.Listener { t.Helper() l, err := net.Listen("tcp", ":0") if err != nil { t.Fatal(err) } return l } for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { for k, v := range tc.restoreFeatures { defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, k, v)() } fs := pflag.NewFlagSet("test", pflag.PanicOnError) opts := options.NewOptions() // use listeners instead of static ports so parallel test runs don't conflict opts.SecureServing.Listener = makeListener(t) defer opts.SecureServing.Listener.Close() nfs := opts.Flags for _, f := range nfs.FlagSets { fs.AddFlagSet(f) } if err := fs.Parse(tc.flags); err != nil { t.Fatal(err) } // use listeners instead of static ports so parallel test runs don't conflict opts.SecureServing.Listener = makeListener(t) defer opts.SecureServing.Listener.Close() ctx, cancel := context.WithCancel(context.Background()) defer cancel() _, sched, err := Setup(ctx, opts, tc.registryOptions...) if err != nil { t.Fatal(err) } if tc.wantPlugins != nil { gotPlugins := make(map[string]*config.Plugins) for n, p := range sched.Profiles { gotPlugins[n] = p.ListPlugins() } if diff := cmp.Diff(tc.wantPlugins, gotPlugins); diff != "" { t.Errorf("Unexpected plugins diff (-want, +got): %s", diff) } } if tc.wantLeaderElection != nil { gotLeaderElection := opts.ComponentConfig.LeaderElection if diff := cmp.Diff(*tc.wantLeaderElection, gotLeaderElection); diff != "" { t.Errorf("Unexpected leaderElection diff (-want, +got): %s", diff) } } }) } } // Simulates an out-of-tree plugin. type foo struct{} var _ framework.PreFilterPlugin = &foo{} var _ framework.FilterPlugin = &foo{} func (*foo) Name() string { return "Foo" } func newFoo(_ runtime.Object, _ framework.Handle) (framework.Plugin, error) { return &foo{}, nil } func (*foo) PreFilter(_ context.Context, _ *framework.CycleState, _ *v1.Pod) (*framework.PreFilterResult, *framework.Status) { return nil, nil } func (*foo) PreFilterExtensions() framework.PreFilterExtensions { return nil } func (*foo) Filter(_ context.Context, _ *framework.CycleState, _ *v1.Pod, nodeInfo *framework.NodeInfo) *framework.Status { return nil }