From 14d31aff480a4399e83720a066627b23cdef4c54 Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Tue, 26 Jun 2018 20:11:35 -0400 Subject: [PATCH] fix scheduler client construction from configuration files --- cmd/kube-scheduler/app/options/BUILD | 5 +- cmd/kube-scheduler/app/options/options.go | 32 +-- .../app/options/options_test.go | 193 ++++++++++++++++++ 3 files changed, 213 insertions(+), 17 deletions(-) create mode 100644 cmd/kube-scheduler/app/options/options_test.go diff --git a/cmd/kube-scheduler/app/options/BUILD b/cmd/kube-scheduler/app/options/BUILD index 0da97077f08..6ac1fdb76cf 100644 --- a/cmd/kube-scheduler/app/options/BUILD +++ b/cmd/kube-scheduler/app/options/BUILD @@ -56,7 +56,10 @@ filegroup( go_test( name = "go_default_test", - srcs = ["insecure_serving_test.go"], + srcs = [ + "insecure_serving_test.go", + "options_test.go", + ], embed = [":go_default_library"], deps = [ "//cmd/controller-manager/app/options:go_default_library", diff --git a/cmd/kube-scheduler/app/options/options.go b/cmd/kube-scheduler/app/options/options.go index 871c7d8c511..431354d61cd 100644 --- a/cmd/kube-scheduler/app/options/options.go +++ b/cmd/kube-scheduler/app/options/options.go @@ -197,37 +197,37 @@ func (o *Options) Validate() []error { // Config return a scheduler config object func (o *Options) Config() (*schedulerappconfig.Config, error) { + c := &schedulerappconfig.Config{} + if err := o.ApplyTo(c); err != nil { + return nil, err + } + // prepare kube clients. - client, leaderElectionClient, eventClient, err := createClients(o.ComponentConfig.ClientConnection, o.Master) + client, leaderElectionClient, eventClient, err := createClients(c.ComponentConfig.ClientConnection, o.Master) if err != nil { return nil, err } // Prepare event clients. eventBroadcaster := record.NewBroadcaster() - recorder := eventBroadcaster.NewRecorder(legacyscheme.Scheme, corev1.EventSource{Component: o.ComponentConfig.SchedulerName}) + recorder := eventBroadcaster.NewRecorder(legacyscheme.Scheme, corev1.EventSource{Component: c.ComponentConfig.SchedulerName}) // Set up leader election if enabled. var leaderElectionConfig *leaderelection.LeaderElectionConfig - if o.ComponentConfig.LeaderElection.LeaderElect { - leaderElectionConfig, err = makeLeaderElectionConfig(o.ComponentConfig.LeaderElection, leaderElectionClient, recorder) + if c.ComponentConfig.LeaderElection.LeaderElect { + leaderElectionConfig, err = makeLeaderElectionConfig(c.ComponentConfig.LeaderElection, leaderElectionClient, recorder) if err != nil { return nil, err } } - c := &schedulerappconfig.Config{ - Client: client, - InformerFactory: informers.NewSharedInformerFactory(client, 0), - PodInformer: factory.NewPodInformer(client, 0), - EventClient: eventClient, - Recorder: recorder, - Broadcaster: eventBroadcaster, - LeaderElection: leaderElectionConfig, - } - if err := o.ApplyTo(c); err != nil { - return nil, err - } + c.Client = client + c.InformerFactory = informers.NewSharedInformerFactory(client, 0) + c.PodInformer = factory.NewPodInformer(client, 0) + c.EventClient = eventClient + c.Recorder = recorder + c.Broadcaster = eventBroadcaster + c.LeaderElection = leaderElectionConfig return c, nil } diff --git a/cmd/kube-scheduler/app/options/options_test.go b/cmd/kube-scheduler/app/options/options_test.go new file mode 100644 index 00000000000..455b4d6ef26 --- /dev/null +++ b/cmd/kube-scheduler/app/options/options_test.go @@ -0,0 +1,193 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "k8s.io/kubernetes/pkg/apis/componentconfig" +) + +func TestSchedulerOptions(t *testing.T) { + // temp dir + tmpDir, err := ioutil.TempDir("", "scheduler-options") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpDir) + + // record the username requests were made with + username := "" + // https server + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + username, _, _ = req.BasicAuth() + if username == "" { + username = "none, tls" + } + w.WriteHeader(200) + w.Write([]byte(`ok`)) + })) + defer server.Close() + // http server + insecureserver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + username, _, _ = req.BasicAuth() + if username == "" { + username = "none, http" + } + w.WriteHeader(200) + w.Write([]byte(`ok`)) + })) + defer insecureserver.Close() + + // config file and kubeconfig + configFile := filepath.Join(tmpDir, "scheduler.yaml") + configKubeconfig := filepath.Join(tmpDir, "config.kubeconfig") + if err := ioutil.WriteFile(configFile, []byte(fmt.Sprintf(` +apiVersion: componentconfig/v1alpha1 +kind: KubeSchedulerConfiguration +clientConnection: + kubeconfig: "%s" +leaderElection: + leaderElect: true`, configKubeconfig)), os.FileMode(0600)); err != nil { + t.Fatal(err) + } + if err := ioutil.WriteFile(configKubeconfig, []byte(fmt.Sprintf(` +apiVersion: v1 +kind: Config +clusters: +- cluster: + insecure-skip-tls-verify: true + server: %s + name: default +contexts: +- context: + cluster: default + user: default + name: default +current-context: default +users: +- name: default + user: + username: config +`, server.URL)), os.FileMode(0600)); err != nil { + t.Fatal(err) + } + + // flag-specified kubeconfig + flagKubeconfig := filepath.Join(tmpDir, "flag.kubeconfig") + if err := ioutil.WriteFile(flagKubeconfig, []byte(fmt.Sprintf(` +apiVersion: v1 +kind: Config +clusters: +- cluster: + insecure-skip-tls-verify: true + server: %s + name: default +contexts: +- context: + cluster: default + user: default + name: default +current-context: default +users: +- name: default + user: + username: flag +`, server.URL)), os.FileMode(0600)); err != nil { + t.Fatal(err) + } + + // Insulate this test from picking up in-cluster config when run inside a pod + // We can't assume we have permissions to write to /var/run/secrets/... from a unit test to mock in-cluster config for testing + originalHost := os.Getenv("KUBERNETES_SERVICE_HOST") + if len(originalHost) > 0 { + os.Setenv("KUBERNETES_SERVICE_HOST", "") + defer os.Setenv("KUBERNETES_SERVICE_HOST", originalHost) + } + + testcases := []struct { + name string + options *Options + expectedUsername string + expectedError string + }{ + { + name: "config file", + options: &Options{ConfigFile: configFile}, + expectedUsername: "config", + }, + { + name: "kubeconfig flag", + options: &Options{ + ComponentConfig: componentconfig.KubeSchedulerConfiguration{ + ClientConnection: componentconfig.ClientConnectionConfiguration{ + KubeConfigFile: flagKubeconfig}}}, + expectedUsername: "flag", + }, + { + name: "overridden master", + options: &Options{Master: insecureserver.URL}, + expectedUsername: "none, http", + }, + { + name: "no config", + options: &Options{}, + expectedError: "no configuration has been provided", + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + // create the config + config, err := tc.options.Config() + + // handle errors + if err != nil { + if tc.expectedError == "" { + t.Error(err) + } else if !strings.Contains(err.Error(), tc.expectedError) { + t.Errorf("expected %q, got %q", tc.expectedError, err.Error()) + } + return + } + + // ensure we have a client + if config.Client == nil { + t.Error("unexpected nil client") + return + } + + // test the client talks to the endpoint we expect with the credentials we expect + username = "" + _, err = config.Client.Discovery().RESTClient().Get().AbsPath("/").DoRaw() + if err != nil { + t.Error(err) + return + } + if username != tc.expectedUsername { + t.Errorf("expected server call with user %s, got %s", tc.expectedUsername, username) + } + }) + } +}