From 708180be6961a29f02d1b738a37695eea2b9b24d Mon Sep 17 00:00:00 2001 From: Eric Lin Date: Sat, 20 May 2023 16:04:53 +0000 Subject: [PATCH] Add /livez to kube-scheduler Health endpoint `/livez` only contains ping check. --- cmd/kube-scheduler/app/server.go | 7 +- .../scheduler/serving/healthcheck_test.go | 200 ++++++++++++++++++ .../scheduler/serving/main_test.go | 27 +++ 3 files changed, 231 insertions(+), 3 deletions(-) create mode 100644 test/integration/scheduler/serving/healthcheck_test.go create mode 100644 test/integration/scheduler/serving/main_test.go diff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go index 4a7e171ae30..9d75834fc4e 100644 --- a/cmd/kube-scheduler/app/server.go +++ b/cmd/kube-scheduler/app/server.go @@ -186,7 +186,7 @@ func Run(ctx context.Context, cc *schedulerserverconfig.CompletedConfig, sched * // Start up the healthz server. if cc.SecureServing != nil { - handler := buildHandlerChain(newHealthzAndMetricsHandler(&cc.ComponentConfig, cc.InformerFactory, isLeader, checks...), cc.Authentication.Authenticator, cc.Authorization.Authorizer) + handler := buildHandlerChain(newHealthEndpointsAndMetricsHandler(&cc.ComponentConfig, cc.InformerFactory, isLeader, checks...), cc.Authentication.Authenticator, cc.Authorization.Authorizer) // TODO: handle stoppedCh and listenerStoppedCh returned by c.SecureServing.Serve if _, _, err := cc.SecureServing.Serve(handler, 0, ctx.Done()); err != nil { // fail early for secure handlers, removing the old error loop from above @@ -288,11 +288,12 @@ func installMetricHandler(pathRecorderMux *mux.PathRecorderMux, informers inform }) } -// newHealthzAndMetricsHandler creates a healthz server from the config, and will also +// newHealthEndpointsAndMetricsHandler creates an API health server from the config, and will also // embed the metrics handler. -func newHealthzAndMetricsHandler(config *kubeschedulerconfig.KubeSchedulerConfiguration, informers informers.SharedInformerFactory, isLeader func() bool, checks ...healthz.HealthChecker) http.Handler { +func newHealthEndpointsAndMetricsHandler(config *kubeschedulerconfig.KubeSchedulerConfiguration, informers informers.SharedInformerFactory, isLeader func() bool, checks ...healthz.HealthChecker) http.Handler { pathRecorderMux := mux.NewPathRecorderMux("kube-scheduler") healthz.InstallHandler(pathRecorderMux, checks...) + healthz.InstallLivezHandler(pathRecorderMux) installMetricHandler(pathRecorderMux, informers, isLeader) slis.SLIMetricsWithReset{}.Install(pathRecorderMux) diff --git a/test/integration/scheduler/serving/healthcheck_test.go b/test/integration/scheduler/serving/healthcheck_test.go new file mode 100644 index 00000000000..47104970d64 --- /dev/null +++ b/test/integration/scheduler/serving/healthcheck_test.go @@ -0,0 +1,200 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package serving + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "io" + "net/http" + "os" + "path" + "strings" + "testing" + + "k8s.io/klog/v2/ktesting" + kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" + kubeschedulertesting "k8s.io/kubernetes/cmd/kube-scheduler/app/testing" + "k8s.io/kubernetes/test/integration/framework" +) + +func TestHealthEndpoints(t *testing.T) { + server, configStr, _, err := startTestAPIServer(t) + if err != nil { + t.Fatalf("Failed to start kube-apiserver server: %v", err) + } + defer server.TearDownFn() + apiserverConfig, err := os.CreateTemp("", "kubeconfig") + if err != nil { + t.Fatalf("Failed to create config file: %v", err) + } + defer func() { + _ = os.Remove(apiserverConfig.Name()) + }() + if _, err = apiserverConfig.WriteString(configStr); err != nil { + t.Fatalf("Failed to write config file: %v", err) + } + + tests := []struct { + name string + path string + wantResponseCode int + }{ + { + "/healthz", + "/healthz", + http.StatusOK, + }, + { + "/livez", + "/livez", + http.StatusOK, + }, + { + "/livez with ping check", + "/livez/ping", + http.StatusOK, + }, + { + "/readyz", + "/readyz", + http.StatusNotFound, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt := tt + _, ctx := ktesting.NewTestContext(t) + + result, err := kubeschedulertesting.StartTestServer( + ctx, + []string{"--kubeconfig", apiserverConfig.Name(), "--leader-elect=false", "--authorization-always-allow-paths", tt.path}) + + if err != nil { + t.Fatalf("Failed to start kube-scheduler server: %v", err) + } + if result.TearDownFn != nil { + defer result.TearDownFn() + } + + client, base, err := clientAndURLFromTestServer(result) + if err != nil { + t.Fatalf("Failed to get client from test server: %v", err) + } + req, err := http.NewRequest("GET", base+tt.path, nil) + if err != nil { + t.Fatal(err) + } + r, err := client.Do(req) + if err != nil { + t.Fatalf("failed to GET %s from component: %v", tt.path, err) + } + + body, err := io.ReadAll(r.Body) + if err != nil { + t.Fatalf("failed to read response body: %v", err) + } + if err = r.Body.Close(); err != nil { + t.Fatalf("failed to close response body: %v", err) + } + if got, expected := r.StatusCode, tt.wantResponseCode; got != expected { + t.Fatalf("expected http %d at %s of component, got: %d %q", expected, tt.path, got, string(body)) + } + }) + } +} + +// TODO: Make this a util function once there is a unified way to start a testing apiserver so that we can reuse it. +func startTestAPIServer(t *testing.T) (server *kubeapiservertesting.TestServer, apiserverConfig, token string, err error) { + // Insulate this test from picking up in-cluster config when run inside a pod + // We can't assume we have permissions to write to /var/run/secrets/... from a unit test to mock in-cluster config for testing + originalHost := os.Getenv("KUBERNETES_SERVICE_HOST") + if len(originalHost) > 0 { + if err = os.Setenv("KUBERNETES_SERVICE_HOST", ""); err != nil { + return + } + defer func() { + err = os.Setenv("KUBERNETES_SERVICE_HOST", originalHost) + }() + } + + // authenticate to apiserver via bearer token + token = "flwqkenfjasasdfmwerasd" // Fake token for testing. + var tokenFile *os.File + tokenFile, err = os.CreateTemp("", "kubeconfig") + if err != nil { + return + } + if _, err = tokenFile.WriteString(fmt.Sprintf(`%s,system:kube-scheduler,system:kube-scheduler,""`, token)); err != nil { + return + } + if err = tokenFile.Close(); err != nil { + return + } + + // start apiserver + server = kubeapiservertesting.StartTestServerOrDie(t, nil, []string{ + "--token-auth-file", tokenFile.Name(), + "--authorization-mode", "RBAC", + }, framework.SharedEtcd()) + + apiserverConfig = fmt.Sprintf(` +apiVersion: v1 +kind: Config +clusters: +- cluster: + server: %s + certificate-authority: %s + name: integration +contexts: +- context: + cluster: integration + user: kube-scheduler + name: default-context +current-context: default-context +users: +- name: kube-scheduler + user: + token: %s +`, server.ClientConfig.Host, server.ServerOpts.SecureServing.ServerCert.CertKey.CertFile, token) + return server, apiserverConfig, token, nil +} + +func clientAndURLFromTestServer(s kubeschedulertesting.TestServer) (*http.Client, string, error) { + secureInfo := s.Config.SecureServing + secureOptions := s.Options.SecureServing + url := fmt.Sprintf("https://%s", secureInfo.Listener.Addr().String()) + url = strings.ReplaceAll(url, "[::]", "127.0.0.1") // switch to IPv4 because the self-signed cert does not support [::] + + // read self-signed server cert disk + pool := x509.NewCertPool() + serverCertPath := path.Join(secureOptions.ServerCert.CertDirectory, secureOptions.ServerCert.PairName+".crt") + serverCert, err := os.ReadFile(serverCertPath) + if err != nil { + return nil, "", fmt.Errorf("Failed to read component server cert %q: %w", serverCertPath, err) + } + pool.AppendCertsFromPEM(serverCert) + tr := &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: pool, + }, + } + client := &http.Client{Transport: tr} + return client, url, nil +} diff --git a/test/integration/scheduler/serving/main_test.go b/test/integration/scheduler/serving/main_test.go new file mode 100644 index 00000000000..1694670f32c --- /dev/null +++ b/test/integration/scheduler/serving/main_test.go @@ -0,0 +1,27 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package serving + +import ( + "testing" + + "k8s.io/kubernetes/test/integration/framework" +) + +func TestMain(m *testing.M) { + framework.EtcdMain(m.Run) +}