From f3ff7d0518fbf577171adc5b87ef8231305c4769 Mon Sep 17 00:00:00 2001 From: yongruilin Date: Fri, 15 Nov 2024 10:32:30 -0800 Subject: [PATCH 1/2] chore: update comment for NamedFlagSetsReader --- staging/src/k8s.io/component-base/zpages/flagz/flagreader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/staging/src/k8s.io/component-base/zpages/flagz/flagreader.go b/staging/src/k8s.io/component-base/zpages/flagz/flagreader.go index 7e05928eb98..40fe09b450d 100644 --- a/staging/src/k8s.io/component-base/zpages/flagz/flagreader.go +++ b/staging/src/k8s.io/component-base/zpages/flagz/flagreader.go @@ -25,7 +25,7 @@ type Reader interface { GetFlagz() map[string]string } -// NamedFlagSetsGetter implements Reader for cliflag.NamedFlagSets +// NamedFlagSetsReader implements Reader for cliflag.NamedFlagSets type NamedFlagSetsReader struct { FlagSets cliflag.NamedFlagSets } From db6bf0221738a008cdfad13f82dc07a489558179 Mon Sep 17 00:00:00 2001 From: yongruilin Date: Fri, 15 Nov 2024 10:31:27 -0800 Subject: [PATCH 2/2] feat: Add flagz endpoint for kube-scheduler --- cmd/kube-scheduler/app/config/config.go | 4 + cmd/kube-scheduler/app/options/options.go | 7 ++ cmd/kube-scheduler/app/server.go | 26 +++-- ...{healthcheck_test.go => endpoints_test.go} | 97 ++++++++++++------- 4 files changed, 94 insertions(+), 40 deletions(-) rename test/integration/scheduler/serving/{healthcheck_test.go => endpoints_test.go} (71%) diff --git a/cmd/kube-scheduler/app/config/config.go b/cmd/kube-scheduler/app/config/config.go index 774cab62dc6..398349d70fe 100644 --- a/cmd/kube-scheduler/app/config/config.go +++ b/cmd/kube-scheduler/app/config/config.go @@ -26,11 +26,15 @@ import ( restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/events" "k8s.io/client-go/tools/leaderelection" + "k8s.io/component-base/zpages/flagz" kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" ) // Config has all the context to run a Scheduler type Config struct { + // Flagz is the Reader interface to get flags for flagz page. + Flagz flagz.Reader + // ComponentConfig is the scheduler server's configuration object. ComponentConfig kubeschedulerconfig.KubeSchedulerConfiguration diff --git a/cmd/kube-scheduler/app/options/options.go b/cmd/kube-scheduler/app/options/options.go index 95e1c002884..eb92adbeef0 100644 --- a/cmd/kube-scheduler/app/options/options.go +++ b/cmd/kube-scheduler/app/options/options.go @@ -46,6 +46,8 @@ import ( logsapi "k8s.io/component-base/logs/api/v1" "k8s.io/component-base/metrics" utilversion "k8s.io/component-base/version" + zpagesfeatures "k8s.io/component-base/zpages/features" + "k8s.io/component-base/zpages/flagz" "k8s.io/klog/v2" schedulerappconfig "k8s.io/kubernetes/cmd/kube-scheduler/app/config" "k8s.io/kubernetes/pkg/scheduler" @@ -260,6 +262,11 @@ func (o *Options) ApplyTo(logger klog.Logger, c *schedulerappconfig.Config) erro if o.Deprecated != nil { c.PodMaxInUnschedulablePodsDuration = o.Deprecated.PodMaxInUnschedulablePodsDuration } + if utilfeature.DefaultFeatureGate.Enabled(zpagesfeatures.ComponentFlagz) { + if o.Flags != nil { + c.Flagz = flagz.NamedFlagSetsReader{FlagSets: *o.Flags} + } + } return nil } diff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go index 1785bbdcc91..0a1809504c8 100644 --- a/cmd/kube-scheduler/app/server.go +++ b/cmd/kube-scheduler/app/server.go @@ -56,6 +56,8 @@ import ( "k8s.io/component-base/term" utilversion "k8s.io/component-base/version" "k8s.io/component-base/version/verflag" + zpagesfeatures "k8s.io/component-base/zpages/features" + "k8s.io/component-base/zpages/flagz" "k8s.io/klog/v2" schedulerserverconfig "k8s.io/kubernetes/cmd/kube-scheduler/app/config" "k8s.io/kubernetes/cmd/kube-scheduler/app/options" @@ -68,6 +70,10 @@ import ( "k8s.io/kubernetes/pkg/scheduler/profile" ) +const ( + kubeScheduler = "kube-scheduler" +) + func init() { utilruntime.Must(logsapi.AddFeatureGates(utilfeature.DefaultMutableFeatureGate)) utilruntime.Must(features.AddFeatureGates(utilfeature.DefaultMutableFeatureGate)) @@ -224,7 +230,7 @@ func Run(ctx context.Context, cc *schedulerserverconfig.CompletedConfig, sched * cc.Client, metav1.NamespaceSystem, cc.LeaderElection.Lock.Identity(), - "kube-scheduler", + kubeScheduler, binaryVersion.FinalizeVersion(), emulationVersion.FinalizeVersion(), coordinationv1.OldestEmulationVersion, @@ -236,9 +242,9 @@ func Run(ctx context.Context, cc *schedulerserverconfig.CompletedConfig, sched * go leaseCandidate.Run(ctx) } - // Start up the healthz server. + // Start up the server for endpoints. if cc.SecureServing != nil { - handler := buildHandlerChain(newHealthEndpointsAndMetricsHandler(&cc.ComponentConfig, cc.InformerFactory, isLeader, checks, readyzChecks), cc.Authentication.Authenticator, cc.Authorization.Authorizer) + handler := buildHandlerChain(newEndpointsHandler(&cc.ComponentConfig, cc.InformerFactory, isLeader, checks, readyzChecks, cc.Flagz), cc.Authentication.Authenticator, cc.Authorization.Authorizer) // TODO: handle stoppedCh and listenerStoppedCh returned by c.SecureServing.Serve if _, _, err := cc.SecureServing.Serve(handler, 0, ctx.Done()); err != nil { // fail early for secure handlers, removing the old error loop from above @@ -344,11 +350,11 @@ func installMetricHandler(pathRecorderMux *mux.PathRecorderMux, informers inform }) } -// newHealthEndpointsAndMetricsHandler creates an API health server from the config, and will also -// embed the metrics handler. +// newEndpointsHandler creates an API health server from the config, and will also +// embed the metrics handler and z-pages handler. // TODO: healthz check is deprecated, please use livez and readyz instead. Will be removed in the future. -func newHealthEndpointsAndMetricsHandler(config *kubeschedulerconfig.KubeSchedulerConfiguration, informers informers.SharedInformerFactory, isLeader func() bool, healthzChecks, readyzChecks []healthz.HealthChecker) http.Handler { - pathRecorderMux := mux.NewPathRecorderMux("kube-scheduler") +func newEndpointsHandler(config *kubeschedulerconfig.KubeSchedulerConfiguration, informers informers.SharedInformerFactory, isLeader func() bool, healthzChecks, readyzChecks []healthz.HealthChecker, flagReader flagz.Reader) http.Handler { + pathRecorderMux := mux.NewPathRecorderMux(kubeScheduler) healthz.InstallHandler(pathRecorderMux, healthzChecks...) healthz.InstallLivezHandler(pathRecorderMux) healthz.InstallReadyzHandler(pathRecorderMux, readyzChecks...) @@ -362,6 +368,12 @@ func newHealthEndpointsAndMetricsHandler(config *kubeschedulerconfig.KubeSchedul } routes.DebugFlags{}.Install(pathRecorderMux, "v", routes.StringFlagPutHandler(logs.GlogSetter)) } + + if utilfeature.DefaultFeatureGate.Enabled(zpagesfeatures.ComponentFlagz) { + if flagReader != nil { + flagz.Install(pathRecorderMux, kubeScheduler, flagReader) + } + } return pathRecorderMux } diff --git a/test/integration/scheduler/serving/healthcheck_test.go b/test/integration/scheduler/serving/endpoints_test.go similarity index 71% rename from test/integration/scheduler/serving/healthcheck_test.go rename to test/integration/scheduler/serving/endpoints_test.go index c094fcd9ace..91cb2d5ad84 100644 --- a/test/integration/scheduler/serving/healthcheck_test.go +++ b/test/integration/scheduler/serving/endpoints_test.go @@ -24,16 +24,21 @@ import ( "net/http" "os" "path" + "regexp" "strings" "testing" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/component-base/zpages/features" "k8s.io/klog/v2/ktesting" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" kubeschedulertesting "k8s.io/kubernetes/cmd/kube-scheduler/app/testing" "k8s.io/kubernetes/test/integration/framework" ) -func TestHealthEndpoints(t *testing.T) { +func TestEndpointHandlers(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ComponentFlagz, true) server, configStr, _, err := startTestAPIServer(t) if err != nil { t.Fatalf("Failed to start kube-apiserver server: %v", err) @@ -64,52 +69,64 @@ func TestHealthEndpoints(t *testing.T) { }() tests := []struct { - name string - path string - useBrokenConfig bool - wantResponseCode int + name string + path string + useBrokenConfig bool + requestHeader map[string]string + wantResponseCode int + wantResponseBodyRegx string }{ { - "/healthz", - "/healthz", - false, - http.StatusOK, + name: "/healthz", + path: "/healthz", + useBrokenConfig: false, + wantResponseCode: http.StatusOK, }, { - "/livez", - "/livez", - false, - http.StatusOK, + name: "/livez", + path: "/livez", + useBrokenConfig: false, + wantResponseCode: http.StatusOK, }, { - "/livez with ping check", - "/livez/ping", - false, - http.StatusOK, + name: "/livez with ping check", + path: "/livez/ping", + useBrokenConfig: false, + wantResponseCode: http.StatusOK, }, { - "/readyz", - "/readyz", - false, - http.StatusOK, + name: "/readyz", + path: "/readyz", + useBrokenConfig: false, + wantResponseCode: http.StatusOK, }, { - "/readyz with sched-handler-sync", - "/readyz/sched-handler-sync", - false, - http.StatusOK, + name: "/readyz with sched-handler-sync", + path: "/readyz/sched-handler-sync", + useBrokenConfig: false, + wantResponseCode: http.StatusOK, }, { - "/readyz with shutdown", - "/readyz/shutdown", - false, - http.StatusOK, + name: "/readyz with shutdown", + path: "/readyz/shutdown", + useBrokenConfig: false, + wantResponseCode: http.StatusOK, }, { - "/readyz with broken apiserver", - "/readyz", - true, - http.StatusInternalServerError, + name: "/readyz with broken apiserver", + path: "/readyz", + useBrokenConfig: true, + wantResponseCode: http.StatusInternalServerError, + }, + { + name: "/flagz", + path: "/flagz", + requestHeader: map[string]string{"Accept": "text/plain"}, + wantResponseCode: http.StatusOK, + wantResponseBodyRegx: `^\n` + + `kube-scheduler flags\n` + + `Warning: This endpoint is not meant to be machine parseable, ` + + `has no formatting compatibility guarantees and is for debugging purposes only.`, }, } @@ -141,6 +158,11 @@ func TestHealthEndpoints(t *testing.T) { if err != nil { t.Fatalf("failed to request: %v", err) } + if tt.requestHeader != nil { + for k, v := range tt.requestHeader { + req.Header.Set(k, v) + } + } r, err := client.Do(req) if err != nil { t.Fatalf("failed to GET %s from component: %v", tt.path, err) @@ -156,6 +178,15 @@ func TestHealthEndpoints(t *testing.T) { if got, expected := r.StatusCode, tt.wantResponseCode; got != expected { t.Fatalf("expected http %d at %s of component, got: %d %q", expected, tt.path, got, string(body)) } + if tt.wantResponseBodyRegx != "" { + matched, err := regexp.MatchString(tt.wantResponseBodyRegx, string(body)) + if err != nil { + t.Fatalf("failed to compile regex: %v", err) + } + if !matched { + t.Fatalf("response body does not match regex.\nExpected:\n%s\n\nGot:\n%s", tt.wantResponseBodyRegx, string(body)) + } + } }) } }