Merge pull request #128818 from yongruilin/flagz-kube-scheduler

feat: Add flagz endpoint for kube-scheduler

cmd/kube-scheduler/app/config/config.go

@@ -26,11 +26,15 @@ import (
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/events"
 	"k8s.io/client-go/tools/leaderelection"
+	"k8s.io/component-base/zpages/flagz"
 	kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config"
 )
 
 // Config has all the context to run a Scheduler
 type Config struct {
+	// Flagz is the Reader interface to get flags for flagz page.
+	Flagz flagz.Reader
+
 	// ComponentConfig is the scheduler server's configuration object.
 	ComponentConfig kubeschedulerconfig.KubeSchedulerConfiguration
 

cmd/kube-scheduler/app/options/options.go

@@ -46,6 +46,8 @@ import (
 	logsapi "k8s.io/component-base/logs/api/v1"
 	"k8s.io/component-base/metrics"
 	utilversion "k8s.io/component-base/version"
+	zpagesfeatures "k8s.io/component-base/zpages/features"
+	"k8s.io/component-base/zpages/flagz"
 	"k8s.io/klog/v2"
 	schedulerappconfig "k8s.io/kubernetes/cmd/kube-scheduler/app/config"
 	"k8s.io/kubernetes/pkg/scheduler"
@@ -260,6 +262,11 @@ func (o *Options) ApplyTo(logger klog.Logger, c *schedulerappconfig.Config) erro
 	if o.Deprecated != nil {
 		c.PodMaxInUnschedulablePodsDuration = o.Deprecated.PodMaxInUnschedulablePodsDuration
 	}
+	if utilfeature.DefaultFeatureGate.Enabled(zpagesfeatures.ComponentFlagz) {
+		if o.Flags != nil {
+			c.Flagz = flagz.NamedFlagSetsReader{FlagSets: *o.Flags}
+		}
+	}
 
 	return nil
 }

cmd/kube-scheduler/app/server.go

@@ -56,6 +56,8 @@ import (
 	"k8s.io/component-base/term"
 	utilversion "k8s.io/component-base/version"
 	"k8s.io/component-base/version/verflag"
+	zpagesfeatures "k8s.io/component-base/zpages/features"
+	"k8s.io/component-base/zpages/flagz"
 	"k8s.io/klog/v2"
 	schedulerserverconfig "k8s.io/kubernetes/cmd/kube-scheduler/app/config"
 	"k8s.io/kubernetes/cmd/kube-scheduler/app/options"
@@ -68,6 +70,10 @@ import (
 	"k8s.io/kubernetes/pkg/scheduler/profile"
 )
 
+const (
+	kubeScheduler = "kube-scheduler"
+)
+
 func init() {
 	utilruntime.Must(logsapi.AddFeatureGates(utilfeature.DefaultMutableFeatureGate))
 	utilruntime.Must(features.AddFeatureGates(utilfeature.DefaultMutableFeatureGate))
@@ -224,7 +230,7 @@ func Run(ctx context.Context, cc *schedulerserverconfig.CompletedConfig, sched *
 			cc.Client,
 			metav1.NamespaceSystem,
 			cc.LeaderElection.Lock.Identity(),
-			"kube-scheduler",
+			kubeScheduler,
 			binaryVersion.FinalizeVersion(),
 			emulationVersion.FinalizeVersion(),
 			coordinationv1.OldestEmulationVersion,
@@ -236,9 +242,9 @@ func Run(ctx context.Context, cc *schedulerserverconfig.CompletedConfig, sched *
 		go leaseCandidate.Run(ctx)
 	}
 
-	// Start up the healthz server.
+	// Start up the server for endpoints.
 	if cc.SecureServing != nil {
-		handler := buildHandlerChain(newHealthEndpointsAndMetricsHandler(&cc.ComponentConfig, cc.InformerFactory, isLeader, checks, readyzChecks), cc.Authentication.Authenticator, cc.Authorization.Authorizer)
+		handler := buildHandlerChain(newEndpointsHandler(&cc.ComponentConfig, cc.InformerFactory, isLeader, checks, readyzChecks, cc.Flagz), cc.Authentication.Authenticator, cc.Authorization.Authorizer)
 		// TODO: handle stoppedCh and listenerStoppedCh returned by c.SecureServing.Serve
 		if _, _, err := cc.SecureServing.Serve(handler, 0, ctx.Done()); err != nil {
 			// fail early for secure handlers, removing the old error loop from above
@@ -344,11 +350,11 @@ func installMetricHandler(pathRecorderMux *mux.PathRecorderMux, informers inform
 	})
 }
 
-// newHealthEndpointsAndMetricsHandler creates an API health server from the config, and will also
+// newEndpointsHandler creates an API health server from the config, and will also
-// embed the metrics handler.
+// embed the metrics handler and z-pages handler.
 // TODO: healthz check is deprecated, please use livez and readyz instead. Will be removed in the future.
-func newHealthEndpointsAndMetricsHandler(config *kubeschedulerconfig.KubeSchedulerConfiguration, informers informers.SharedInformerFactory, isLeader func() bool, healthzChecks, readyzChecks []healthz.HealthChecker) http.Handler {
+func newEndpointsHandler(config *kubeschedulerconfig.KubeSchedulerConfiguration, informers informers.SharedInformerFactory, isLeader func() bool, healthzChecks, readyzChecks []healthz.HealthChecker, flagReader flagz.Reader) http.Handler {
-	pathRecorderMux := mux.NewPathRecorderMux("kube-scheduler")
+	pathRecorderMux := mux.NewPathRecorderMux(kubeScheduler)
 	healthz.InstallHandler(pathRecorderMux, healthzChecks...)
 	healthz.InstallLivezHandler(pathRecorderMux)
 	healthz.InstallReadyzHandler(pathRecorderMux, readyzChecks...)
@@ -362,6 +368,12 @@ func newHealthEndpointsAndMetricsHandler(config *kubeschedulerconfig.KubeSchedul
 		}
 		routes.DebugFlags{}.Install(pathRecorderMux, "v", routes.StringFlagPutHandler(logs.GlogSetter))
 	}
+
+	if utilfeature.DefaultFeatureGate.Enabled(zpagesfeatures.ComponentFlagz) {
+		if flagReader != nil {
+			flagz.Install(pathRecorderMux, kubeScheduler, flagReader)
+		}
+	}
 	return pathRecorderMux
 }
 

staging/src/k8s.io/component-base/zpages/flagz/flagreader.go

@@ -25,7 +25,7 @@ type Reader interface {
 	GetFlagz() map[string]string
 }
 
-// NamedFlagSetsGetter implements Reader for cliflag.NamedFlagSets
+// NamedFlagSetsReader implements Reader for cliflag.NamedFlagSets
 type NamedFlagSetsReader struct {
 	FlagSets cliflag.NamedFlagSets
 }

test/integration/scheduler/serving/endpoints_test.go

@@ -24,16 +24,21 @@ import (
 	"net/http"
 	"os"
 	"path"
+	"regexp"
 	"strings"
 	"testing"
 
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	featuregatetesting "k8s.io/component-base/featuregate/testing"
+	"k8s.io/component-base/zpages/features"
 	"k8s.io/klog/v2/ktesting"
 	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
 	kubeschedulertesting "k8s.io/kubernetes/cmd/kube-scheduler/app/testing"
 	"k8s.io/kubernetes/test/integration/framework"
 )
 
-func TestHealthEndpoints(t *testing.T) {
+func TestEndpointHandlers(t *testing.T) {
+	featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ComponentFlagz, true)
 	server, configStr, _, err := startTestAPIServer(t)
 	if err != nil {
 		t.Fatalf("Failed to start kube-apiserver server: %v", err)
@@ -64,52 +69,64 @@ func TestHealthEndpoints(t *testing.T) {
 	}()
 
 	tests := []struct {
-		name             string
+		name                 string
-		path             string
+		path                 string
-		useBrokenConfig  bool
+		useBrokenConfig      bool
-		wantResponseCode int
+		requestHeader        map[string]string
+		wantResponseCode     int
+		wantResponseBodyRegx string
 	}{
 		{
-			"/healthz",
+			name:             "/healthz",
-			"/healthz",
+			path:             "/healthz",
-			false,
+			useBrokenConfig:  false,
-			http.StatusOK,
+			wantResponseCode: http.StatusOK,
 		},
 		{
-			"/livez",
+			name:             "/livez",
-			"/livez",
+			path:             "/livez",
-			false,
+			useBrokenConfig:  false,
-			http.StatusOK,
+			wantResponseCode: http.StatusOK,
 		},
 		{
-			"/livez with ping check",
+			name:             "/livez with ping check",
-			"/livez/ping",
+			path:             "/livez/ping",
-			false,
+			useBrokenConfig:  false,
-			http.StatusOK,
+			wantResponseCode: http.StatusOK,
 		},
 		{
-			"/readyz",
+			name:             "/readyz",
-			"/readyz",
+			path:             "/readyz",
-			false,
+			useBrokenConfig:  false,
-			http.StatusOK,
+			wantResponseCode: http.StatusOK,
 		},
 		{
-			"/readyz with sched-handler-sync",
+			name:             "/readyz with sched-handler-sync",
-			"/readyz/sched-handler-sync",
+			path:             "/readyz/sched-handler-sync",
-			false,
+			useBrokenConfig:  false,
-			http.StatusOK,
+			wantResponseCode: http.StatusOK,
 		},
 		{
-			"/readyz with shutdown",
+			name:             "/readyz with shutdown",
-			"/readyz/shutdown",
+			path:             "/readyz/shutdown",
-			false,
+			useBrokenConfig:  false,
-			http.StatusOK,
+			wantResponseCode: http.StatusOK,
 		},
 		{
-			"/readyz with broken apiserver",
+			name:             "/readyz with broken apiserver",
-			"/readyz",
+			path:             "/readyz",
-			true,
+			useBrokenConfig:  true,
-			http.StatusInternalServerError,
+			wantResponseCode: http.StatusInternalServerError,
+		},
+		{
+			name:             "/flagz",
+			path:             "/flagz",
+			requestHeader:    map[string]string{"Accept": "text/plain"},
+			wantResponseCode: http.StatusOK,
+			wantResponseBodyRegx: `^\n` +
+				`kube-scheduler flags\n` +
+				`Warning: This endpoint is not meant to be machine parseable, ` +
+				`has no formatting compatibility guarantees and is for debugging purposes only.`,
 		},
 	}
 
@@ -141,6 +158,11 @@ func TestHealthEndpoints(t *testing.T) {
 			if err != nil {
 				t.Fatalf("failed to request: %v", err)
 			}
+			if tt.requestHeader != nil {
+				for k, v := range tt.requestHeader {
+					req.Header.Set(k, v)
+				}
+			}
 			r, err := client.Do(req)
 			if err != nil {
 				t.Fatalf("failed to GET %s from component: %v", tt.path, err)
@@ -156,6 +178,15 @@ func TestHealthEndpoints(t *testing.T) {
 			if got, expected := r.StatusCode, tt.wantResponseCode; got != expected {
 				t.Fatalf("expected http %d at %s of component, got: %d %q", expected, tt.path, got, string(body))
 			}
+			if tt.wantResponseBodyRegx != "" {
+				matched, err := regexp.MatchString(tt.wantResponseBodyRegx, string(body))
+				if err != nil {
+					t.Fatalf("failed to compile regex: %v", err)
+				}
+				if !matched {
+					t.Fatalf("response body does not match regex.\nExpected:\n%s\n\nGot:\n%s", tt.wantResponseBodyRegx, string(body))
+				}
+			}
 		})
 	}
 }