Merge pull request #98458 from SaranBalaji90/profiling

Add support to disable /debug/pprof and /debug/flags/v endpoint

cmd/kubelet/app/server.go

@@ -1187,9 +1187,7 @@ func startKubelet(k kubelet.Bootstrap, podCfg *config.PodConfig, kubeCfg *kubele
 
 	// start the kubelet server
 	if enableServer {
-		go k.ListenAndServe(net.ParseIP(kubeCfg.Address), uint(kubeCfg.Port), kubeDeps.TLSOptions, kubeDeps.Auth,
+		go k.ListenAndServe(kubeCfg, kubeDeps.TLSOptions, kubeDeps.Auth, enableCAdvisorJSONEndpoints)
-			enableCAdvisorJSONEndpoints, kubeCfg.EnableDebuggingHandlers, kubeCfg.EnableContentionProfiling, kubeCfg.EnableSystemLogHandler)
-
 	}
 	if kubeCfg.ReadOnlyPort > 0 {
 		go k.ListenAndServeReadOnly(net.ParseIP(kubeCfg.Address), uint(kubeCfg.ReadOnlyPort), enableCAdvisorJSONEndpoints)

pkg/kubelet/apis/config/helpers_test.go

@@ -165,7 +165,9 @@ var (
 		"ContentType",
 		"EnableContentionProfiling",
 		"EnableControllerAttachDetach",
+		"EnableDebugFlagsHandler",
 		"EnableDebuggingHandlers",
+		"EnableProfilingHandler",
 		"EnableServer",
 		"EnableSystemLogHandler",
 		"EnforceNodeAllocatable[*]",

pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml

@@ -23,7 +23,9 @@ cpuCFSQuotaPeriod: 100ms
 cpuManagerPolicy: none
 cpuManagerReconcilePeriod: 10s
 enableControllerAttachDetach: true
+enableDebugFlagsHandler: true
 enableDebuggingHandlers: true
+enableProfilingHandler: true
 enableServer: true
 enableSystemLogHandler: true
 enforceNodeAllocatable:

pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml

@@ -23,7 +23,9 @@ cpuCFSQuotaPeriod: 100ms
 cpuManagerPolicy: none
 cpuManagerReconcilePeriod: 10s
 enableControllerAttachDetach: true
+enableDebugFlagsHandler: true
 enableDebuggingHandlers: true
+enableProfilingHandler: true
 enableServer: true
 enableSystemLogHandler: true
 enforceNodeAllocatable:

pkg/kubelet/apis/config/types.go

@@ -399,6 +399,10 @@ type KubeletConfiguration struct {
 	// 3. NUMAs nodes IDs that do not exist under the machine.
 	// 4. memory types except for memory and hugepages-<size>
 	ReservedMemory []MemoryReservation
+	// EnableProfiling enables /debug/pprof handler.
+	EnableProfilingHandler bool
+	// EnableDebugFlagsHandler enables/debug/flags/v handler.
+	EnableDebugFlagsHandler bool
 }
 
 // KubeletAuthorizationMode denotes the authorization mode for the kubelet

pkg/kubelet/apis/config/v1beta1/defaults.go

@@ -246,4 +246,10 @@ func SetDefaults_KubeletConfiguration(obj *kubeletconfigv1beta1.KubeletConfigura
 	if obj.EnableSystemLogHandler == nil {
 		obj.EnableSystemLogHandler = utilpointer.BoolPtr(true)
 	}
+	if obj.EnableProfilingHandler == nil {
+		obj.EnableProfilingHandler = utilpointer.BoolPtr(true)
+	}
+	if obj.EnableDebugFlagsHandler == nil {
+		obj.EnableDebugFlagsHandler = utilpointer.BoolPtr(true)
+	}
 }

pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go

@@ -365,6 +365,12 @@ func autoConvert_v1beta1_KubeletConfiguration_To_config_KubeletConfiguration(in
 	out.ShutdownGracePeriod = in.ShutdownGracePeriod
 	out.ShutdownGracePeriodCriticalPods = in.ShutdownGracePeriodCriticalPods
 	out.ReservedMemory = *(*[]config.MemoryReservation)(unsafe.Pointer(&in.ReservedMemory))
+	if err := v1.Convert_Pointer_bool_To_bool(&in.EnableProfilingHandler, &out.EnableProfilingHandler, s); err != nil {
+		return err
+	}
+	if err := v1.Convert_Pointer_bool_To_bool(&in.EnableDebugFlagsHandler, &out.EnableDebugFlagsHandler, s); err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -520,6 +526,12 @@ func autoConvert_config_KubeletConfiguration_To_v1beta1_KubeletConfiguration(in
 	out.ShutdownGracePeriod = in.ShutdownGracePeriod
 	out.ShutdownGracePeriodCriticalPods = in.ShutdownGracePeriodCriticalPods
 	out.ReservedMemory = *(*[]v1beta1.MemoryReservation)(unsafe.Pointer(&in.ReservedMemory))
+	if err := v1.Convert_bool_To_Pointer_bool(&in.EnableProfilingHandler, &out.EnableProfilingHandler, s); err != nil {
+		return err
+	}
+	if err := v1.Convert_bool_To_Pointer_bool(&in.EnableDebugFlagsHandler, &out.EnableDebugFlagsHandler, s); err != nil {
+		return err
+	}
 	return nil
 }
 

pkg/kubelet/kubelet.go

@@ -195,7 +195,8 @@ type Bootstrap interface {
 	GetConfiguration() kubeletconfiginternal.KubeletConfiguration
 	BirthCry()
 	StartGarbageCollection()
-	ListenAndServe(address net.IP, port uint, tlsOptions *server.TLSOptions, auth server.AuthInterface, enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableContentionProfiling, enableSystemLogHandler bool)
+	ListenAndServe(kubeCfg *kubeletconfiginternal.KubeletConfiguration, tlsOptions *server.TLSOptions, auth server.AuthInterface,
+		enableCAdvisorJSONEndpoints bool)
 	ListenAndServeReadOnly(address net.IP, port uint, enableCAdvisorJSONEndpoints bool)
 	ListenAndServePodResources()
 	Run(<-chan kubetypes.PodUpdate)
@@ -2225,8 +2226,9 @@ func (kl *Kubelet) ResyncInterval() time.Duration {
 }
 
 // ListenAndServe runs the kubelet HTTP server.
-func (kl *Kubelet) ListenAndServe(address net.IP, port uint, tlsOptions *server.TLSOptions, auth server.AuthInterface, enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableContentionProfiling, enableSystemLogHandler bool) {
+func (kl *Kubelet) ListenAndServe(kubeCfg *kubeletconfiginternal.KubeletConfiguration, tlsOptions *server.TLSOptions,
-	server.ListenAndServeKubeletServer(kl, kl.resourceAnalyzer, address, port, tlsOptions, auth, enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableContentionProfiling, enableSystemLogHandler)
+	auth server.AuthInterface, enableCAdvisorJSONEndpoints bool) {
+	server.ListenAndServeKubeletServer(kl, kl.resourceAnalyzer, kubeCfg, tlsOptions, auth, enableCAdvisorJSONEndpoints)
 }
 
 // ListenAndServeReadOnly runs the kubelet HTTP server in read-only mode.

pkg/kubelet/server/BUILD

@@ -19,6 +19,7 @@ go_library(
         "//pkg/apis/core:go_default_library",
         "//pkg/apis/core/v1/validation:go_default_library",
         "//pkg/features:go_default_library",
+        "//pkg/kubelet/apis/config:go_default_library",
         "//pkg/kubelet/apis/podresources:go_default_library",
         "//pkg/kubelet/container:go_default_library",
         "//pkg/kubelet/cri/streaming:go_default_library",
@@ -74,6 +75,7 @@ go_test(
     deps = [
         "//pkg/apis/core:go_default_library",
         "//pkg/apis/core/install:go_default_library",
+        "//pkg/kubelet/apis/config:go_default_library",
         "//pkg/kubelet/cm:go_default_library",
         "//pkg/kubelet/container:go_default_library",
         "//pkg/kubelet/cri/streaming:go_default_library",

pkg/kubelet/server/server.go

@@ -66,6 +66,7 @@ import (
 	api "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/apis/core/v1/validation"
 	"k8s.io/kubernetes/pkg/features"
+	kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config"
 	"k8s.io/kubernetes/pkg/kubelet/apis/podresources"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 	"k8s.io/kubernetes/pkg/kubelet/cri/streaming"
@@ -86,6 +87,8 @@ const (
 	specPath            = "/spec/"
 	statsPath           = "/stats/"
 	logsPath            = "/logs/"
+	pprofBasePath       = "/debug/pprof/"
+	debugFlagPath       = "/debug/flags/v"
 )
 
 // Server is a http.Handler which exposes kubelet functionality over HTTP.
@@ -137,16 +140,15 @@ func (a *filteringContainer) RegisteredHandlePaths() []string {
 func ListenAndServeKubeletServer(
 	host HostInterface,
 	resourceAnalyzer stats.ResourceAnalyzer,
-	address net.IP,
+	kubeCfg *kubeletconfiginternal.KubeletConfiguration,
-	port uint,
 	tlsOptions *TLSOptions,
 	auth AuthInterface,
-	enableCAdvisorJSONEndpoints,
+	enableCAdvisorJSONEndpoints bool) {
-	enableDebuggingHandlers,
+
-	enableContentionProfiling,
+	address := net.ParseIP(kubeCfg.Address)
-	enableSystemLogHandler bool) {
+	port := uint(kubeCfg.Port)
 	klog.InfoS("Starting to listen", "address", address, "port", port)
-	handler := NewServer(host, resourceAnalyzer, auth, enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableContentionProfiling, enableSystemLogHandler)
+	handler := NewServer(host, resourceAnalyzer, auth, enableCAdvisorJSONEndpoints, kubeCfg)
 	s := &http.Server{
 		Addr:           net.JoinHostPort(address.String(), strconv.FormatUint(uint64(port), 10)),
 		Handler:        &handler,
@@ -168,7 +170,7 @@ func ListenAndServeKubeletServer(
 // ListenAndServeKubeletReadOnlyServer initializes a server to respond to HTTP network requests on the Kubelet.
 func ListenAndServeKubeletReadOnlyServer(host HostInterface, resourceAnalyzer stats.ResourceAnalyzer, address net.IP, port uint, enableCAdvisorJSONEndpoints bool) {
 	klog.InfoS("Starting to listen read-only", "address", address, "port", port)
-	s := NewServer(host, resourceAnalyzer, nil, enableCAdvisorJSONEndpoints, false, false, false)
+	s := NewServer(host, resourceAnalyzer, nil, enableCAdvisorJSONEndpoints, nil)
 
 	server := &http.Server{
 		Addr:           net.JoinHostPort(address.String(), strconv.FormatUint(uint64(port), 10)),
@@ -220,10 +222,8 @@ func NewServer(
 	host HostInterface,
 	resourceAnalyzer stats.ResourceAnalyzer,
 	auth AuthInterface,
-	enableCAdvisorJSONEndpoints,
+	enableCAdvisorJSONEndpoints bool,
-	enableDebuggingHandlers,
+	kubeCfg *kubeletconfiginternal.KubeletConfiguration) Server {
-	enableContentionProfiling,
-	enableSystemLogHandler bool) Server {
 	server := Server{
 		host:                 host,
 		resourceAnalyzer:     resourceAnalyzer,
@@ -236,14 +236,13 @@ func NewServer(
 		server.InstallAuthFilter()
 	}
 	server.InstallDefaultHandlers(enableCAdvisorJSONEndpoints)
-	if enableDebuggingHandlers {
+	if kubeCfg != nil && kubeCfg.EnableDebuggingHandlers {
 		server.InstallDebuggingHandlers()
-		// To maintain backward compatibility serve logs only when enableDebuggingHandlers is also enabled
+		// To maintain backward compatibility serve logs and pprof only when enableDebuggingHandlers is also enabled
 		// see https://github.com/kubernetes/kubernetes/pull/87273
-		server.InstallSystemLogHandler(enableSystemLogHandler)
+		server.InstallSystemLogHandler(kubeCfg.EnableSystemLogHandler)
-		if enableContentionProfiling {
+		server.InstallProfilingHandler(kubeCfg.EnableProfilingHandler, kubeCfg.EnableContentionProfiling)
-			goruntime.SetBlockProfileRate(1)
+		server.InstallDebugFlagsHandler(kubeCfg.EnableDebugFlagsHandler)
-		}
 	} else {
 		server.InstallDebuggingDisabledHandlers()
 	}
@@ -403,8 +402,6 @@ func (s *Server) InstallDefaultHandlers(enableCAdvisorJSONEndpoints bool) {
 	}
 }
 
-const pprofBasePath = "/debug/pprof/"
-
 // InstallDebuggingHandlers registers the HTTP request patterns that serve logs or run commands/containers
 func (s *Server) InstallDebuggingHandlers() {
 	klog.InfoS("Adding debug handlers to kubelet server")
@@ -487,33 +484,6 @@ func (s *Server) InstallDebuggingHandlers() {
 	s.addMetricsBucketMatcher("configz")
 	configz.InstallHandler(s.restfulCont)
 
-	s.addMetricsBucketMatcher("debug")
-	handlePprofEndpoint := func(req *restful.Request, resp *restful.Response) {
-		name := strings.TrimPrefix(req.Request.URL.Path, pprofBasePath)
-		switch name {
-		case "profile":
-			pprof.Profile(resp, req.Request)
-		case "symbol":
-			pprof.Symbol(resp, req.Request)
-		case "cmdline":
-			pprof.Cmdline(resp, req.Request)
-		case "trace":
-			pprof.Trace(resp, req.Request)
-		default:
-			pprof.Index(resp, req.Request)
-		}
-	}
-	// Setup pprof handlers.
-	ws = new(restful.WebService).Path(pprofBasePath)
-	ws.Route(ws.GET("/{subpath:*}").To(func(req *restful.Request, resp *restful.Response) {
-		handlePprofEndpoint(req, resp)
-	})).Doc("pprof endpoint")
-	s.restfulCont.Add(ws)
-
-	// Setup flags handlers.
-	// so far, only logging related endpoints are considered valid to add for these debug flags.
-	s.restfulCont.Handle("/debug/flags/v", routes.StringFlagPutHandler(logs.GlogSetter))
-
 	// The /runningpods endpoint is used for testing only.
 	s.addMetricsBucketMatcher("runningpods")
 	ws = new(restful.WebService)
@@ -563,10 +533,59 @@ func (s *Server) InstallSystemLogHandler(enableSystemLogHandler bool) {
 			Param(ws.PathParameter("logpath", "path to the log").DataType("string")))
 		s.restfulCont.Add(ws)
 	} else {
-		h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		s.restfulCont.Handle(logsPath, getHandlerForDisabledEndpoint("logs endpoint is disabled."))
-			http.Error(w, "logs endpoint is disabled.", http.StatusMethodNotAllowed)
+	}
+}
+
+func getHandlerForDisabledEndpoint(errorMessage string) http.HandlerFunc {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, errorMessage, http.StatusMethodNotAllowed)
 	})
-		s.restfulCont.Handle(logsPath, h)
+}
+
+// InstallDebugFlagsHandler registers the HTTP request patterns for /debug/flags/v endpoint.
+func (s *Server) InstallDebugFlagsHandler(enableDebugFlagsHandler bool) {
+	if enableDebugFlagsHandler {
+		// Setup flags handlers.
+		// so far, only logging related endpoints are considered valid to add for these debug flags.
+		s.restfulCont.Handle(debugFlagPath, routes.StringFlagPutHandler(logs.GlogSetter))
+	} else {
+		s.restfulCont.Handle(debugFlagPath, getHandlerForDisabledEndpoint("flags endpoint is disabled."))
+		return
+	}
+}
+
+// InstallProfilingHandler registers the HTTP request patterns for /debug/pprof endpoint.
+func (s *Server) InstallProfilingHandler(enableProfilingLogHandler bool, enableContentionProfiling bool) {
+	s.addMetricsBucketMatcher("debug")
+	if !enableProfilingLogHandler {
+		s.restfulCont.Handle(pprofBasePath, getHandlerForDisabledEndpoint("profiling endpoint is disabled."))
+		return
+	}
+
+	handlePprofEndpoint := func(req *restful.Request, resp *restful.Response) {
+		name := strings.TrimPrefix(req.Request.URL.Path, pprofBasePath)
+		switch name {
+		case "profile":
+			pprof.Profile(resp, req.Request)
+		case "symbol":
+			pprof.Symbol(resp, req.Request)
+		case "cmdline":
+			pprof.Cmdline(resp, req.Request)
+		case "trace":
+			pprof.Trace(resp, req.Request)
+		default:
+			pprof.Index(resp, req.Request)
+		}
+	}
+
+	// Setup pprof handlers.
+	ws := new(restful.WebService).Path(pprofBasePath)
+	ws.Route(ws.GET("/{subpath:*}").To(handlePprofEndpoint)).Doc("pprof endpoint")
+	s.restfulCont.Add(ws)
+
+	if enableContentionProfiling {
+		goruntime.SetBlockProfileRate(1)
 	}
 }
 

pkg/kubelet/server/server_test.go

@@ -55,6 +55,7 @@ import (
 
 	// Do some initialization to decode the query parameters correctly.
 	_ "k8s.io/kubernetes/pkg/apis/core/install"
+	kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config"
 	"k8s.io/kubernetes/pkg/kubelet/cm"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 	"k8s.io/kubernetes/pkg/kubelet/cri/streaming"
@@ -304,10 +305,17 @@ func newServerTest() *serverTestFramework {
 }
 
 func newServerTestWithDebug(enableDebugging bool, streamingServer streaming.Server) *serverTestFramework {
-	return newServerTestWithDebuggingHandlers(enableDebugging, enableDebugging, streamingServer)
+	kubeCfg := &kubeletconfiginternal.KubeletConfiguration{
+		EnableDebuggingHandlers: enableDebugging,
+		EnableSystemLogHandler:  enableDebugging,
+		EnableProfilingHandler:  enableDebugging,
+		EnableDebugFlagsHandler: enableDebugging,
+	}
+	return newServerTestWithDebuggingHandlers(kubeCfg, streamingServer)
 }
 
-func newServerTestWithDebuggingHandlers(enableDebugging, enableSystemLogHandler bool, streamingServer streaming.Server) *serverTestFramework {
+func newServerTestWithDebuggingHandlers(kubeCfg *kubeletconfiginternal.KubeletConfiguration, streamingServer streaming.Server) *serverTestFramework {
+
 	fw := &serverTestFramework{}
 	fw.fakeKubelet = &fakeKubelet{
 		hostnameFunc: func() string {
@@ -341,9 +349,7 @@ func newServerTestWithDebuggingHandlers(enableDebugging, enableSystemLogHandler
 		stats.NewResourceAnalyzer(fw.fakeKubelet, time.Minute),
 		fw.fakeAuth,
 		true,
-		enableDebugging,
+		kubeCfg)
-		false,
-		enableSystemLogHandler)
 	fw.serverUnderTest = &server
 	fw.testHTTPServer = httptest.NewServer(fw.serverUnderTest)
 	return fw
@@ -1504,9 +1510,15 @@ func TestMetricMethodBuckets(t *testing.T) {
 }
 
 func TestDebuggingDisabledHandlers(t *testing.T) {
-	// for backward compatibility even if enablesystemLogHandler is set but not enableDebuggingHandler then /logs
+	// for backward compatibility even if enablesystemLogHandler or enableProfilingHandler is set but not
-	//shouldn't be served.
+	// enableDebuggingHandler then /logs, /pprof and /flags shouldn't be served.
-	fw := newServerTestWithDebuggingHandlers(false, true, nil)
+	kubeCfg := &kubeletconfiginternal.KubeletConfiguration{
+		EnableDebuggingHandlers: false,
+		EnableSystemLogHandler:  true,
+		EnableDebugFlagsHandler: true,
+		EnableProfilingHandler:  true,
+	}
+	fw := newServerTestWithDebuggingHandlers(kubeCfg, nil)
 	defer fw.testHTTPServer.Close()
 
 	paths := []string{
@@ -1546,15 +1558,19 @@ func TestDebuggingDisabledHandlers(t *testing.T) {
 	resp, err = http.Get(fw.testHTTPServer.URL + "/spec")
 	require.NoError(t, err)
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
-
 }
 
-func TestDisablingSystemLogHandler(t *testing.T) {
+func TestDisablingLogAndProfilingHandler(t *testing.T) {
-	fw := newServerTestWithDebuggingHandlers(true, false, nil)
+	kubeCfg := &kubeletconfiginternal.KubeletConfiguration{
+		EnableDebuggingHandlers: true,
+	}
+	fw := newServerTestWithDebuggingHandlers(kubeCfg, nil)
 	defer fw.testHTTPServer.Close()
 
-	// verify logs endpoint is disabled
+	// verify debug endpoints are disabled
 	verifyEndpointResponse(t, fw, "/logs/kubelet.log", "logs endpoint is disabled.\n")
+	verifyEndpointResponse(t, fw, "/debug/pprof/profile?seconds=2", "profiling endpoint is disabled.\n")
+	verifyEndpointResponse(t, fw, "/debug/flags/v", "flags endpoint is disabled.\n")
 }
 
 func TestFailedParseParamsSummaryHandler(t *testing.T) {

staging/src/k8s.io/kubelet/config/v1beta1/types.go

@@ -854,6 +854,14 @@ type KubeletConfiguration struct {
 	// Default: nil
 	// +optional
 	ReservedMemory []MemoryReservation `json:"reservedMemory,omitempty"`
+	// enableProfilingHandler enables profiling via web interface host:port/debug/pprof/
+	// Default: true
+	// +optional
+	EnableProfilingHandler *bool `json:"enableProfilingHandler,omitempty"`
+	// enableDebugFlagsHandler enables flags endpoint via web interface host:port/debug/flags/v
+	// Default: true
+	// +optional
+	EnableDebugFlagsHandler *bool `json:"enableDebugFlagsHandler,omitempty"`
 }
 
 type KubeletAuthorizationMode string

staging/src/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go

@@ -311,6 +311,16 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	if in.EnableProfilingHandler != nil {
+		in, out := &in.EnableProfilingHandler, &out.EnableProfilingHandler
+		*out = new(bool)
+		**out = **in
+	}
+	if in.EnableDebugFlagsHandler != nil {
+		in, out := &in.EnableDebugFlagsHandler, &out.EnableDebugFlagsHandler
+		*out = new(bool)
+		**out = **in
+	}
 	return
 }