From 51cdf4e97bbd03b353dcb852d6b9c39642f4f7a8 Mon Sep 17 00:00:00 2001 From: Sri Saran Balaji Vellore Rajakumar Date: Sat, 23 Jan 2021 09:16:39 -0800 Subject: [PATCH 1/2] Add support to disable /debug/pprof and /debug/flags/v endpoint Co-authored-by: xiaofei.sun Co-authored-by: SaranBalaji90 --- cmd/kubelet/app/server.go | 4 +- pkg/kubelet/apis/config/helpers_test.go | 2 + .../KubeletConfiguration/after/v1beta1.yaml | 2 + .../roundtrip/default/v1beta1.yaml | 2 + pkg/kubelet/apis/config/types.go | 4 + pkg/kubelet/apis/config/v1beta1/defaults.go | 6 + .../config/v1beta1/zz_generated.conversion.go | 12 ++ pkg/kubelet/kubelet.go | 12 +- pkg/kubelet/server/server.go | 112 +++++++++++------- pkg/kubelet/server/server_test.go | 12 +- .../k8s.io/kubelet/config/v1beta1/types.go | 8 ++ .../config/v1beta1/zz_generated.deepcopy.go | 10 ++ 12 files changed, 134 insertions(+), 52 deletions(-) diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 79908ff608a..ddba3464b38 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -1188,8 +1188,8 @@ func startKubelet(k kubelet.Bootstrap, podCfg *config.PodConfig, kubeCfg *kubele // start the kubelet server if enableServer { go k.ListenAndServe(net.ParseIP(kubeCfg.Address), uint(kubeCfg.Port), kubeDeps.TLSOptions, kubeDeps.Auth, - enableCAdvisorJSONEndpoints, kubeCfg.EnableDebuggingHandlers, kubeCfg.EnableContentionProfiling, kubeCfg.EnableSystemLogHandler) - + enableCAdvisorJSONEndpoints, kubeCfg.EnableDebuggingHandlers, kubeCfg.EnableContentionProfiling, + kubeCfg.EnableSystemLogHandler, kubeCfg.EnableProfilingHandler, kubeCfg.EnableDebugFlagsHandler) } if kubeCfg.ReadOnlyPort > 0 { go k.ListenAndServeReadOnly(net.ParseIP(kubeCfg.Address), uint(kubeCfg.ReadOnlyPort), enableCAdvisorJSONEndpoints) diff --git a/pkg/kubelet/apis/config/helpers_test.go b/pkg/kubelet/apis/config/helpers_test.go index b1782b6fb52..2ec98df77f5 100644 --- a/pkg/kubelet/apis/config/helpers_test.go +++ b/pkg/kubelet/apis/config/helpers_test.go @@ -165,7 +165,9 @@ var ( "ContentType", "EnableContentionProfiling", "EnableControllerAttachDetach", + "EnableDebugFlagsHandler", "EnableDebuggingHandlers", + "EnableProfilingHandler", "EnableServer", "EnableSystemLogHandler", "EnforceNodeAllocatable[*]", diff --git a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml index 4ebbd726791..344f8543737 100644 --- a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml +++ b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml @@ -23,7 +23,9 @@ cpuCFSQuotaPeriod: 100ms cpuManagerPolicy: none cpuManagerReconcilePeriod: 10s enableControllerAttachDetach: true +enableDebugFlagsHandler: true enableDebuggingHandlers: true +enableProfilingHandler: true enableServer: true enableSystemLogHandler: true enforceNodeAllocatable: diff --git a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml index 4ebbd726791..344f8543737 100644 --- a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml +++ b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml @@ -23,7 +23,9 @@ cpuCFSQuotaPeriod: 100ms cpuManagerPolicy: none cpuManagerReconcilePeriod: 10s enableControllerAttachDetach: true +enableDebugFlagsHandler: true enableDebuggingHandlers: true +enableProfilingHandler: true enableServer: true enableSystemLogHandler: true enforceNodeAllocatable: diff --git a/pkg/kubelet/apis/config/types.go b/pkg/kubelet/apis/config/types.go index 1dddcf3bd7b..749a6df2a65 100644 --- a/pkg/kubelet/apis/config/types.go +++ b/pkg/kubelet/apis/config/types.go @@ -399,6 +399,10 @@ type KubeletConfiguration struct { // 3. NUMAs nodes IDs that do not exist under the machine. // 4. memory types except for memory and hugepages- ReservedMemory []MemoryReservation + // EnableProfiling enables /debug/pprof handler. + EnableProfilingHandler bool + // EnableDebugFlagsHandler enables/debug/flags/v handler. + EnableDebugFlagsHandler bool } // KubeletAuthorizationMode denotes the authorization mode for the kubelet diff --git a/pkg/kubelet/apis/config/v1beta1/defaults.go b/pkg/kubelet/apis/config/v1beta1/defaults.go index c45764ae5e9..f16c8b49027 100644 --- a/pkg/kubelet/apis/config/v1beta1/defaults.go +++ b/pkg/kubelet/apis/config/v1beta1/defaults.go @@ -246,4 +246,10 @@ func SetDefaults_KubeletConfiguration(obj *kubeletconfigv1beta1.KubeletConfigura if obj.EnableSystemLogHandler == nil { obj.EnableSystemLogHandler = utilpointer.BoolPtr(true) } + if obj.EnableProfilingHandler == nil { + obj.EnableProfilingHandler = utilpointer.BoolPtr(true) + } + if obj.EnableDebugFlagsHandler == nil { + obj.EnableDebugFlagsHandler = utilpointer.BoolPtr(true) + } } diff --git a/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go b/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go index a1c7ddcfca2..48606111fa4 100644 --- a/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go +++ b/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go @@ -365,6 +365,12 @@ func autoConvert_v1beta1_KubeletConfiguration_To_config_KubeletConfiguration(in out.ShutdownGracePeriod = in.ShutdownGracePeriod out.ShutdownGracePeriodCriticalPods = in.ShutdownGracePeriodCriticalPods out.ReservedMemory = *(*[]config.MemoryReservation)(unsafe.Pointer(&in.ReservedMemory)) + if err := v1.Convert_Pointer_bool_To_bool(&in.EnableProfilingHandler, &out.EnableProfilingHandler, s); err != nil { + return err + } + if err := v1.Convert_Pointer_bool_To_bool(&in.EnableDebugFlagsHandler, &out.EnableDebugFlagsHandler, s); err != nil { + return err + } return nil } @@ -520,6 +526,12 @@ func autoConvert_config_KubeletConfiguration_To_v1beta1_KubeletConfiguration(in out.ShutdownGracePeriod = in.ShutdownGracePeriod out.ShutdownGracePeriodCriticalPods = in.ShutdownGracePeriodCriticalPods out.ReservedMemory = *(*[]v1beta1.MemoryReservation)(unsafe.Pointer(&in.ReservedMemory)) + if err := v1.Convert_bool_To_Pointer_bool(&in.EnableProfilingHandler, &out.EnableProfilingHandler, s); err != nil { + return err + } + if err := v1.Convert_bool_To_Pointer_bool(&in.EnableDebugFlagsHandler, &out.EnableDebugFlagsHandler, s); err != nil { + return err + } return nil } diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 7b82842f155..70b95b3cafa 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -195,7 +195,9 @@ type Bootstrap interface { GetConfiguration() kubeletconfiginternal.KubeletConfiguration BirthCry() StartGarbageCollection() - ListenAndServe(address net.IP, port uint, tlsOptions *server.TLSOptions, auth server.AuthInterface, enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableContentionProfiling, enableSystemLogHandler bool) + ListenAndServe(address net.IP, port uint, tlsOptions *server.TLSOptions, auth server.AuthInterface, + enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableContentionProfiling, enableSystemLogHandler, + enableProfilingHandler, enableDebugFlagsHandler bool) ListenAndServeReadOnly(address net.IP, port uint, enableCAdvisorJSONEndpoints bool) ListenAndServePodResources() Run(<-chan kubetypes.PodUpdate) @@ -2225,8 +2227,12 @@ func (kl *Kubelet) ResyncInterval() time.Duration { } // ListenAndServe runs the kubelet HTTP server. -func (kl *Kubelet) ListenAndServe(address net.IP, port uint, tlsOptions *server.TLSOptions, auth server.AuthInterface, enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableContentionProfiling, enableSystemLogHandler bool) { - server.ListenAndServeKubeletServer(kl, kl.resourceAnalyzer, address, port, tlsOptions, auth, enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableContentionProfiling, enableSystemLogHandler) +func (kl *Kubelet) ListenAndServe(address net.IP, port uint, tlsOptions *server.TLSOptions, auth server.AuthInterface, + enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableContentionProfiling, enableSystemLogHandler, + enableProfilingHandler, enableDebugFlagsHandler bool) { + server.ListenAndServeKubeletServer(kl, kl.resourceAnalyzer, address, port, tlsOptions, auth, + enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableContentionProfiling, enableSystemLogHandler, + enableProfilingHandler, enableDebugFlagsHandler) } // ListenAndServeReadOnly runs the kubelet HTTP server in read-only mode. diff --git a/pkg/kubelet/server/server.go b/pkg/kubelet/server/server.go index eb5c60a19f7..4484fb9dc63 100644 --- a/pkg/kubelet/server/server.go +++ b/pkg/kubelet/server/server.go @@ -86,6 +86,8 @@ const ( specPath = "/spec/" statsPath = "/stats/" logsPath = "/logs/" + pprofBasePath = "/debug/pprof/" + debugFlagPath = "/debug/flags/v" ) // Server is a http.Handler which exposes kubelet functionality over HTTP. @@ -144,9 +146,12 @@ func ListenAndServeKubeletServer( enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableContentionProfiling, - enableSystemLogHandler bool) { - klog.InfoS("Starting to listen", "address", address, "port", port) - handler := NewServer(host, resourceAnalyzer, auth, enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableContentionProfiling, enableSystemLogHandler) + enableSystemLogHandler, + enableProfilingHandler, + enableDebugFlagsHandler bool) { + klog.Infof("Starting to listen on %s:%d", address, port) + handler := NewServer(host, resourceAnalyzer, auth, enableCAdvisorJSONEndpoints, enableDebuggingHandlers, + enableContentionProfiling, enableSystemLogHandler, enableProfilingHandler, enableDebugFlagsHandler) s := &http.Server{ Addr: net.JoinHostPort(address.String(), strconv.FormatUint(uint64(port), 10)), Handler: &handler, @@ -167,8 +172,8 @@ func ListenAndServeKubeletServer( // ListenAndServeKubeletReadOnlyServer initializes a server to respond to HTTP network requests on the Kubelet. func ListenAndServeKubeletReadOnlyServer(host HostInterface, resourceAnalyzer stats.ResourceAnalyzer, address net.IP, port uint, enableCAdvisorJSONEndpoints bool) { - klog.InfoS("Starting to listen read-only", "address", address, "port", port) - s := NewServer(host, resourceAnalyzer, nil, enableCAdvisorJSONEndpoints, false, false, false) + klog.V(1).Infof("Starting to listen read-only on %s:%d", address, port) + s := NewServer(host, resourceAnalyzer, nil, enableCAdvisorJSONEndpoints, false, false, false, false, false) server := &http.Server{ Addr: net.JoinHostPort(address.String(), strconv.FormatUint(uint64(port), 10)), @@ -223,7 +228,9 @@ func NewServer( enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableContentionProfiling, - enableSystemLogHandler bool) Server { + enableSystemLogHandler, + enableProfilingHandler, + enableDebugFlagsHandler bool) Server { server := Server{ host: host, resourceAnalyzer: resourceAnalyzer, @@ -238,12 +245,11 @@ func NewServer( server.InstallDefaultHandlers(enableCAdvisorJSONEndpoints) if enableDebuggingHandlers { server.InstallDebuggingHandlers() - // To maintain backward compatibility serve logs only when enableDebuggingHandlers is also enabled + // To maintain backward compatibility serve logs and pprof only when enableDebuggingHandlers is also enabled // see https://github.com/kubernetes/kubernetes/pull/87273 server.InstallSystemLogHandler(enableSystemLogHandler) - if enableContentionProfiling { - goruntime.SetBlockProfileRate(1) - } + server.InstallProfilingHandler(enableProfilingHandler, enableContentionProfiling) + server.InstallDebugFlagsHandler(enableDebugFlagsHandler) } else { server.InstallDebuggingDisabledHandlers() } @@ -403,8 +409,6 @@ func (s *Server) InstallDefaultHandlers(enableCAdvisorJSONEndpoints bool) { } } -const pprofBasePath = "/debug/pprof/" - // InstallDebuggingHandlers registers the HTTP request patterns that serve logs or run commands/containers func (s *Server) InstallDebuggingHandlers() { klog.InfoS("Adding debug handlers to kubelet server") @@ -487,33 +491,6 @@ func (s *Server) InstallDebuggingHandlers() { s.addMetricsBucketMatcher("configz") configz.InstallHandler(s.restfulCont) - s.addMetricsBucketMatcher("debug") - handlePprofEndpoint := func(req *restful.Request, resp *restful.Response) { - name := strings.TrimPrefix(req.Request.URL.Path, pprofBasePath) - switch name { - case "profile": - pprof.Profile(resp, req.Request) - case "symbol": - pprof.Symbol(resp, req.Request) - case "cmdline": - pprof.Cmdline(resp, req.Request) - case "trace": - pprof.Trace(resp, req.Request) - default: - pprof.Index(resp, req.Request) - } - } - // Setup pprof handlers. - ws = new(restful.WebService).Path(pprofBasePath) - ws.Route(ws.GET("/{subpath:*}").To(func(req *restful.Request, resp *restful.Response) { - handlePprofEndpoint(req, resp) - })).Doc("pprof endpoint") - s.restfulCont.Add(ws) - - // Setup flags handlers. - // so far, only logging related endpoints are considered valid to add for these debug flags. - s.restfulCont.Handle("/debug/flags/v", routes.StringFlagPutHandler(logs.GlogSetter)) - // The /runningpods endpoint is used for testing only. s.addMetricsBucketMatcher("runningpods") ws = new(restful.WebService) @@ -563,10 +540,59 @@ func (s *Server) InstallSystemLogHandler(enableSystemLogHandler bool) { Param(ws.PathParameter("logpath", "path to the log").DataType("string"))) s.restfulCont.Add(ws) } else { - h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - http.Error(w, "logs endpoint is disabled.", http.StatusMethodNotAllowed) - }) - s.restfulCont.Handle(logsPath, h) + s.restfulCont.Handle(logsPath, getHandlerForDisabledEndpoint("logs endpoint is disabled.")) + } +} + +func getHandlerForDisabledEndpoint(errorMessage string) http.HandlerFunc { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, errorMessage, http.StatusMethodNotAllowed) + }) +} + +// InstallDebugFlagsHandler registers the HTTP request patterns for /debug/flags/v endpoint. +func (s *Server) InstallDebugFlagsHandler(enableDebugFlagsHandler bool) { + if enableDebugFlagsHandler { + // Setup flags handlers. + // so far, only logging related endpoints are considered valid to add for these debug flags. + s.restfulCont.Handle(debugFlagPath, routes.StringFlagPutHandler(logs.GlogSetter)) + } else { + s.restfulCont.Handle(debugFlagPath, getHandlerForDisabledEndpoint("flags endpoint is disabled.")) + return + } +} + +// InstallProfilingHandler registers the HTTP request patterns for /debug/pprof endpoint. +func (s *Server) InstallProfilingHandler(enableSystemLogHandler bool, enableContentionProfiling bool) { + s.addMetricsBucketMatcher("debug") + if !enableSystemLogHandler { + s.restfulCont.Handle(pprofBasePath, getHandlerForDisabledEndpoint("profiling endpoint is disabled.")) + return + } + + handlePprofEndpoint := func(req *restful.Request, resp *restful.Response) { + name := strings.TrimPrefix(req.Request.URL.Path, pprofBasePath) + switch name { + case "profile": + pprof.Profile(resp, req.Request) + case "symbol": + pprof.Symbol(resp, req.Request) + case "cmdline": + pprof.Cmdline(resp, req.Request) + case "trace": + pprof.Trace(resp, req.Request) + default: + pprof.Index(resp, req.Request) + } + } + + // Setup pprof handlers. + ws := new(restful.WebService).Path(pprofBasePath) + ws.Route(ws.GET("/{subpath:*}").To(handlePprofEndpoint)).Doc("pprof endpoint") + s.restfulCont.Add(ws) + + if enableContentionProfiling { + goruntime.SetBlockProfileRate(1) } } diff --git a/pkg/kubelet/server/server_test.go b/pkg/kubelet/server/server_test.go index 263e9c9ac8a..9072c6b4304 100644 --- a/pkg/kubelet/server/server_test.go +++ b/pkg/kubelet/server/server_test.go @@ -307,7 +307,7 @@ func newServerTestWithDebug(enableDebugging bool, streamingServer streaming.Serv return newServerTestWithDebuggingHandlers(enableDebugging, enableDebugging, streamingServer) } -func newServerTestWithDebuggingHandlers(enableDebugging, enableSystemLogHandler bool, streamingServer streaming.Server) *serverTestFramework { +func newServerTestWithDebuggingHandlers(enableDebugging, enableLogAndProfilingHandler bool, streamingServer streaming.Server) *serverTestFramework { fw := &serverTestFramework{} fw.fakeKubelet = &fakeKubelet{ hostnameFunc: func() string { @@ -343,7 +343,9 @@ func newServerTestWithDebuggingHandlers(enableDebugging, enableSystemLogHandler true, enableDebugging, false, - enableSystemLogHandler) + enableLogAndProfilingHandler, + enableLogAndProfilingHandler, + enableLogAndProfilingHandler) fw.serverUnderTest = &server fw.testHTTPServer = httptest.NewServer(fw.serverUnderTest) return fw @@ -1549,12 +1551,14 @@ func TestDebuggingDisabledHandlers(t *testing.T) { } -func TestDisablingSystemLogHandler(t *testing.T) { +func TestDisablingLogAndProfilingHandler(t *testing.T) { fw := newServerTestWithDebuggingHandlers(true, false, nil) defer fw.testHTTPServer.Close() - // verify logs endpoint is disabled + // verify debug endpoints are disabled verifyEndpointResponse(t, fw, "/logs/kubelet.log", "logs endpoint is disabled.\n") + verifyEndpointResponse(t, fw, "/debug/pprof/profile?seconds=2", "profiling endpoint is disabled.\n") + verifyEndpointResponse(t, fw, "/debug/flags/v", "flags endpoint is disabled.\n") } func TestFailedParseParamsSummaryHandler(t *testing.T) { diff --git a/staging/src/k8s.io/kubelet/config/v1beta1/types.go b/staging/src/k8s.io/kubelet/config/v1beta1/types.go index 1c009b96eaa..bc7409e9e11 100644 --- a/staging/src/k8s.io/kubelet/config/v1beta1/types.go +++ b/staging/src/k8s.io/kubelet/config/v1beta1/types.go @@ -854,6 +854,14 @@ type KubeletConfiguration struct { // Default: nil // +optional ReservedMemory []MemoryReservation `json:"reservedMemory,omitempty"` + // enableProfilingHandler enables profiling via web interface host:port/debug/pprof/ + // Default: true + // +optional + EnableProfilingHandler *bool `json:"enableProfilingHandler,omitempty"` + // enableDebugFlagsHandler enables flags endpoint via web interface host:port/debug/flags/v + // Default: true + // +optional + EnableDebugFlagsHandler *bool `json:"enableDebugFlagsHandler,omitempty"` } type KubeletAuthorizationMode string diff --git a/staging/src/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go b/staging/src/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go index 0cca1feecde..cd8b343a9fa 100644 --- a/staging/src/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go @@ -311,6 +311,16 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.EnableProfilingHandler != nil { + in, out := &in.EnableProfilingHandler, &out.EnableProfilingHandler + *out = new(bool) + **out = **in + } + if in.EnableDebugFlagsHandler != nil { + in, out := &in.EnableDebugFlagsHandler, &out.EnableDebugFlagsHandler + *out = new(bool) + **out = **in + } return } From af05a7eca32c0d63a0da521db64f53d685ea0ce0 Mon Sep 17 00:00:00 2001 From: Sri Saran Balaji Vellore Rajakumar Date: Wed, 27 Jan 2021 07:34:38 -0800 Subject: [PATCH 2/2] Refactor Kubelet Server to take kubeConfiguration instead of multiple fields --- cmd/kubelet/app/server.go | 4 +-- pkg/kubelet/kubelet.go | 14 ++++------ pkg/kubelet/server/BUILD | 2 ++ pkg/kubelet/server/server.go | 43 +++++++++++++------------------ pkg/kubelet/server/server_test.go | 36 +++++++++++++++++--------- 5 files changed, 50 insertions(+), 49 deletions(-) diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index ddba3464b38..6a60dc3206d 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -1187,9 +1187,7 @@ func startKubelet(k kubelet.Bootstrap, podCfg *config.PodConfig, kubeCfg *kubele // start the kubelet server if enableServer { - go k.ListenAndServe(net.ParseIP(kubeCfg.Address), uint(kubeCfg.Port), kubeDeps.TLSOptions, kubeDeps.Auth, - enableCAdvisorJSONEndpoints, kubeCfg.EnableDebuggingHandlers, kubeCfg.EnableContentionProfiling, - kubeCfg.EnableSystemLogHandler, kubeCfg.EnableProfilingHandler, kubeCfg.EnableDebugFlagsHandler) + go k.ListenAndServe(kubeCfg, kubeDeps.TLSOptions, kubeDeps.Auth, enableCAdvisorJSONEndpoints) } if kubeCfg.ReadOnlyPort > 0 { go k.ListenAndServeReadOnly(net.ParseIP(kubeCfg.Address), uint(kubeCfg.ReadOnlyPort), enableCAdvisorJSONEndpoints) diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 70b95b3cafa..6513882d8e1 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -195,9 +195,8 @@ type Bootstrap interface { GetConfiguration() kubeletconfiginternal.KubeletConfiguration BirthCry() StartGarbageCollection() - ListenAndServe(address net.IP, port uint, tlsOptions *server.TLSOptions, auth server.AuthInterface, - enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableContentionProfiling, enableSystemLogHandler, - enableProfilingHandler, enableDebugFlagsHandler bool) + ListenAndServe(kubeCfg *kubeletconfiginternal.KubeletConfiguration, tlsOptions *server.TLSOptions, auth server.AuthInterface, + enableCAdvisorJSONEndpoints bool) ListenAndServeReadOnly(address net.IP, port uint, enableCAdvisorJSONEndpoints bool) ListenAndServePodResources() Run(<-chan kubetypes.PodUpdate) @@ -2227,12 +2226,9 @@ func (kl *Kubelet) ResyncInterval() time.Duration { } // ListenAndServe runs the kubelet HTTP server. -func (kl *Kubelet) ListenAndServe(address net.IP, port uint, tlsOptions *server.TLSOptions, auth server.AuthInterface, - enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableContentionProfiling, enableSystemLogHandler, - enableProfilingHandler, enableDebugFlagsHandler bool) { - server.ListenAndServeKubeletServer(kl, kl.resourceAnalyzer, address, port, tlsOptions, auth, - enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableContentionProfiling, enableSystemLogHandler, - enableProfilingHandler, enableDebugFlagsHandler) +func (kl *Kubelet) ListenAndServe(kubeCfg *kubeletconfiginternal.KubeletConfiguration, tlsOptions *server.TLSOptions, + auth server.AuthInterface, enableCAdvisorJSONEndpoints bool) { + server.ListenAndServeKubeletServer(kl, kl.resourceAnalyzer, kubeCfg, tlsOptions, auth, enableCAdvisorJSONEndpoints) } // ListenAndServeReadOnly runs the kubelet HTTP server in read-only mode. diff --git a/pkg/kubelet/server/BUILD b/pkg/kubelet/server/BUILD index 2d6bf833237..187fcc37543 100644 --- a/pkg/kubelet/server/BUILD +++ b/pkg/kubelet/server/BUILD @@ -19,6 +19,7 @@ go_library( "//pkg/apis/core:go_default_library", "//pkg/apis/core/v1/validation:go_default_library", "//pkg/features:go_default_library", + "//pkg/kubelet/apis/config:go_default_library", "//pkg/kubelet/apis/podresources:go_default_library", "//pkg/kubelet/container:go_default_library", "//pkg/kubelet/cri/streaming:go_default_library", @@ -74,6 +75,7 @@ go_test( deps = [ "//pkg/apis/core:go_default_library", "//pkg/apis/core/install:go_default_library", + "//pkg/kubelet/apis/config:go_default_library", "//pkg/kubelet/cm:go_default_library", "//pkg/kubelet/container:go_default_library", "//pkg/kubelet/cri/streaming:go_default_library", diff --git a/pkg/kubelet/server/server.go b/pkg/kubelet/server/server.go index 4484fb9dc63..8f183f02108 100644 --- a/pkg/kubelet/server/server.go +++ b/pkg/kubelet/server/server.go @@ -66,6 +66,7 @@ import ( api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/core/v1/validation" "k8s.io/kubernetes/pkg/features" + kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config" "k8s.io/kubernetes/pkg/kubelet/apis/podresources" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/cri/streaming" @@ -139,19 +140,15 @@ func (a *filteringContainer) RegisteredHandlePaths() []string { func ListenAndServeKubeletServer( host HostInterface, resourceAnalyzer stats.ResourceAnalyzer, - address net.IP, - port uint, + kubeCfg *kubeletconfiginternal.KubeletConfiguration, tlsOptions *TLSOptions, auth AuthInterface, - enableCAdvisorJSONEndpoints, - enableDebuggingHandlers, - enableContentionProfiling, - enableSystemLogHandler, - enableProfilingHandler, - enableDebugFlagsHandler bool) { - klog.Infof("Starting to listen on %s:%d", address, port) - handler := NewServer(host, resourceAnalyzer, auth, enableCAdvisorJSONEndpoints, enableDebuggingHandlers, - enableContentionProfiling, enableSystemLogHandler, enableProfilingHandler, enableDebugFlagsHandler) + enableCAdvisorJSONEndpoints bool) { + + address := net.ParseIP(kubeCfg.Address) + port := uint(kubeCfg.Port) + klog.InfoS("Starting to listen", "address", address, "port", port) + handler := NewServer(host, resourceAnalyzer, auth, enableCAdvisorJSONEndpoints, kubeCfg) s := &http.Server{ Addr: net.JoinHostPort(address.String(), strconv.FormatUint(uint64(port), 10)), Handler: &handler, @@ -172,8 +169,8 @@ func ListenAndServeKubeletServer( // ListenAndServeKubeletReadOnlyServer initializes a server to respond to HTTP network requests on the Kubelet. func ListenAndServeKubeletReadOnlyServer(host HostInterface, resourceAnalyzer stats.ResourceAnalyzer, address net.IP, port uint, enableCAdvisorJSONEndpoints bool) { - klog.V(1).Infof("Starting to listen read-only on %s:%d", address, port) - s := NewServer(host, resourceAnalyzer, nil, enableCAdvisorJSONEndpoints, false, false, false, false, false) + klog.InfoS("Starting to listen read-only", "address", address, "port", port) + s := NewServer(host, resourceAnalyzer, nil, enableCAdvisorJSONEndpoints, nil) server := &http.Server{ Addr: net.JoinHostPort(address.String(), strconv.FormatUint(uint64(port), 10)), @@ -225,12 +222,8 @@ func NewServer( host HostInterface, resourceAnalyzer stats.ResourceAnalyzer, auth AuthInterface, - enableCAdvisorJSONEndpoints, - enableDebuggingHandlers, - enableContentionProfiling, - enableSystemLogHandler, - enableProfilingHandler, - enableDebugFlagsHandler bool) Server { + enableCAdvisorJSONEndpoints bool, + kubeCfg *kubeletconfiginternal.KubeletConfiguration) Server { server := Server{ host: host, resourceAnalyzer: resourceAnalyzer, @@ -243,13 +236,13 @@ func NewServer( server.InstallAuthFilter() } server.InstallDefaultHandlers(enableCAdvisorJSONEndpoints) - if enableDebuggingHandlers { + if kubeCfg != nil && kubeCfg.EnableDebuggingHandlers { server.InstallDebuggingHandlers() // To maintain backward compatibility serve logs and pprof only when enableDebuggingHandlers is also enabled // see https://github.com/kubernetes/kubernetes/pull/87273 - server.InstallSystemLogHandler(enableSystemLogHandler) - server.InstallProfilingHandler(enableProfilingHandler, enableContentionProfiling) - server.InstallDebugFlagsHandler(enableDebugFlagsHandler) + server.InstallSystemLogHandler(kubeCfg.EnableSystemLogHandler) + server.InstallProfilingHandler(kubeCfg.EnableProfilingHandler, kubeCfg.EnableContentionProfiling) + server.InstallDebugFlagsHandler(kubeCfg.EnableDebugFlagsHandler) } else { server.InstallDebuggingDisabledHandlers() } @@ -563,9 +556,9 @@ func (s *Server) InstallDebugFlagsHandler(enableDebugFlagsHandler bool) { } // InstallProfilingHandler registers the HTTP request patterns for /debug/pprof endpoint. -func (s *Server) InstallProfilingHandler(enableSystemLogHandler bool, enableContentionProfiling bool) { +func (s *Server) InstallProfilingHandler(enableProfilingLogHandler bool, enableContentionProfiling bool) { s.addMetricsBucketMatcher("debug") - if !enableSystemLogHandler { + if !enableProfilingLogHandler { s.restfulCont.Handle(pprofBasePath, getHandlerForDisabledEndpoint("profiling endpoint is disabled.")) return } diff --git a/pkg/kubelet/server/server_test.go b/pkg/kubelet/server/server_test.go index 9072c6b4304..745ff5843c6 100644 --- a/pkg/kubelet/server/server_test.go +++ b/pkg/kubelet/server/server_test.go @@ -55,6 +55,7 @@ import ( // Do some initialization to decode the query parameters correctly. _ "k8s.io/kubernetes/pkg/apis/core/install" + kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config" "k8s.io/kubernetes/pkg/kubelet/cm" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/cri/streaming" @@ -304,10 +305,17 @@ func newServerTest() *serverTestFramework { } func newServerTestWithDebug(enableDebugging bool, streamingServer streaming.Server) *serverTestFramework { - return newServerTestWithDebuggingHandlers(enableDebugging, enableDebugging, streamingServer) + kubeCfg := &kubeletconfiginternal.KubeletConfiguration{ + EnableDebuggingHandlers: enableDebugging, + EnableSystemLogHandler: enableDebugging, + EnableProfilingHandler: enableDebugging, + EnableDebugFlagsHandler: enableDebugging, + } + return newServerTestWithDebuggingHandlers(kubeCfg, streamingServer) } -func newServerTestWithDebuggingHandlers(enableDebugging, enableLogAndProfilingHandler bool, streamingServer streaming.Server) *serverTestFramework { +func newServerTestWithDebuggingHandlers(kubeCfg *kubeletconfiginternal.KubeletConfiguration, streamingServer streaming.Server) *serverTestFramework { + fw := &serverTestFramework{} fw.fakeKubelet = &fakeKubelet{ hostnameFunc: func() string { @@ -341,11 +349,7 @@ func newServerTestWithDebuggingHandlers(enableDebugging, enableLogAndProfilingHa stats.NewResourceAnalyzer(fw.fakeKubelet, time.Minute), fw.fakeAuth, true, - enableDebugging, - false, - enableLogAndProfilingHandler, - enableLogAndProfilingHandler, - enableLogAndProfilingHandler) + kubeCfg) fw.serverUnderTest = &server fw.testHTTPServer = httptest.NewServer(fw.serverUnderTest) return fw @@ -1506,9 +1510,15 @@ func TestMetricMethodBuckets(t *testing.T) { } func TestDebuggingDisabledHandlers(t *testing.T) { - // for backward compatibility even if enablesystemLogHandler is set but not enableDebuggingHandler then /logs - //shouldn't be served. - fw := newServerTestWithDebuggingHandlers(false, true, nil) + // for backward compatibility even if enablesystemLogHandler or enableProfilingHandler is set but not + // enableDebuggingHandler then /logs, /pprof and /flags shouldn't be served. + kubeCfg := &kubeletconfiginternal.KubeletConfiguration{ + EnableDebuggingHandlers: false, + EnableSystemLogHandler: true, + EnableDebugFlagsHandler: true, + EnableProfilingHandler: true, + } + fw := newServerTestWithDebuggingHandlers(kubeCfg, nil) defer fw.testHTTPServer.Close() paths := []string{ @@ -1548,11 +1558,13 @@ func TestDebuggingDisabledHandlers(t *testing.T) { resp, err = http.Get(fw.testHTTPServer.URL + "/spec") require.NoError(t, err) assert.Equal(t, http.StatusOK, resp.StatusCode) - } func TestDisablingLogAndProfilingHandler(t *testing.T) { - fw := newServerTestWithDebuggingHandlers(true, false, nil) + kubeCfg := &kubeletconfiginternal.KubeletConfiguration{ + EnableDebuggingHandlers: true, + } + fw := newServerTestWithDebuggingHandlers(kubeCfg, nil) defer fw.testHTTPServer.Close() // verify debug endpoints are disabled