mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-24 04:06:03 +00:00
Add support to disable /debug/pprof and /debug/flags/v endpoint
Co-authored-by: xiaofei.sun <sunxiaofei@kuaishou.com> Co-authored-by: SaranBalaji90 <srisaranbalaji@gmail.com>
This commit is contained in:
parent
f41def8d76
commit
51cdf4e97b
@ -1188,8 +1188,8 @@ func startKubelet(k kubelet.Bootstrap, podCfg *config.PodConfig, kubeCfg *kubele
|
||||
// start the kubelet server
|
||||
if enableServer {
|
||||
go k.ListenAndServe(net.ParseIP(kubeCfg.Address), uint(kubeCfg.Port), kubeDeps.TLSOptions, kubeDeps.Auth,
|
||||
enableCAdvisorJSONEndpoints, kubeCfg.EnableDebuggingHandlers, kubeCfg.EnableContentionProfiling, kubeCfg.EnableSystemLogHandler)
|
||||
|
||||
enableCAdvisorJSONEndpoints, kubeCfg.EnableDebuggingHandlers, kubeCfg.EnableContentionProfiling,
|
||||
kubeCfg.EnableSystemLogHandler, kubeCfg.EnableProfilingHandler, kubeCfg.EnableDebugFlagsHandler)
|
||||
}
|
||||
if kubeCfg.ReadOnlyPort > 0 {
|
||||
go k.ListenAndServeReadOnly(net.ParseIP(kubeCfg.Address), uint(kubeCfg.ReadOnlyPort), enableCAdvisorJSONEndpoints)
|
||||
|
@ -165,7 +165,9 @@ var (
|
||||
"ContentType",
|
||||
"EnableContentionProfiling",
|
||||
"EnableControllerAttachDetach",
|
||||
"EnableDebugFlagsHandler",
|
||||
"EnableDebuggingHandlers",
|
||||
"EnableProfilingHandler",
|
||||
"EnableServer",
|
||||
"EnableSystemLogHandler",
|
||||
"EnforceNodeAllocatable[*]",
|
||||
|
@ -23,7 +23,9 @@ cpuCFSQuotaPeriod: 100ms
|
||||
cpuManagerPolicy: none
|
||||
cpuManagerReconcilePeriod: 10s
|
||||
enableControllerAttachDetach: true
|
||||
enableDebugFlagsHandler: true
|
||||
enableDebuggingHandlers: true
|
||||
enableProfilingHandler: true
|
||||
enableServer: true
|
||||
enableSystemLogHandler: true
|
||||
enforceNodeAllocatable:
|
||||
|
@ -23,7 +23,9 @@ cpuCFSQuotaPeriod: 100ms
|
||||
cpuManagerPolicy: none
|
||||
cpuManagerReconcilePeriod: 10s
|
||||
enableControllerAttachDetach: true
|
||||
enableDebugFlagsHandler: true
|
||||
enableDebuggingHandlers: true
|
||||
enableProfilingHandler: true
|
||||
enableServer: true
|
||||
enableSystemLogHandler: true
|
||||
enforceNodeAllocatable:
|
||||
|
@ -399,6 +399,10 @@ type KubeletConfiguration struct {
|
||||
// 3. NUMAs nodes IDs that do not exist under the machine.
|
||||
// 4. memory types except for memory and hugepages-<size>
|
||||
ReservedMemory []MemoryReservation
|
||||
// EnableProfiling enables /debug/pprof handler.
|
||||
EnableProfilingHandler bool
|
||||
// EnableDebugFlagsHandler enables/debug/flags/v handler.
|
||||
EnableDebugFlagsHandler bool
|
||||
}
|
||||
|
||||
// KubeletAuthorizationMode denotes the authorization mode for the kubelet
|
||||
|
@ -246,4 +246,10 @@ func SetDefaults_KubeletConfiguration(obj *kubeletconfigv1beta1.KubeletConfigura
|
||||
if obj.EnableSystemLogHandler == nil {
|
||||
obj.EnableSystemLogHandler = utilpointer.BoolPtr(true)
|
||||
}
|
||||
if obj.EnableProfilingHandler == nil {
|
||||
obj.EnableProfilingHandler = utilpointer.BoolPtr(true)
|
||||
}
|
||||
if obj.EnableDebugFlagsHandler == nil {
|
||||
obj.EnableDebugFlagsHandler = utilpointer.BoolPtr(true)
|
||||
}
|
||||
}
|
||||
|
@ -365,6 +365,12 @@ func autoConvert_v1beta1_KubeletConfiguration_To_config_KubeletConfiguration(in
|
||||
out.ShutdownGracePeriod = in.ShutdownGracePeriod
|
||||
out.ShutdownGracePeriodCriticalPods = in.ShutdownGracePeriodCriticalPods
|
||||
out.ReservedMemory = *(*[]config.MemoryReservation)(unsafe.Pointer(&in.ReservedMemory))
|
||||
if err := v1.Convert_Pointer_bool_To_bool(&in.EnableProfilingHandler, &out.EnableProfilingHandler, s); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := v1.Convert_Pointer_bool_To_bool(&in.EnableDebugFlagsHandler, &out.EnableDebugFlagsHandler, s); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -520,6 +526,12 @@ func autoConvert_config_KubeletConfiguration_To_v1beta1_KubeletConfiguration(in
|
||||
out.ShutdownGracePeriod = in.ShutdownGracePeriod
|
||||
out.ShutdownGracePeriodCriticalPods = in.ShutdownGracePeriodCriticalPods
|
||||
out.ReservedMemory = *(*[]v1beta1.MemoryReservation)(unsafe.Pointer(&in.ReservedMemory))
|
||||
if err := v1.Convert_bool_To_Pointer_bool(&in.EnableProfilingHandler, &out.EnableProfilingHandler, s); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := v1.Convert_bool_To_Pointer_bool(&in.EnableDebugFlagsHandler, &out.EnableDebugFlagsHandler, s); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
@ -195,7 +195,9 @@ type Bootstrap interface {
|
||||
GetConfiguration() kubeletconfiginternal.KubeletConfiguration
|
||||
BirthCry()
|
||||
StartGarbageCollection()
|
||||
ListenAndServe(address net.IP, port uint, tlsOptions *server.TLSOptions, auth server.AuthInterface, enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableContentionProfiling, enableSystemLogHandler bool)
|
||||
ListenAndServe(address net.IP, port uint, tlsOptions *server.TLSOptions, auth server.AuthInterface,
|
||||
enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableContentionProfiling, enableSystemLogHandler,
|
||||
enableProfilingHandler, enableDebugFlagsHandler bool)
|
||||
ListenAndServeReadOnly(address net.IP, port uint, enableCAdvisorJSONEndpoints bool)
|
||||
ListenAndServePodResources()
|
||||
Run(<-chan kubetypes.PodUpdate)
|
||||
@ -2225,8 +2227,12 @@ func (kl *Kubelet) ResyncInterval() time.Duration {
|
||||
}
|
||||
|
||||
// ListenAndServe runs the kubelet HTTP server.
|
||||
func (kl *Kubelet) ListenAndServe(address net.IP, port uint, tlsOptions *server.TLSOptions, auth server.AuthInterface, enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableContentionProfiling, enableSystemLogHandler bool) {
|
||||
server.ListenAndServeKubeletServer(kl, kl.resourceAnalyzer, address, port, tlsOptions, auth, enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableContentionProfiling, enableSystemLogHandler)
|
||||
func (kl *Kubelet) ListenAndServe(address net.IP, port uint, tlsOptions *server.TLSOptions, auth server.AuthInterface,
|
||||
enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableContentionProfiling, enableSystemLogHandler,
|
||||
enableProfilingHandler, enableDebugFlagsHandler bool) {
|
||||
server.ListenAndServeKubeletServer(kl, kl.resourceAnalyzer, address, port, tlsOptions, auth,
|
||||
enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableContentionProfiling, enableSystemLogHandler,
|
||||
enableProfilingHandler, enableDebugFlagsHandler)
|
||||
}
|
||||
|
||||
// ListenAndServeReadOnly runs the kubelet HTTP server in read-only mode.
|
||||
|
@ -86,6 +86,8 @@ const (
|
||||
specPath = "/spec/"
|
||||
statsPath = "/stats/"
|
||||
logsPath = "/logs/"
|
||||
pprofBasePath = "/debug/pprof/"
|
||||
debugFlagPath = "/debug/flags/v"
|
||||
)
|
||||
|
||||
// Server is a http.Handler which exposes kubelet functionality over HTTP.
|
||||
@ -144,9 +146,12 @@ func ListenAndServeKubeletServer(
|
||||
enableCAdvisorJSONEndpoints,
|
||||
enableDebuggingHandlers,
|
||||
enableContentionProfiling,
|
||||
enableSystemLogHandler bool) {
|
||||
klog.InfoS("Starting to listen", "address", address, "port", port)
|
||||
handler := NewServer(host, resourceAnalyzer, auth, enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableContentionProfiling, enableSystemLogHandler)
|
||||
enableSystemLogHandler,
|
||||
enableProfilingHandler,
|
||||
enableDebugFlagsHandler bool) {
|
||||
klog.Infof("Starting to listen on %s:%d", address, port)
|
||||
handler := NewServer(host, resourceAnalyzer, auth, enableCAdvisorJSONEndpoints, enableDebuggingHandlers,
|
||||
enableContentionProfiling, enableSystemLogHandler, enableProfilingHandler, enableDebugFlagsHandler)
|
||||
s := &http.Server{
|
||||
Addr: net.JoinHostPort(address.String(), strconv.FormatUint(uint64(port), 10)),
|
||||
Handler: &handler,
|
||||
@ -167,8 +172,8 @@ func ListenAndServeKubeletServer(
|
||||
|
||||
// ListenAndServeKubeletReadOnlyServer initializes a server to respond to HTTP network requests on the Kubelet.
|
||||
func ListenAndServeKubeletReadOnlyServer(host HostInterface, resourceAnalyzer stats.ResourceAnalyzer, address net.IP, port uint, enableCAdvisorJSONEndpoints bool) {
|
||||
klog.InfoS("Starting to listen read-only", "address", address, "port", port)
|
||||
s := NewServer(host, resourceAnalyzer, nil, enableCAdvisorJSONEndpoints, false, false, false)
|
||||
klog.V(1).Infof("Starting to listen read-only on %s:%d", address, port)
|
||||
s := NewServer(host, resourceAnalyzer, nil, enableCAdvisorJSONEndpoints, false, false, false, false, false)
|
||||
|
||||
server := &http.Server{
|
||||
Addr: net.JoinHostPort(address.String(), strconv.FormatUint(uint64(port), 10)),
|
||||
@ -223,7 +228,9 @@ func NewServer(
|
||||
enableCAdvisorJSONEndpoints,
|
||||
enableDebuggingHandlers,
|
||||
enableContentionProfiling,
|
||||
enableSystemLogHandler bool) Server {
|
||||
enableSystemLogHandler,
|
||||
enableProfilingHandler,
|
||||
enableDebugFlagsHandler bool) Server {
|
||||
server := Server{
|
||||
host: host,
|
||||
resourceAnalyzer: resourceAnalyzer,
|
||||
@ -238,12 +245,11 @@ func NewServer(
|
||||
server.InstallDefaultHandlers(enableCAdvisorJSONEndpoints)
|
||||
if enableDebuggingHandlers {
|
||||
server.InstallDebuggingHandlers()
|
||||
// To maintain backward compatibility serve logs only when enableDebuggingHandlers is also enabled
|
||||
// To maintain backward compatibility serve logs and pprof only when enableDebuggingHandlers is also enabled
|
||||
// see https://github.com/kubernetes/kubernetes/pull/87273
|
||||
server.InstallSystemLogHandler(enableSystemLogHandler)
|
||||
if enableContentionProfiling {
|
||||
goruntime.SetBlockProfileRate(1)
|
||||
}
|
||||
server.InstallProfilingHandler(enableProfilingHandler, enableContentionProfiling)
|
||||
server.InstallDebugFlagsHandler(enableDebugFlagsHandler)
|
||||
} else {
|
||||
server.InstallDebuggingDisabledHandlers()
|
||||
}
|
||||
@ -403,8 +409,6 @@ func (s *Server) InstallDefaultHandlers(enableCAdvisorJSONEndpoints bool) {
|
||||
}
|
||||
}
|
||||
|
||||
const pprofBasePath = "/debug/pprof/"
|
||||
|
||||
// InstallDebuggingHandlers registers the HTTP request patterns that serve logs or run commands/containers
|
||||
func (s *Server) InstallDebuggingHandlers() {
|
||||
klog.InfoS("Adding debug handlers to kubelet server")
|
||||
@ -487,33 +491,6 @@ func (s *Server) InstallDebuggingHandlers() {
|
||||
s.addMetricsBucketMatcher("configz")
|
||||
configz.InstallHandler(s.restfulCont)
|
||||
|
||||
s.addMetricsBucketMatcher("debug")
|
||||
handlePprofEndpoint := func(req *restful.Request, resp *restful.Response) {
|
||||
name := strings.TrimPrefix(req.Request.URL.Path, pprofBasePath)
|
||||
switch name {
|
||||
case "profile":
|
||||
pprof.Profile(resp, req.Request)
|
||||
case "symbol":
|
||||
pprof.Symbol(resp, req.Request)
|
||||
case "cmdline":
|
||||
pprof.Cmdline(resp, req.Request)
|
||||
case "trace":
|
||||
pprof.Trace(resp, req.Request)
|
||||
default:
|
||||
pprof.Index(resp, req.Request)
|
||||
}
|
||||
}
|
||||
// Setup pprof handlers.
|
||||
ws = new(restful.WebService).Path(pprofBasePath)
|
||||
ws.Route(ws.GET("/{subpath:*}").To(func(req *restful.Request, resp *restful.Response) {
|
||||
handlePprofEndpoint(req, resp)
|
||||
})).Doc("pprof endpoint")
|
||||
s.restfulCont.Add(ws)
|
||||
|
||||
// Setup flags handlers.
|
||||
// so far, only logging related endpoints are considered valid to add for these debug flags.
|
||||
s.restfulCont.Handle("/debug/flags/v", routes.StringFlagPutHandler(logs.GlogSetter))
|
||||
|
||||
// The /runningpods endpoint is used for testing only.
|
||||
s.addMetricsBucketMatcher("runningpods")
|
||||
ws = new(restful.WebService)
|
||||
@ -563,10 +540,59 @@ func (s *Server) InstallSystemLogHandler(enableSystemLogHandler bool) {
|
||||
Param(ws.PathParameter("logpath", "path to the log").DataType("string")))
|
||||
s.restfulCont.Add(ws)
|
||||
} else {
|
||||
h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "logs endpoint is disabled.", http.StatusMethodNotAllowed)
|
||||
})
|
||||
s.restfulCont.Handle(logsPath, h)
|
||||
s.restfulCont.Handle(logsPath, getHandlerForDisabledEndpoint("logs endpoint is disabled."))
|
||||
}
|
||||
}
|
||||
|
||||
func getHandlerForDisabledEndpoint(errorMessage string) http.HandlerFunc {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, errorMessage, http.StatusMethodNotAllowed)
|
||||
})
|
||||
}
|
||||
|
||||
// InstallDebugFlagsHandler registers the HTTP request patterns for /debug/flags/v endpoint.
|
||||
func (s *Server) InstallDebugFlagsHandler(enableDebugFlagsHandler bool) {
|
||||
if enableDebugFlagsHandler {
|
||||
// Setup flags handlers.
|
||||
// so far, only logging related endpoints are considered valid to add for these debug flags.
|
||||
s.restfulCont.Handle(debugFlagPath, routes.StringFlagPutHandler(logs.GlogSetter))
|
||||
} else {
|
||||
s.restfulCont.Handle(debugFlagPath, getHandlerForDisabledEndpoint("flags endpoint is disabled."))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// InstallProfilingHandler registers the HTTP request patterns for /debug/pprof endpoint.
|
||||
func (s *Server) InstallProfilingHandler(enableSystemLogHandler bool, enableContentionProfiling bool) {
|
||||
s.addMetricsBucketMatcher("debug")
|
||||
if !enableSystemLogHandler {
|
||||
s.restfulCont.Handle(pprofBasePath, getHandlerForDisabledEndpoint("profiling endpoint is disabled."))
|
||||
return
|
||||
}
|
||||
|
||||
handlePprofEndpoint := func(req *restful.Request, resp *restful.Response) {
|
||||
name := strings.TrimPrefix(req.Request.URL.Path, pprofBasePath)
|
||||
switch name {
|
||||
case "profile":
|
||||
pprof.Profile(resp, req.Request)
|
||||
case "symbol":
|
||||
pprof.Symbol(resp, req.Request)
|
||||
case "cmdline":
|
||||
pprof.Cmdline(resp, req.Request)
|
||||
case "trace":
|
||||
pprof.Trace(resp, req.Request)
|
||||
default:
|
||||
pprof.Index(resp, req.Request)
|
||||
}
|
||||
}
|
||||
|
||||
// Setup pprof handlers.
|
||||
ws := new(restful.WebService).Path(pprofBasePath)
|
||||
ws.Route(ws.GET("/{subpath:*}").To(handlePprofEndpoint)).Doc("pprof endpoint")
|
||||
s.restfulCont.Add(ws)
|
||||
|
||||
if enableContentionProfiling {
|
||||
goruntime.SetBlockProfileRate(1)
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -307,7 +307,7 @@ func newServerTestWithDebug(enableDebugging bool, streamingServer streaming.Serv
|
||||
return newServerTestWithDebuggingHandlers(enableDebugging, enableDebugging, streamingServer)
|
||||
}
|
||||
|
||||
func newServerTestWithDebuggingHandlers(enableDebugging, enableSystemLogHandler bool, streamingServer streaming.Server) *serverTestFramework {
|
||||
func newServerTestWithDebuggingHandlers(enableDebugging, enableLogAndProfilingHandler bool, streamingServer streaming.Server) *serverTestFramework {
|
||||
fw := &serverTestFramework{}
|
||||
fw.fakeKubelet = &fakeKubelet{
|
||||
hostnameFunc: func() string {
|
||||
@ -343,7 +343,9 @@ func newServerTestWithDebuggingHandlers(enableDebugging, enableSystemLogHandler
|
||||
true,
|
||||
enableDebugging,
|
||||
false,
|
||||
enableSystemLogHandler)
|
||||
enableLogAndProfilingHandler,
|
||||
enableLogAndProfilingHandler,
|
||||
enableLogAndProfilingHandler)
|
||||
fw.serverUnderTest = &server
|
||||
fw.testHTTPServer = httptest.NewServer(fw.serverUnderTest)
|
||||
return fw
|
||||
@ -1549,12 +1551,14 @@ func TestDebuggingDisabledHandlers(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
func TestDisablingSystemLogHandler(t *testing.T) {
|
||||
func TestDisablingLogAndProfilingHandler(t *testing.T) {
|
||||
fw := newServerTestWithDebuggingHandlers(true, false, nil)
|
||||
defer fw.testHTTPServer.Close()
|
||||
|
||||
// verify logs endpoint is disabled
|
||||
// verify debug endpoints are disabled
|
||||
verifyEndpointResponse(t, fw, "/logs/kubelet.log", "logs endpoint is disabled.\n")
|
||||
verifyEndpointResponse(t, fw, "/debug/pprof/profile?seconds=2", "profiling endpoint is disabled.\n")
|
||||
verifyEndpointResponse(t, fw, "/debug/flags/v", "flags endpoint is disabled.\n")
|
||||
}
|
||||
|
||||
func TestFailedParseParamsSummaryHandler(t *testing.T) {
|
||||
|
@ -854,6 +854,14 @@ type KubeletConfiguration struct {
|
||||
// Default: nil
|
||||
// +optional
|
||||
ReservedMemory []MemoryReservation `json:"reservedMemory,omitempty"`
|
||||
// enableProfilingHandler enables profiling via web interface host:port/debug/pprof/
|
||||
// Default: true
|
||||
// +optional
|
||||
EnableProfilingHandler *bool `json:"enableProfilingHandler,omitempty"`
|
||||
// enableDebugFlagsHandler enables flags endpoint via web interface host:port/debug/flags/v
|
||||
// Default: true
|
||||
// +optional
|
||||
EnableDebugFlagsHandler *bool `json:"enableDebugFlagsHandler,omitempty"`
|
||||
}
|
||||
|
||||
type KubeletAuthorizationMode string
|
||||
|
@ -311,6 +311,16 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) {
|
||||
(*in)[i].DeepCopyInto(&(*out)[i])
|
||||
}
|
||||
}
|
||||
if in.EnableProfilingHandler != nil {
|
||||
in, out := &in.EnableProfilingHandler, &out.EnableProfilingHandler
|
||||
*out = new(bool)
|
||||
**out = **in
|
||||
}
|
||||
if in.EnableDebugFlagsHandler != nil {
|
||||
in, out := &in.EnableDebugFlagsHandler, &out.EnableDebugFlagsHandler
|
||||
*out = new(bool)
|
||||
**out = **in
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user