Merge pull request #98458 from SaranBalaji90/profiling

Add support to disable /debug/pprof and /debug/flags/v endpoint
This commit is contained in:
Kubernetes Prow Robot 2021-02-16 10:12:58 -08:00 committed by GitHub
commit d0a433fa45
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 152 additions and 69 deletions

View File

@ -1187,9 +1187,7 @@ func startKubelet(k kubelet.Bootstrap, podCfg *config.PodConfig, kubeCfg *kubele
// start the kubelet server
if enableServer {
go k.ListenAndServe(net.ParseIP(kubeCfg.Address), uint(kubeCfg.Port), kubeDeps.TLSOptions, kubeDeps.Auth,
enableCAdvisorJSONEndpoints, kubeCfg.EnableDebuggingHandlers, kubeCfg.EnableContentionProfiling, kubeCfg.EnableSystemLogHandler)
go k.ListenAndServe(kubeCfg, kubeDeps.TLSOptions, kubeDeps.Auth, enableCAdvisorJSONEndpoints)
}
if kubeCfg.ReadOnlyPort > 0 {
go k.ListenAndServeReadOnly(net.ParseIP(kubeCfg.Address), uint(kubeCfg.ReadOnlyPort), enableCAdvisorJSONEndpoints)

View File

@ -165,7 +165,9 @@ var (
"ContentType",
"EnableContentionProfiling",
"EnableControllerAttachDetach",
"EnableDebugFlagsHandler",
"EnableDebuggingHandlers",
"EnableProfilingHandler",
"EnableServer",
"EnableSystemLogHandler",
"EnforceNodeAllocatable[*]",

View File

@ -23,7 +23,9 @@ cpuCFSQuotaPeriod: 100ms
cpuManagerPolicy: none
cpuManagerReconcilePeriod: 10s
enableControllerAttachDetach: true
enableDebugFlagsHandler: true
enableDebuggingHandlers: true
enableProfilingHandler: true
enableServer: true
enableSystemLogHandler: true
enforceNodeAllocatable:

View File

@ -23,7 +23,9 @@ cpuCFSQuotaPeriod: 100ms
cpuManagerPolicy: none
cpuManagerReconcilePeriod: 10s
enableControllerAttachDetach: true
enableDebugFlagsHandler: true
enableDebuggingHandlers: true
enableProfilingHandler: true
enableServer: true
enableSystemLogHandler: true
enforceNodeAllocatable:

View File

@ -399,6 +399,10 @@ type KubeletConfiguration struct {
// 3. NUMAs nodes IDs that do not exist under the machine.
// 4. memory types except for memory and hugepages-<size>
ReservedMemory []MemoryReservation
// EnableProfiling enables /debug/pprof handler.
EnableProfilingHandler bool
// EnableDebugFlagsHandler enables/debug/flags/v handler.
EnableDebugFlagsHandler bool
}
// KubeletAuthorizationMode denotes the authorization mode for the kubelet

View File

@ -246,4 +246,10 @@ func SetDefaults_KubeletConfiguration(obj *kubeletconfigv1beta1.KubeletConfigura
if obj.EnableSystemLogHandler == nil {
obj.EnableSystemLogHandler = utilpointer.BoolPtr(true)
}
if obj.EnableProfilingHandler == nil {
obj.EnableProfilingHandler = utilpointer.BoolPtr(true)
}
if obj.EnableDebugFlagsHandler == nil {
obj.EnableDebugFlagsHandler = utilpointer.BoolPtr(true)
}
}

View File

@ -365,6 +365,12 @@ func autoConvert_v1beta1_KubeletConfiguration_To_config_KubeletConfiguration(in
out.ShutdownGracePeriod = in.ShutdownGracePeriod
out.ShutdownGracePeriodCriticalPods = in.ShutdownGracePeriodCriticalPods
out.ReservedMemory = *(*[]config.MemoryReservation)(unsafe.Pointer(&in.ReservedMemory))
if err := v1.Convert_Pointer_bool_To_bool(&in.EnableProfilingHandler, &out.EnableProfilingHandler, s); err != nil {
return err
}
if err := v1.Convert_Pointer_bool_To_bool(&in.EnableDebugFlagsHandler, &out.EnableDebugFlagsHandler, s); err != nil {
return err
}
return nil
}
@ -520,6 +526,12 @@ func autoConvert_config_KubeletConfiguration_To_v1beta1_KubeletConfiguration(in
out.ShutdownGracePeriod = in.ShutdownGracePeriod
out.ShutdownGracePeriodCriticalPods = in.ShutdownGracePeriodCriticalPods
out.ReservedMemory = *(*[]v1beta1.MemoryReservation)(unsafe.Pointer(&in.ReservedMemory))
if err := v1.Convert_bool_To_Pointer_bool(&in.EnableProfilingHandler, &out.EnableProfilingHandler, s); err != nil {
return err
}
if err := v1.Convert_bool_To_Pointer_bool(&in.EnableDebugFlagsHandler, &out.EnableDebugFlagsHandler, s); err != nil {
return err
}
return nil
}

View File

@ -195,7 +195,8 @@ type Bootstrap interface {
GetConfiguration() kubeletconfiginternal.KubeletConfiguration
BirthCry()
StartGarbageCollection()
ListenAndServe(address net.IP, port uint, tlsOptions *server.TLSOptions, auth server.AuthInterface, enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableContentionProfiling, enableSystemLogHandler bool)
ListenAndServe(kubeCfg *kubeletconfiginternal.KubeletConfiguration, tlsOptions *server.TLSOptions, auth server.AuthInterface,
enableCAdvisorJSONEndpoints bool)
ListenAndServeReadOnly(address net.IP, port uint, enableCAdvisorJSONEndpoints bool)
ListenAndServePodResources()
Run(<-chan kubetypes.PodUpdate)
@ -2225,8 +2226,9 @@ func (kl *Kubelet) ResyncInterval() time.Duration {
}
// ListenAndServe runs the kubelet HTTP server.
func (kl *Kubelet) ListenAndServe(address net.IP, port uint, tlsOptions *server.TLSOptions, auth server.AuthInterface, enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableContentionProfiling, enableSystemLogHandler bool) {
server.ListenAndServeKubeletServer(kl, kl.resourceAnalyzer, address, port, tlsOptions, auth, enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableContentionProfiling, enableSystemLogHandler)
func (kl *Kubelet) ListenAndServe(kubeCfg *kubeletconfiginternal.KubeletConfiguration, tlsOptions *server.TLSOptions,
auth server.AuthInterface, enableCAdvisorJSONEndpoints bool) {
server.ListenAndServeKubeletServer(kl, kl.resourceAnalyzer, kubeCfg, tlsOptions, auth, enableCAdvisorJSONEndpoints)
}
// ListenAndServeReadOnly runs the kubelet HTTP server in read-only mode.

View File

@ -19,6 +19,7 @@ go_library(
"//pkg/apis/core:go_default_library",
"//pkg/apis/core/v1/validation:go_default_library",
"//pkg/features:go_default_library",
"//pkg/kubelet/apis/config:go_default_library",
"//pkg/kubelet/apis/podresources:go_default_library",
"//pkg/kubelet/container:go_default_library",
"//pkg/kubelet/cri/streaming:go_default_library",
@ -74,6 +75,7 @@ go_test(
deps = [
"//pkg/apis/core:go_default_library",
"//pkg/apis/core/install:go_default_library",
"//pkg/kubelet/apis/config:go_default_library",
"//pkg/kubelet/cm:go_default_library",
"//pkg/kubelet/container:go_default_library",
"//pkg/kubelet/cri/streaming:go_default_library",

View File

@ -66,6 +66,7 @@ import (
api "k8s.io/kubernetes/pkg/apis/core"
"k8s.io/kubernetes/pkg/apis/core/v1/validation"
"k8s.io/kubernetes/pkg/features"
kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config"
"k8s.io/kubernetes/pkg/kubelet/apis/podresources"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/kubelet/cri/streaming"
@ -86,6 +87,8 @@ const (
specPath = "/spec/"
statsPath = "/stats/"
logsPath = "/logs/"
pprofBasePath = "/debug/pprof/"
debugFlagPath = "/debug/flags/v"
)
// Server is a http.Handler which exposes kubelet functionality over HTTP.
@ -137,16 +140,15 @@ func (a *filteringContainer) RegisteredHandlePaths() []string {
func ListenAndServeKubeletServer(
host HostInterface,
resourceAnalyzer stats.ResourceAnalyzer,
address net.IP,
port uint,
kubeCfg *kubeletconfiginternal.KubeletConfiguration,
tlsOptions *TLSOptions,
auth AuthInterface,
enableCAdvisorJSONEndpoints,
enableDebuggingHandlers,
enableContentionProfiling,
enableSystemLogHandler bool) {
enableCAdvisorJSONEndpoints bool) {
address := net.ParseIP(kubeCfg.Address)
port := uint(kubeCfg.Port)
klog.InfoS("Starting to listen", "address", address, "port", port)
handler := NewServer(host, resourceAnalyzer, auth, enableCAdvisorJSONEndpoints, enableDebuggingHandlers, enableContentionProfiling, enableSystemLogHandler)
handler := NewServer(host, resourceAnalyzer, auth, enableCAdvisorJSONEndpoints, kubeCfg)
s := &http.Server{
Addr: net.JoinHostPort(address.String(), strconv.FormatUint(uint64(port), 10)),
Handler: &handler,
@ -168,7 +170,7 @@ func ListenAndServeKubeletServer(
// ListenAndServeKubeletReadOnlyServer initializes a server to respond to HTTP network requests on the Kubelet.
func ListenAndServeKubeletReadOnlyServer(host HostInterface, resourceAnalyzer stats.ResourceAnalyzer, address net.IP, port uint, enableCAdvisorJSONEndpoints bool) {
klog.InfoS("Starting to listen read-only", "address", address, "port", port)
s := NewServer(host, resourceAnalyzer, nil, enableCAdvisorJSONEndpoints, false, false, false)
s := NewServer(host, resourceAnalyzer, nil, enableCAdvisorJSONEndpoints, nil)
server := &http.Server{
Addr: net.JoinHostPort(address.String(), strconv.FormatUint(uint64(port), 10)),
@ -220,10 +222,8 @@ func NewServer(
host HostInterface,
resourceAnalyzer stats.ResourceAnalyzer,
auth AuthInterface,
enableCAdvisorJSONEndpoints,
enableDebuggingHandlers,
enableContentionProfiling,
enableSystemLogHandler bool) Server {
enableCAdvisorJSONEndpoints bool,
kubeCfg *kubeletconfiginternal.KubeletConfiguration) Server {
server := Server{
host: host,
resourceAnalyzer: resourceAnalyzer,
@ -236,14 +236,13 @@ func NewServer(
server.InstallAuthFilter()
}
server.InstallDefaultHandlers(enableCAdvisorJSONEndpoints)
if enableDebuggingHandlers {
if kubeCfg != nil && kubeCfg.EnableDebuggingHandlers {
server.InstallDebuggingHandlers()
// To maintain backward compatibility serve logs only when enableDebuggingHandlers is also enabled
// To maintain backward compatibility serve logs and pprof only when enableDebuggingHandlers is also enabled
// see https://github.com/kubernetes/kubernetes/pull/87273
server.InstallSystemLogHandler(enableSystemLogHandler)
if enableContentionProfiling {
goruntime.SetBlockProfileRate(1)
}
server.InstallSystemLogHandler(kubeCfg.EnableSystemLogHandler)
server.InstallProfilingHandler(kubeCfg.EnableProfilingHandler, kubeCfg.EnableContentionProfiling)
server.InstallDebugFlagsHandler(kubeCfg.EnableDebugFlagsHandler)
} else {
server.InstallDebuggingDisabledHandlers()
}
@ -403,8 +402,6 @@ func (s *Server) InstallDefaultHandlers(enableCAdvisorJSONEndpoints bool) {
}
}
const pprofBasePath = "/debug/pprof/"
// InstallDebuggingHandlers registers the HTTP request patterns that serve logs or run commands/containers
func (s *Server) InstallDebuggingHandlers() {
klog.InfoS("Adding debug handlers to kubelet server")
@ -487,33 +484,6 @@ func (s *Server) InstallDebuggingHandlers() {
s.addMetricsBucketMatcher("configz")
configz.InstallHandler(s.restfulCont)
s.addMetricsBucketMatcher("debug")
handlePprofEndpoint := func(req *restful.Request, resp *restful.Response) {
name := strings.TrimPrefix(req.Request.URL.Path, pprofBasePath)
switch name {
case "profile":
pprof.Profile(resp, req.Request)
case "symbol":
pprof.Symbol(resp, req.Request)
case "cmdline":
pprof.Cmdline(resp, req.Request)
case "trace":
pprof.Trace(resp, req.Request)
default:
pprof.Index(resp, req.Request)
}
}
// Setup pprof handlers.
ws = new(restful.WebService).Path(pprofBasePath)
ws.Route(ws.GET("/{subpath:*}").To(func(req *restful.Request, resp *restful.Response) {
handlePprofEndpoint(req, resp)
})).Doc("pprof endpoint")
s.restfulCont.Add(ws)
// Setup flags handlers.
// so far, only logging related endpoints are considered valid to add for these debug flags.
s.restfulCont.Handle("/debug/flags/v", routes.StringFlagPutHandler(logs.GlogSetter))
// The /runningpods endpoint is used for testing only.
s.addMetricsBucketMatcher("runningpods")
ws = new(restful.WebService)
@ -563,10 +533,59 @@ func (s *Server) InstallSystemLogHandler(enableSystemLogHandler bool) {
Param(ws.PathParameter("logpath", "path to the log").DataType("string")))
s.restfulCont.Add(ws)
} else {
h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "logs endpoint is disabled.", http.StatusMethodNotAllowed)
})
s.restfulCont.Handle(logsPath, h)
s.restfulCont.Handle(logsPath, getHandlerForDisabledEndpoint("logs endpoint is disabled."))
}
}
func getHandlerForDisabledEndpoint(errorMessage string) http.HandlerFunc {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, errorMessage, http.StatusMethodNotAllowed)
})
}
// InstallDebugFlagsHandler registers the HTTP request patterns for /debug/flags/v endpoint.
func (s *Server) InstallDebugFlagsHandler(enableDebugFlagsHandler bool) {
if enableDebugFlagsHandler {
// Setup flags handlers.
// so far, only logging related endpoints are considered valid to add for these debug flags.
s.restfulCont.Handle(debugFlagPath, routes.StringFlagPutHandler(logs.GlogSetter))
} else {
s.restfulCont.Handle(debugFlagPath, getHandlerForDisabledEndpoint("flags endpoint is disabled."))
return
}
}
// InstallProfilingHandler registers the HTTP request patterns for /debug/pprof endpoint.
func (s *Server) InstallProfilingHandler(enableProfilingLogHandler bool, enableContentionProfiling bool) {
s.addMetricsBucketMatcher("debug")
if !enableProfilingLogHandler {
s.restfulCont.Handle(pprofBasePath, getHandlerForDisabledEndpoint("profiling endpoint is disabled."))
return
}
handlePprofEndpoint := func(req *restful.Request, resp *restful.Response) {
name := strings.TrimPrefix(req.Request.URL.Path, pprofBasePath)
switch name {
case "profile":
pprof.Profile(resp, req.Request)
case "symbol":
pprof.Symbol(resp, req.Request)
case "cmdline":
pprof.Cmdline(resp, req.Request)
case "trace":
pprof.Trace(resp, req.Request)
default:
pprof.Index(resp, req.Request)
}
}
// Setup pprof handlers.
ws := new(restful.WebService).Path(pprofBasePath)
ws.Route(ws.GET("/{subpath:*}").To(handlePprofEndpoint)).Doc("pprof endpoint")
s.restfulCont.Add(ws)
if enableContentionProfiling {
goruntime.SetBlockProfileRate(1)
}
}

View File

@ -55,6 +55,7 @@ import (
// Do some initialization to decode the query parameters correctly.
_ "k8s.io/kubernetes/pkg/apis/core/install"
kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config"
"k8s.io/kubernetes/pkg/kubelet/cm"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/kubelet/cri/streaming"
@ -304,10 +305,17 @@ func newServerTest() *serverTestFramework {
}
func newServerTestWithDebug(enableDebugging bool, streamingServer streaming.Server) *serverTestFramework {
return newServerTestWithDebuggingHandlers(enableDebugging, enableDebugging, streamingServer)
kubeCfg := &kubeletconfiginternal.KubeletConfiguration{
EnableDebuggingHandlers: enableDebugging,
EnableSystemLogHandler: enableDebugging,
EnableProfilingHandler: enableDebugging,
EnableDebugFlagsHandler: enableDebugging,
}
return newServerTestWithDebuggingHandlers(kubeCfg, streamingServer)
}
func newServerTestWithDebuggingHandlers(enableDebugging, enableSystemLogHandler bool, streamingServer streaming.Server) *serverTestFramework {
func newServerTestWithDebuggingHandlers(kubeCfg *kubeletconfiginternal.KubeletConfiguration, streamingServer streaming.Server) *serverTestFramework {
fw := &serverTestFramework{}
fw.fakeKubelet = &fakeKubelet{
hostnameFunc: func() string {
@ -341,9 +349,7 @@ func newServerTestWithDebuggingHandlers(enableDebugging, enableSystemLogHandler
stats.NewResourceAnalyzer(fw.fakeKubelet, time.Minute),
fw.fakeAuth,
true,
enableDebugging,
false,
enableSystemLogHandler)
kubeCfg)
fw.serverUnderTest = &server
fw.testHTTPServer = httptest.NewServer(fw.serverUnderTest)
return fw
@ -1504,9 +1510,15 @@ func TestMetricMethodBuckets(t *testing.T) {
}
func TestDebuggingDisabledHandlers(t *testing.T) {
// for backward compatibility even if enablesystemLogHandler is set but not enableDebuggingHandler then /logs
//shouldn't be served.
fw := newServerTestWithDebuggingHandlers(false, true, nil)
// for backward compatibility even if enablesystemLogHandler or enableProfilingHandler is set but not
// enableDebuggingHandler then /logs, /pprof and /flags shouldn't be served.
kubeCfg := &kubeletconfiginternal.KubeletConfiguration{
EnableDebuggingHandlers: false,
EnableSystemLogHandler: true,
EnableDebugFlagsHandler: true,
EnableProfilingHandler: true,
}
fw := newServerTestWithDebuggingHandlers(kubeCfg, nil)
defer fw.testHTTPServer.Close()
paths := []string{
@ -1546,15 +1558,19 @@ func TestDebuggingDisabledHandlers(t *testing.T) {
resp, err = http.Get(fw.testHTTPServer.URL + "/spec")
require.NoError(t, err)
assert.Equal(t, http.StatusOK, resp.StatusCode)
}
func TestDisablingSystemLogHandler(t *testing.T) {
fw := newServerTestWithDebuggingHandlers(true, false, nil)
func TestDisablingLogAndProfilingHandler(t *testing.T) {
kubeCfg := &kubeletconfiginternal.KubeletConfiguration{
EnableDebuggingHandlers: true,
}
fw := newServerTestWithDebuggingHandlers(kubeCfg, nil)
defer fw.testHTTPServer.Close()
// verify logs endpoint is disabled
// verify debug endpoints are disabled
verifyEndpointResponse(t, fw, "/logs/kubelet.log", "logs endpoint is disabled.\n")
verifyEndpointResponse(t, fw, "/debug/pprof/profile?seconds=2", "profiling endpoint is disabled.\n")
verifyEndpointResponse(t, fw, "/debug/flags/v", "flags endpoint is disabled.\n")
}
func TestFailedParseParamsSummaryHandler(t *testing.T) {

View File

@ -854,6 +854,14 @@ type KubeletConfiguration struct {
// Default: nil
// +optional
ReservedMemory []MemoryReservation `json:"reservedMemory,omitempty"`
// enableProfilingHandler enables profiling via web interface host:port/debug/pprof/
// Default: true
// +optional
EnableProfilingHandler *bool `json:"enableProfilingHandler,omitempty"`
// enableDebugFlagsHandler enables flags endpoint via web interface host:port/debug/flags/v
// Default: true
// +optional
EnableDebugFlagsHandler *bool `json:"enableDebugFlagsHandler,omitempty"`
}
type KubeletAuthorizationMode string

View File

@ -311,6 +311,16 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
if in.EnableProfilingHandler != nil {
in, out := &in.EnableProfilingHandler, &out.EnableProfilingHandler
*out = new(bool)
**out = **in
}
if in.EnableDebugFlagsHandler != nil {
in, out := &in.EnableDebugFlagsHandler, &out.EnableDebugFlagsHandler
*out = new(bool)
**out = **in
}
return
}