diff --git a/src/runtime/cli/config/configuration-acrn.toml.in b/src/runtime/cli/config/configuration-acrn.toml.in index d1c581243..98e279dce 100644 --- a/src/runtime/cli/config/configuration-acrn.toml.in +++ b/src/runtime/cli/config/configuration-acrn.toml.in @@ -235,3 +235,7 @@ sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@ # Supported experimental features: # (default: []) experimental=@DEFAULTEXPFEATURES@ + +# If enabled, user can run pprof tools with shim v2 process through kata-monitor. +# (default: false) +# EnablePprof = true diff --git a/src/runtime/cli/config/configuration-clh.toml.in b/src/runtime/cli/config/configuration-clh.toml.in index 6718f4a0d..197a6f323 100644 --- a/src/runtime/cli/config/configuration-clh.toml.in +++ b/src/runtime/cli/config/configuration-clh.toml.in @@ -236,3 +236,7 @@ sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@ # Supported experimental features: # (default: []) experimental=@DEFAULTEXPFEATURES@ + +# If enabled, user can run pprof tools with shim v2 process through kata-monitor. +# (default: false) +# EnablePprof = true diff --git a/src/runtime/cli/config/configuration-fc.toml.in b/src/runtime/cli/config/configuration-fc.toml.in index 4e5158f5b..d04fd3106 100644 --- a/src/runtime/cli/config/configuration-fc.toml.in +++ b/src/runtime/cli/config/configuration-fc.toml.in @@ -355,3 +355,7 @@ sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@ # Supported experimental features: # (default: []) experimental=@DEFAULTEXPFEATURES@ + +# If enabled, user can run pprof tools with shim v2 process through kata-monitor. +# (default: false) +# EnablePprof = true diff --git a/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in b/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in index 579d26434..8d28e13d0 100644 --- a/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in +++ b/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in @@ -462,3 +462,7 @@ sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@ # Supported experimental features: # (default: []) experimental=@DEFAULTEXPFEATURES@ + +# If enabled, user can run pprof tools with shim v2 process through kata-monitor. +# (default: false) +# EnablePprof = true diff --git a/src/runtime/cli/config/configuration-qemu.toml.in b/src/runtime/cli/config/configuration-qemu.toml.in index 4bf1d6914..81bdfb430 100644 --- a/src/runtime/cli/config/configuration-qemu.toml.in +++ b/src/runtime/cli/config/configuration-qemu.toml.in @@ -480,3 +480,7 @@ sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@ # Supported experimental features: # (default: []) experimental=@DEFAULTEXPFEATURES@ + +# If enabled, user can run pprof tools with shim v2 process through kata-monitor. +# (default: false) +# EnablePprof = true diff --git a/src/runtime/cli/kata-monitor/main.go b/src/runtime/cli/kata-monitor/main.go index da8b868dd..1b5920361 100644 --- a/src/runtime/cli/kata-monitor/main.go +++ b/src/runtime/cli/kata-monitor/main.go @@ -15,7 +15,7 @@ import ( "github.com/sirupsen/logrus" ) -var metricListenAddr = flag.String("listen-address", ":8090", "The address to listen on for HTTP requests.") +var monitorListenAddr = flag.String("listen-address", ":8090", "The address to listen on for HTTP requests.") var containerdAddr = flag.String("containerd-address", "/run/containerd/containerd.sock", "Containerd address to accept client requests.") var containerdConfig = flag.String("containerd-conf", "/etc/containerd/config.toml", "Containerd config file.") var logLevel = flag.String("log-level", "info", "Log level of logrus(trace/debug/info/warn/error/fatal/panic).") @@ -26,17 +26,31 @@ func main() { // init logrus initLog() - // create new MAgent - ma, err := kataMonitor.NewKataMonitor(*containerdAddr, *containerdConfig) + // create new kataMonitor + km, err := kataMonitor.NewKataMonitor(*containerdAddr, *containerdConfig) if err != nil { panic(err) } // setup handlers, now only metrics is supported - http.HandleFunc("/metrics", ma.ProcessMetricsRequest) + m := http.NewServeMux() + m.Handle("/metrics", http.HandlerFunc(km.ProcessMetricsRequest)) + m.Handle("/sandboxes", http.HandlerFunc(km.ListSandboxes)) + + // for debug shim process + m.Handle("/debug/vars", http.HandlerFunc(km.ExpvarHandler)) + m.Handle("/debug/pprof/", http.HandlerFunc(km.PprofIndex)) + m.Handle("/debug/pprof/cmdline", http.HandlerFunc(km.PprofCmdline)) + m.Handle("/debug/pprof/profile", http.HandlerFunc(km.PprofProfile)) + m.Handle("/debug/pprof/symbol", http.HandlerFunc(km.PprofSymbol)) + m.Handle("/debug/pprof/trace", http.HandlerFunc(km.PprofTrace)) // listening on the server - logrus.Fatal(http.ListenAndServe(*metricListenAddr, nil)) + svr := &http.Server{ + Handler: m, + Addr: *monitorListenAddr, + } + logrus.Fatal(svr.ListenAndServe()) } // initLog setup logger diff --git a/src/runtime/containerd-shim-v2/create.go b/src/runtime/containerd-shim-v2/create.go index 83c428050..9c18f3899 100644 --- a/src/runtime/containerd-shim-v2/create.go +++ b/src/runtime/containerd-shim-v2/create.go @@ -88,7 +88,7 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest) (*con return nil, err } s.sandbox = sandbox - go s.startManagementServer(ctx) + go s.startManagementServer(ctx, ociSpec) case vc.PodContainer: if s.sandbox == nil { diff --git a/src/runtime/containerd-shim-v2/shim_management.go b/src/runtime/containerd-shim-v2/shim_management.go index 69ba88e3f..b9309be1b 100644 --- a/src/runtime/containerd-shim-v2/shim_management.go +++ b/src/runtime/containerd-shim-v2/shim_management.go @@ -7,14 +7,19 @@ package containerdshim import ( "context" + "expvar" "io" "net/http" + "net/http/pprof" "path/filepath" + "strconv" "strings" "github.com/containerd/containerd/namespaces" cdshim "github.com/containerd/containerd/runtime/v2/shim" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" + vcAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations" + "github.com/opencontainers/runtime-spec/specs-go" "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" @@ -30,6 +35,7 @@ var ( ifSupportAgentMetricsAPI = true ) +// serveMetrics handle /metrics requests func (s *service) serveMetrics(w http.ResponseWriter, r *http.Request) { // update metrics from sandbox @@ -109,7 +115,7 @@ func decodeAgentMetrics(body string) []*dto.MetricFamily { return list } -func (s *service) startManagementServer(ctx context.Context) { +func (s *service) startManagementServer(ctx context.Context, ociSpec *specs.Spec) { // metrics socket will under sandbox's bundle path metricsAddress, err := socketAddress(ctx, s.id) if err != nil { @@ -132,7 +138,9 @@ func (s *service) startManagementServer(ctx context.Context) { logrus.Info("kata monitor inited") // bind hanlder - http.HandleFunc("/metrics", s.serveMetrics) + m := http.NewServeMux() + m.Handle("/metrics", http.HandlerFunc(s.serveMetrics)) + s.mountPprofHandle(m, ociSpec) // register shim metrics registerMetrics() @@ -141,10 +149,32 @@ func (s *service) startManagementServer(ctx context.Context) { vc.RegisterMetrics() // start serve - svr := &http.Server{Handler: http.DefaultServeMux} + svr := &http.Server{Handler: m} svr.Serve(listener) } +// mountServeDebug provides a debug endpoint +func (s *service) mountPprofHandle(m *http.ServeMux, ociSpec *specs.Spec) { + + // return if not enabled + if !s.config.EnablePprof { + value, ok := ociSpec.Annotations[vcAnnotations.EnablePprof] + if !ok { + return + } + enabled, err := strconv.ParseBool(value) + if err != nil || !enabled { + return + } + } + m.Handle("/debug/vars", expvar.Handler()) + m.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index)) + m.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline)) + m.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile)) + m.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol)) + m.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace)) +} + func socketAddress(ctx context.Context, id string) (string, error) { ns, err := namespaces.NamespaceRequired(ctx) if err != nil { diff --git a/src/runtime/pkg/kata-monitor/metrics.go b/src/runtime/pkg/kata-monitor/metrics.go index e1eb2a6bd..712924ffe 100644 --- a/src/runtime/pkg/kata-monitor/metrics.go +++ b/src/runtime/pkg/kata-monitor/metrics.go @@ -73,11 +73,10 @@ func registerMetrics() { prometheus.MustRegister(scrapeDurationsHistogram) } -// getMetricsAddress get metrics address for a sandbox, the abstract unix socket address is saved +// getMonitorAddress get metrics address for a sandbox, the abstract unix socket address is saved // in `metrics_address` with the same place of `address`. -func (km *KataMonitor) getMetricsAddress(sandboxID, namespace string) (string, error) { +func (km *KataMonitor) getMonitorAddress(sandboxID, namespace string) (string, error) { path := filepath.Join(km.containerdStatePath, types.ContainerdRuntimeTaskPath, namespace, sandboxID, "monitor_address") - data, err := ioutil.ReadFile(path) if err != nil { return "", err @@ -237,7 +236,7 @@ func (km *KataMonitor) aggregateSandboxMetrics(encoder expfmt.Encoder) error { // getSandboxMetrics will get sandbox's metrics from shim func (km *KataMonitor) getSandboxMetrics(sandboxID, namespace string) ([]*dto.MetricFamily, error) { - socket, err := km.getMetricsAddress(sandboxID, namespace) + socket, err := km.getMonitorAddress(sandboxID, namespace) if err != nil { return nil, err } diff --git a/src/runtime/pkg/kata-monitor/monitor.go b/src/runtime/pkg/kata-monitor/monitor.go index fcbf6a5ce..7f4cb4eea 100644 --- a/src/runtime/pkg/kata-monitor/monitor.go +++ b/src/runtime/pkg/kata-monitor/monitor.go @@ -7,6 +7,7 @@ package katamonitor import ( "fmt" + "net/http" "os" "sync" @@ -48,7 +49,7 @@ func NewKataMonitor(containerdAddr, containerdConfigFile string) (*KataMonitor, return nil, err } - ka := &KataMonitor{ + km := &KataMonitor{ containerdAddr: containerdAddr, containerdConfigFile: containerdConfigFile, containerdStatePath: containerdConf.State, @@ -58,23 +59,47 @@ func NewKataMonitor(containerdAddr, containerdConfigFile string) (*KataMonitor, }, } - if err := ka.initSandboxCache(); err != nil { + if err := km.initSandboxCache(); err != nil { return nil, err } // register metrics registerMetrics() - go ka.sandboxCache.startEventsListener(ka.containerdAddr) + go km.sandboxCache.startEventsListener(km.containerdAddr) - return ka, nil + return km, nil } -func (ka *KataMonitor) initSandboxCache() error { - sandboxes, err := ka.getSandboxes() +func (km *KataMonitor) initSandboxCache() error { + sandboxes, err := km.getSandboxes() if err != nil { return err } - ka.sandboxCache.init(sandboxes) + km.sandboxCache.init(sandboxes) return nil } + +// ListSandboxes list all sandboxes running in Kata +func (km *KataMonitor) ListSandboxes(w http.ResponseWriter, r *http.Request) { + sandboxes := km.getSandboxList() + for _, s := range sandboxes { + w.Write([]byte(fmt.Sprintf("%s\n", s))) + } +} + +func (km *KataMonitor) getSandboxList() []string { + sn := km.sandboxCache.getAllSandboxes() + result := make([]string, len(sn)) + + i := 0 + for k := range sn { + result[i] = k + i++ + } + return result +} + +func (km *KataMonitor) getSandboxNamespace(sandbox string) (string, error) { + return km.sandboxCache.getSandboxNamespace(sandbox) +} diff --git a/src/runtime/pkg/kata-monitor/pprof.go b/src/runtime/pkg/kata-monitor/pprof.go new file mode 100644 index 000000000..e043251cb --- /dev/null +++ b/src/runtime/pkg/kata-monitor/pprof.go @@ -0,0 +1,119 @@ +// Copyright (c) 2020 Ant Financial +// +// SPDX-License-Identifier: Apache-2.0 +// + +package katamonitor + +import ( + "fmt" + "io" + "net" + "net/http" +) + +func getSandboxIdFromReq(r *http.Request) (string, error) { + sandbox := r.URL.Query().Get("sandbox") + if sandbox != "" { + return sandbox, nil + } + return "", fmt.Errorf("sandbox not found in %+v", r.URL.Query()) +} + +func serveError(w http.ResponseWriter, status int, txt string) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.Header().Set("X-Go-Pprof", "1") + w.Header().Del("Content-Disposition") + w.WriteHeader(status) + fmt.Fprintln(w, txt) +} + +func (km *KataMonitor) composeSocketAddress(r *http.Request) (string, error) { + sandbox, err := getSandboxIdFromReq(r) + if err != nil { + return "", err + } + + namespace, err := km.getSandboxNamespace(sandbox) + if err != nil { + return "", err + } + + return km.getMonitorAddress(sandbox, namespace) +} + +func (km *KataMonitor) proxyRequest(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Content-Type-Options", "nosniff") + + socket, err := km.composeSocketAddress(r) + if err != nil { + monitorLog.WithError(err).Error("failed to get shim monitor address") + serveError(w, http.StatusBadRequest, "sandbox may be stopped or deleted") + return + } + + transport := &http.Transport{ + DisableKeepAlives: true, + Dial: func(proto, addr string) (conn net.Conn, err error) { + return net.Dial("unix", "\x00"+socket) + }, + } + + client := http.Client{ + Transport: transport, + } + + uri := fmt.Sprintf("http://shim%s", r.URL.String()) + resp, err := client.Get(uri) + if err != nil { + return + } + + output := resp.Body + defer output.Close() + + contentType := resp.Header.Get("Content-Type") + if contentType != "" { + w.Header().Set("Content-Type", contentType) + } + + contentDisposition := resp.Header.Get("Content-Disposition") + if contentDisposition != "" { + w.Header().Set("Content-Disposition", contentDisposition) + } + + io.Copy(w, output) +} + +// ExpvarHandler handles other `/debug/vars` requests +func (km *KataMonitor) ExpvarHandler(w http.ResponseWriter, r *http.Request) { + km.proxyRequest(w, r) +} + +// PprofIndex handles other `/debug/pprof/` requests +func (km *KataMonitor) PprofIndex(w http.ResponseWriter, r *http.Request) { + km.proxyRequest(w, r) +} + +// PprofCmdline handles other `/debug/cmdline` requests +func (km *KataMonitor) PprofCmdline(w http.ResponseWriter, r *http.Request) { + km.proxyRequest(w, r) +} + +// PprofProfile handles other `/debug/profile` requests +func (km *KataMonitor) PprofProfile(w http.ResponseWriter, r *http.Request) { + km.proxyRequest(w, r) +} + +// PprofSymbol handles other `/debug/symbol` requests +func (km *KataMonitor) PprofSymbol(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + km.proxyRequest(w, r) +} + +// PprofTrace handles other `/debug/trace` requests +func (km *KataMonitor) PprofTrace(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Disposition", `attachment; filename="trace"`) + km.proxyRequest(w, r) +} diff --git a/src/runtime/pkg/kata-monitor/pprof_test.go b/src/runtime/pkg/kata-monitor/pprof_test.go new file mode 100644 index 000000000..3f5e3e0c0 --- /dev/null +++ b/src/runtime/pkg/kata-monitor/pprof_test.go @@ -0,0 +1,83 @@ +// Copyright (c) 2020 Ant Financial +// +// SPDX-License-Identifier: Apache-2.0 +// + +package katamonitor + +import ( + "fmt" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestComposeSocketAddress(t *testing.T) { + assert := assert.New(t) + path := fmt.Sprintf("/tmp/TestComposeSocketAddress-%d", time.Now().Nanosecond()) + statePath := filepath.Join(path, "io.containerd.runtime.v2.task") + + sandboxes := map[string]string{"foo": "ns-foo", "bar": "ns-bar"} + defer func() { + os.RemoveAll(path) + }() + + for sandbox, ns := range sandboxes { + err := os.MkdirAll(filepath.Join(statePath, ns, sandbox), 0755) + assert.Nil(err) + f := filepath.Join(statePath, ns, sandbox, "monitor_address") + err = ioutil.WriteFile(f, []byte(sandbox), 0644) + assert.Nil(err) + } + + km := &KataMonitor{ + containerdStatePath: path, + sandboxCache: &sandboxCache{ + Mutex: &sync.Mutex{}, + sandboxes: sandboxes, + }, + } + + testCases := []struct { + url string + err bool + addr string + }{ + { + url: "http://localhost:6060/debug/vars", + err: true, + addr: "", + }, + { + url: "http://localhost:6060/debug/vars?sandbox=abc", + err: true, + addr: "", + }, + { + url: "http://localhost:6060/debug/vars?sandbox=foo", + err: false, + addr: "foo", + }, + { + url: "http://localhost:6060/debug/vars?sandbox=bar", + err: false, + addr: "bar", + }, + } + + for _, tc := range testCases { + r, err := http.NewRequest("GET", tc.url, nil) + assert.Nil(err) + + addr, err := km.composeSocketAddress(r) + + assert.Equal(tc.err, err != nil) + assert.Equal(tc.addr, addr) + } +} diff --git a/src/runtime/pkg/kata-monitor/sandbox_cache.go b/src/runtime/pkg/kata-monitor/sandbox_cache.go index eae60e098..974bedf3f 100644 --- a/src/runtime/pkg/kata-monitor/sandbox_cache.go +++ b/src/runtime/pkg/kata-monitor/sandbox_cache.go @@ -7,6 +7,7 @@ package katamonitor import ( "context" + "fmt" "sync" "github.com/containerd/containerd" @@ -34,6 +35,17 @@ func (sc *sandboxCache) getAllSandboxes() map[string]string { return sc.sandboxes } +func (sc *sandboxCache) getSandboxNamespace(sandbox string) (string, error) { + sc.Lock() + defer sc.Unlock() + + if val, found := sc.sandboxes[sandbox]; found { + return val, nil + } + + return "", fmt.Errorf("sandbox %s not in cache", sandbox) +} + func (sc *sandboxCache) deleteIfExists(id string) (string, bool) { sc.Lock() defer sc.Unlock() diff --git a/src/runtime/pkg/katatestutils/utils.go b/src/runtime/pkg/katatestutils/utils.go index 999ab64b2..951aaed5d 100644 --- a/src/runtime/pkg/katatestutils/utils.go +++ b/src/runtime/pkg/katatestutils/utils.go @@ -42,6 +42,7 @@ type RuntimeConfigOptions struct { NetmonDebug bool AgentDebug bool AgentTrace bool + EnablePprof bool } func MakeRuntimeConfigFileData(config RuntimeConfigOptions) string { @@ -89,5 +90,6 @@ func MakeRuntimeConfigFileData(config RuntimeConfigOptions) string { [runtime] enable_debug = ` + strconv.FormatBool(config.RuntimeDebug) + ` enable_tracing = ` + strconv.FormatBool(config.RuntimeTrace) + ` - disable_new_netns= ` + strconv.FormatBool(config.DisableNewNetNs) + disable_new_netns= ` + strconv.FormatBool(config.DisableNewNetNs) + ` + enable_pprof= ` + strconv.FormatBool(config.EnablePprof) } diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index e7aed1385..579ce5b53 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -139,6 +139,7 @@ type runtime struct { SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"` Experimental []string `toml:"experimental"` InterNetworkModel string `toml:"internetworking_model"` + EnablePprof bool `toml:"enable_pprof"` } type agent struct { @@ -1165,6 +1166,7 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolved config.SandboxCgroupOnly = tomlConf.Runtime.SandboxCgroupOnly config.DisableNewNetNs = tomlConf.Runtime.DisableNewNetNs + config.EnablePprof = tomlConf.Runtime.EnablePprof for _, f := range tomlConf.Runtime.Experimental { feature := exp.Get(f) if feature == nil { diff --git a/src/runtime/pkg/katautils/config_test.go b/src/runtime/pkg/katautils/config_test.go index f75591ae5..d5f082fb2 100644 --- a/src/runtime/pkg/katautils/config_test.go +++ b/src/runtime/pkg/katautils/config_test.go @@ -34,6 +34,7 @@ var ( netmonDebug = false agentDebug = false agentTrace = false + enablePprof = true ) type testRuntimeConfig struct { @@ -115,6 +116,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf AgentTrace: agentTrace, SharedFS: sharedFS, VirtioFSDaemon: virtioFSdaemon, + EnablePprof: enablePprof, } runtimeConfigFileData := ktu.MakeRuntimeConfigFileData(configFileOptions) @@ -197,6 +199,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf NetmonConfig: netmonConfig, DisableNewNetNs: disableNewNetNs, + EnablePprof: enablePprof, FactoryConfig: factoryConfig, } diff --git a/src/runtime/virtcontainers/pkg/annotations/annotations.go b/src/runtime/virtcontainers/pkg/annotations/annotations.go index f32f90694..464429244 100644 --- a/src/runtime/virtcontainers/pkg/annotations/annotations.go +++ b/src/runtime/virtcontainers/pkg/annotations/annotations.go @@ -221,6 +221,9 @@ const ( // SandboxCgroupOnly is a sandbox annotation that determines if kata processes are managed only in sandbox cgroup. SandboxCgroupOnly = kataAnnotRuntimePrefix + "sandbox_cgroup_only" + // EnablePprof is a sandbox annotation that determines if pprof enabled. + EnablePprof = kataAnnotRuntimePrefix + "enable_pprof" + // Experimental is a sandbox annotation that determines if experimental features enabled. Experimental = kataAnnotRuntimePrefix + "experimental" diff --git a/src/runtime/virtcontainers/pkg/oci/utils.go b/src/runtime/virtcontainers/pkg/oci/utils.go index ee2546bb7..645ea54be 100644 --- a/src/runtime/virtcontainers/pkg/oci/utils.go +++ b/src/runtime/virtcontainers/pkg/oci/utils.go @@ -121,6 +121,9 @@ type RuntimeConfig struct { //Experimental features enabled Experimental []exp.Feature + + // Determines if enable pprof + EnablePprof bool } // AddKernelParam allows the addition of new kernel parameters to an existing