From 37873061072125713b3f9330c52e3103c68d195f Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Thu, 6 May 2021 11:46:12 -0700 Subject: [PATCH 1/2] kata-monitor: export get stats for sandbox Gathering stats for a given sandbox is pretty useful; let's export a function from katamonitor pkg to do this. Signed-off-by: Eric Ernst --- src/runtime/pkg/kata-monitor/metrics.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/runtime/pkg/kata-monitor/metrics.go b/src/runtime/pkg/kata-monitor/metrics.go index aeb9f72c5d..8e0d4c7e4c 100644 --- a/src/runtime/pkg/kata-monitor/metrics.go +++ b/src/runtime/pkg/kata-monitor/metrics.go @@ -176,7 +176,7 @@ func (km *KataMonitor) aggregateSandboxMetrics(encoder expfmt.Encoder) error { for sandboxID, namespace := range sandboxes { wg.Add(1) go func(sandboxID, namespace string, results chan<- []*dto.MetricFamily) { - sandboxMetrics, err := getSandboxMetrics(sandboxID) + sandboxMetrics, err := getParsedMetrics(sandboxID) if err != nil { monitorLog.WithError(err).WithField("sandbox_id", sandboxID).Errorf("failed to get metrics for sandbox") } @@ -229,12 +229,11 @@ func (km *KataMonitor) aggregateSandboxMetrics(encoder expfmt.Encoder) error { return err } } - return nil + } -// getSandboxMetrics will get sandbox's metrics from shim -func getSandboxMetrics(sandboxID string) ([]*dto.MetricFamily, error) { +func getParsedMetrics(sandboxID string) ([]*dto.MetricFamily, error) { body, err := doGet(sandboxID, defaultTimeout, "metrics") if err != nil { return nil, err @@ -243,6 +242,16 @@ func getSandboxMetrics(sandboxID string) ([]*dto.MetricFamily, error) { return parsePrometheusMetrics(sandboxID, body) } +// GetSandboxMetrics will get sandbox's metrics from shim +func GetSandboxMetrics(sandboxID string) (string, error) { + body, err := doGet(sandboxID, defaultTimeout, "metrics") + if err != nil { + return "", err + } + + return string(body), nil +} + // parsePrometheusMetrics will decode metrics from Prometheus text format // and return array of *dto.MetricFamily with an ASC order func parsePrometheusMetrics(sandboxID string, body []byte) ([]*dto.MetricFamily, error) { From 8068a4692fa7f1332411c43e02fa9163b3f2c5d7 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Thu, 6 May 2021 11:46:12 -0700 Subject: [PATCH 2/2] kata-runtime: add `metrics` command For easier debug, let's add subcommand to kata-runtime for gathering metrics associated with a given sandbox. kata-runtime metrics --sandbox-id foobar Fixes: #1815 Signed-off-by: Eric Ernst --- src/runtime/cli/kata-metrics.go | 38 +++++++++++++++++++++++++++++++++ src/runtime/cli/main.go | 1 + 2 files changed, 39 insertions(+) create mode 100644 src/runtime/cli/kata-metrics.go diff --git a/src/runtime/cli/kata-metrics.go b/src/runtime/cli/kata-metrics.go new file mode 100644 index 0000000000..7d0c7307cc --- /dev/null +++ b/src/runtime/cli/kata-metrics.go @@ -0,0 +1,38 @@ +// Copyright (c) 2021 Apple Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "fmt" + + kataMonitor "github.com/kata-containers/kata-containers/src/runtime/pkg/kata-monitor" + "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils" + "github.com/urfave/cli" +) + +var kataMetricsCLICommand = cli.Command{ + Name: "metrics", + Usage: "gather metrics associated with infrastructure used to run a sandbox", + UsageText: "metrics ", + Action: func(context *cli.Context) error { + + sandboxID := context.Args().Get(0) + + if err := katautils.VerifyContainerID(sandboxID); err != nil { + return err + } + + // Get the metrics! + metrics, err := kataMonitor.GetSandboxMetrics(sandboxID) + if err != nil { + return err + } + + fmt.Printf("%s\n", metrics) + + return nil + }, +} diff --git a/src/runtime/cli/main.go b/src/runtime/cli/main.go index 78ddb0739f..27757c0c4f 100644 --- a/src/runtime/cli/main.go +++ b/src/runtime/cli/main.go @@ -122,6 +122,7 @@ var runtimeCommands = []cli.Command{ kataCheckCLICommand, kataEnvCLICommand, kataExecCLICommand, + kataMetricsCLICommand, factoryCLICommand, }