Merge pull request #113609 from haircommander/sandbox-metrics

kubelet: add support for broadcasting metrics from CRI

pkg/kubelet/container/runtime.go

@@ -124,6 +124,14 @@ type Runtime interface {
 	CheckpointContainer(ctx context.Context, options *runtimeapi.CheckpointContainerRequest) error
 	// Generate pod status from the CRI event
 	GeneratePodStatus(event *runtimeapi.ContainerEventResponse) (*PodStatus, error)
+	// ListMetricDescriptors gets the descriptors for the metrics that will be returned in ListPodSandboxMetrics.
+	// This list should be static at startup: either the client and server restart together when
+	// adding or removing metrics descriptors, or they should not change.
+	// Put differently, if ListPodSandboxMetrics references a name that is not described in the initial
+	// ListMetricDescriptors call, then the metric will not be broadcasted.
+	ListMetricDescriptors(ctx context.Context) ([]*runtimeapi.MetricDescriptor, error)
+	// ListPodSandboxMetrics retrieves the metrics for all pod sandboxes.
+	ListPodSandboxMetrics(ctx context.Context) ([]*runtimeapi.PodSandboxMetrics, error)
 }
 
 // StreamingRuntime is the interface implemented by runtimes that handle the serving of the

pkg/kubelet/container/testing/fake_runtime.go

@@ -379,6 +379,22 @@ func (f *FakeRuntime) CheckpointContainer(_ context.Context, options *runtimeapi
 	return f.Err
 }
 
+func (f *FakeRuntime) ListMetricDescriptors(_ context.Context) ([]*runtimeapi.MetricDescriptor, error) {
+	f.Lock()
+	defer f.Unlock()
+
+	f.CalledFunctions = append(f.CalledFunctions, "ListMetricDescriptors")
+	return nil, f.Err
+}
+
+func (f *FakeRuntime) ListPodSandboxMetrics(_ context.Context) ([]*runtimeapi.PodSandboxMetrics, error) {
+	f.Lock()
+	defer f.Unlock()
+
+	f.CalledFunctions = append(f.CalledFunctions, "ListPodSandboxMetrics")
+	return nil, f.Err
+}
+
 func (f *FakeRuntime) ImageStats(_ context.Context) (*kubecontainer.ImageStats, error) {
 	f.Lock()
 	defer f.Unlock()

pkg/kubelet/container/testing/runtime_mock.go

@@ -286,6 +286,36 @@ func (mr *MockRuntimeMockRecorder) ListImages(ctx interface{}) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListImages", reflect.TypeOf((*MockRuntime)(nil).ListImages), ctx)
 }
 
+// ListMetricDescriptors mocks base method.
+func (m *MockRuntime) ListMetricDescriptors(ctx context.Context) ([]*v10.MetricDescriptor, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ListMetricDescriptors", ctx)
+	ret0, _ := ret[0].([]*v10.MetricDescriptor)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// ListMetricDescriptors indicates an expected call of ListMetricDescriptors.
+func (mr *MockRuntimeMockRecorder) ListMetricDescriptors(ctx interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListMetricDescriptors", reflect.TypeOf((*MockRuntime)(nil).ListMetricDescriptors), ctx)
+}
+
+// ListPodSandboxMetrics mocks base method.
+func (m *MockRuntime) ListPodSandboxMetrics(ctx context.Context) ([]*v10.PodSandboxMetrics, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ListPodSandboxMetrics", ctx)
+	ret0, _ := ret[0].([]*v10.PodSandboxMetrics)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// ListPodSandboxMetrics indicates an expected call of ListPodSandboxMetrics.
+func (mr *MockRuntimeMockRecorder) ListPodSandboxMetrics(ctx interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListPodSandboxMetrics", reflect.TypeOf((*MockRuntime)(nil).ListPodSandboxMetrics), ctx)
+}
+
 // PullImage mocks base method.
 func (m *MockRuntime) PullImage(ctx context.Context, image container.ImageSpec, pullSecrets []v1.Secret, podSandboxConfig *v10.PodSandboxConfig) (string, error) {
 	m.ctrl.T.Helper()

pkg/kubelet/cri/remote/fake/fake_runtime.go

@@ -336,3 +336,23 @@ func (f *RemoteRuntime) CheckpointContainer(ctx context.Context, req *kubeapi.Ch
 func (f *RemoteRuntime) GetContainerEvents(req *kubeapi.GetEventsRequest, ces kubeapi.RuntimeService_GetContainerEventsServer) error {
 	return nil
 }
+
+// ListMetricDescriptors gets the descriptors for the metrics that will be returned in ListPodSandboxMetrics.
+func (f *RemoteRuntime) ListMetricDescriptors(ctx context.Context, req *kubeapi.ListMetricDescriptorsRequest) (*kubeapi.ListMetricDescriptorsResponse, error) {
+	descs, err := f.RuntimeService.ListMetricDescriptors(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return &kubeapi.ListMetricDescriptorsResponse{Descriptors: descs}, nil
+}
+
+// ListPodSandboxMetrics retrieves the metrics for all pod sandboxes.
+func (f *RemoteRuntime) ListPodSandboxMetrics(ctx context.Context, req *kubeapi.ListPodSandboxMetricsRequest) (*kubeapi.ListPodSandboxMetricsResponse, error) {
+	podMetrics, err := f.RuntimeService.ListPodSandboxMetrics(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return &kubeapi.ListPodSandboxMetricsResponse{PodMetrics: podMetrics}, nil
+}

pkg/kubelet/cri/remote/remote_runtime.go

@@ -815,3 +815,33 @@ func (r *remoteRuntimeService) GetContainerEvents(containerEventsCh chan *runtim
 		}
 	}
 }
+
+// ListMetricDescriptors gets the descriptors for the metrics that will be returned in ListPodSandboxMetrics.
+func (r *remoteRuntimeService) ListMetricDescriptors(ctx context.Context) ([]*runtimeapi.MetricDescriptor, error) {
+	ctx, cancel := context.WithTimeout(ctx, r.timeout)
+	defer cancel()
+
+	resp, err := r.runtimeClient.ListMetricDescriptors(ctx, &runtimeapi.ListMetricDescriptorsRequest{})
+	if err != nil {
+		klog.ErrorS(err, "ListMetricDescriptors from runtime service failed")
+		return nil, err
+	}
+	klog.V(10).InfoS("[RemoteRuntimeService] ListMetricDescriptors Response", "stats", resp.GetDescriptors())
+
+	return resp.GetDescriptors(), nil
+}
+
+// ListPodSandboxMetrics retrieves the metrics for all pod sandboxes.
+func (r *remoteRuntimeService) ListPodSandboxMetrics(ctx context.Context) ([]*runtimeapi.PodSandboxMetrics, error) {
+	ctx, cancel := context.WithTimeout(ctx, r.timeout)
+	defer cancel()
+
+	resp, err := r.runtimeClient.ListPodSandboxMetrics(ctx, &runtimeapi.ListPodSandboxMetricsRequest{})
+	if err != nil {
+		klog.ErrorS(err, "ListPodSandboxMetrics from runtime service failed")
+		return nil, err
+	}
+	klog.V(10).InfoS("[RemoteRuntimeService] ListPodSandboxMetrics Response", "stats", resp.GetPodMetrics())
+
+	return resp.GetPodMetrics(), nil
+}

pkg/kubelet/kubelet.go

@@ -2562,6 +2562,16 @@ func (kl *Kubelet) CheckpointContainer(
 	return nil
 }
 
+// ListMetricDescriptors gets the descriptors for the metrics that will be returned in ListPodSandboxMetrics.
+func (kl *Kubelet) ListMetricDescriptors(ctx context.Context) ([]*runtimeapi.MetricDescriptor, error) {
+	return kl.containerRuntime.ListMetricDescriptors(ctx)
+}
+
+// ListPodSandboxMetrics retrieves the metrics for all pod sandboxes.
+func (kl *Kubelet) ListPodSandboxMetrics(ctx context.Context) ([]*runtimeapi.PodSandboxMetrics, error) {
+	return kl.containerRuntime.ListPodSandboxMetrics(ctx)
+}
+
 func (kl *Kubelet) supportLocalStorageCapacityIsolation() bool {
 	return kl.GetConfiguration().LocalStorageCapacityIsolation
 }

pkg/kubelet/kuberuntime/instrumented_services.go

@@ -343,3 +343,21 @@ func (in instrumentedRuntimeService) GetContainerEvents(containerEventsCh chan *
 	recordError(operation, err)
 	return err
 }
+
+func (in instrumentedRuntimeService) ListMetricDescriptors(ctx context.Context) ([]*runtimeapi.MetricDescriptor, error) {
+	const operation = "list_metric_descriptors"
+	defer recordOperation(operation, time.Now())
+
+	out, err := in.service.ListMetricDescriptors(ctx)
+	recordError(operation, err)
+	return out, err
+}
+
+func (in instrumentedRuntimeService) ListPodSandboxMetrics(ctx context.Context) ([]*runtimeapi.PodSandboxMetrics, error) {
+	const operation = "list_podsandbox_metrics"
+	defer recordOperation(operation, time.Now())
+
+	out, err := in.service.ListPodSandboxMetrics(ctx)
+	recordError(operation, err)
+	return out, err
+}

pkg/kubelet/kuberuntime/kuberuntime_manager.go

@@ -1123,3 +1123,11 @@ func (m *kubeGenericRuntimeManager) UpdatePodCIDR(ctx context.Context, podCIDR s
 func (m *kubeGenericRuntimeManager) CheckpointContainer(ctx context.Context, options *runtimeapi.CheckpointContainerRequest) error {
 	return m.runtimeService.CheckpointContainer(ctx, options)
 }
+
+func (m *kubeGenericRuntimeManager) ListMetricDescriptors(ctx context.Context) ([]*runtimeapi.MetricDescriptor, error) {
+	return m.runtimeService.ListMetricDescriptors(ctx)
+}
+
+func (m *kubeGenericRuntimeManager) ListPodSandboxMetrics(ctx context.Context) ([]*runtimeapi.PodSandboxMetrics, error) {
+	return m.runtimeService.ListPodSandboxMetrics(ctx)
+}

pkg/kubelet/metrics/collectors/cri_metrics.go

@@ -0,0 +1,129 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package collectors
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"k8s.io/component-base/metrics"
+	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
+	"k8s.io/klog/v2"
+)
+
+type criMetricsCollector struct {
+	metrics.BaseStableCollector
+	// The descriptors structure will be populated by one call to ListMetricDescriptors from the runtime.
+	// They will be saved in this map, where the key is the Name and the value is the Desc.
+	descriptors             map[string]*metrics.Desc
+	listPodSandboxMetricsFn func(context.Context) ([]*runtimeapi.PodSandboxMetrics, error)
+}
+
+// Check if criMetricsCollector implements necessary interface
+var _ metrics.StableCollector = &criMetricsCollector{}
+
+// NewCRIMetricsCollector implements the metrics.Collector interface
+func NewCRIMetricsCollector(ctx context.Context, listPodSandboxMetricsFn func(context.Context) ([]*runtimeapi.PodSandboxMetrics, error), listMetricDescriptorsFn func(context.Context) ([]*runtimeapi.MetricDescriptor, error)) metrics.StableCollector {
+	descs, err := listMetricDescriptorsFn(ctx)
+	if err != nil {
+		klog.ErrorS(err, "Error reading MetricDescriptors")
+		return &criMetricsCollector{
+			listPodSandboxMetricsFn: listPodSandboxMetricsFn,
+		}
+	}
+	c := &criMetricsCollector{
+		listPodSandboxMetricsFn: listPodSandboxMetricsFn,
+		descriptors:             make(map[string]*metrics.Desc, len(descs)),
+	}
+
+	for _, desc := range descs {
+		c.descriptors[desc.Name] = criDescToProm(desc)
+	}
+
+	return c
+}
+
+// Describe implements the metrics.DescribeWithStability interface.
+func (c *criMetricsCollector) DescribeWithStability(ch chan<- *metrics.Desc) {
+	for _, desc := range c.descriptors {
+		ch <- desc
+	}
+}
+
+// Collect implements the metrics.CollectWithStability interface.
+// TODO(haircommander): would it be better if these were processed async?
+func (c *criMetricsCollector) CollectWithStability(ch chan<- metrics.Metric) {
+	podMetrics, err := c.listPodSandboxMetricsFn(context.Background())
+	if err != nil {
+		klog.ErrorS(err, "Failed to get pod metrics")
+		return
+	}
+
+	for _, podMetric := range podMetrics {
+		for _, metric := range podMetric.GetMetrics() {
+			promMetric, err := c.criMetricToProm(metric)
+			if err == nil {
+				ch <- promMetric
+			}
+		}
+		for _, ctrMetric := range podMetric.GetContainerMetrics() {
+			for _, metric := range ctrMetric.GetMetrics() {
+				promMetric, err := c.criMetricToProm(metric)
+				if err == nil {
+					ch <- promMetric
+				}
+			}
+		}
+	}
+}
+
+func criDescToProm(m *runtimeapi.MetricDescriptor) *metrics.Desc {
+	// Labels in the translation are variableLabels, as opposed to constant labels.
+	// This is because the values of the labels will be different for each container.
+	return metrics.NewDesc(m.Name, m.Help, m.LabelKeys, nil, metrics.INTERNAL, "")
+}
+
+func (c *criMetricsCollector) criMetricToProm(m *runtimeapi.Metric) (metrics.Metric, error) {
+	desc, ok := c.descriptors[m.Name]
+	if !ok {
+		err := fmt.Errorf("error converting CRI Metric to prometheus format")
+		klog.V(5).ErrorS(err, "Descriptor not present in pre-populated list of descriptors", "descriptor name", m.Name)
+		return nil, err
+	}
+
+	typ := criTypeToProm[m.MetricType]
+
+	pm, err := metrics.NewConstMetric(desc, typ, float64(m.GetValue().Value), m.LabelValues...)
+	if err != nil {
+		klog.ErrorS(err, "Error getting CRI prometheus metric", "descriptor", desc.String())
+		return nil, err
+	}
+	// If Timestamp is 0, then the runtime did not cache the result.
+	// In this case, a cached result is a metric that was collected ahead of time,
+	// as opposed to on-demand.
+	// If the metric was requested as needed, then Timestamp==0.
+	if m.Timestamp == 0 {
+		return pm, nil
+	}
+	return metrics.NewLazyMetricWithTimestamp(time.Unix(0, m.Timestamp), pm), nil
+}
+
+var criTypeToProm = map[runtimeapi.MetricType]metrics.ValueType{
+	runtimeapi.MetricType_COUNTER: metrics.CounterValue,
+	runtimeapi.MetricType_GAUGE:   metrics.GaugeValue,
+}

pkg/kubelet/server/server.go

@@ -250,6 +250,8 @@ type HostInterface interface {
 	GetExec(ctx context.Context, podFullName string, podUID types.UID, containerName string, cmd []string, streamOpts remotecommandserver.Options) (*url.URL, error)
 	GetAttach(ctx context.Context, podFullName string, podUID types.UID, containerName string, streamOpts remotecommandserver.Options) (*url.URL, error)
 	GetPortForward(ctx context.Context, podName, podNamespace string, podUID types.UID, portForwardOpts portforward.V4Options) (*url.URL, error)
+	ListMetricDescriptors(ctx context.Context) ([]*runtimeapi.MetricDescriptor, error)
+	ListPodSandboxMetrics(ctx context.Context) ([]*runtimeapi.PodSandboxMetrics, error)
 }
 
 // NewServer initializes and configures a kubelet.Server object to handle HTTP requests.
@@ -386,9 +388,6 @@ func (s *Server) InstallDefaultHandlers() {
 	s.addMetricsBucketMatcher("metrics/resource")
 	s.restfulCont.Handle(metricsPath, legacyregistry.Handler())
 
-	// cAdvisor metrics are exposed under the secured handler as well
-	r := compbasemetrics.NewKubeRegistry()
-
 	includedMetrics := cadvisormetrics.MetricSet{
 		cadvisormetrics.CpuUsageMetrics:     struct{}{},
 		cadvisormetrics.MemoryUsageMetrics:  struct{}{},
@@ -400,14 +399,19 @@ func (s *Server) InstallDefaultHandlers() {
 		cadvisormetrics.ProcessMetrics:      struct{}{},
 		cadvisormetrics.OOMMetrics:          struct{}{},
 	}
-
+	// cAdvisor metrics are exposed under the secured handler as well
-	cadvisorOpts := cadvisorv2.RequestOptions{
+	r := compbasemetrics.NewKubeRegistry()
-		IdType:    cadvisorv2.TypeName,
-		Count:     1,
-		Recursive: true,
-	}
-	r.RawMustRegister(metrics.NewPrometheusCollector(prometheusHostAdapter{s.host}, containerPrometheusLabelsFunc(s.host), includedMetrics, clock.RealClock{}, cadvisorOpts))
 	r.RawMustRegister(metrics.NewPrometheusMachineCollector(prometheusHostAdapter{s.host}, includedMetrics))
+	if utilfeature.DefaultFeatureGate.Enabled(features.PodAndContainerStatsFromCRI) {
+		r.CustomRegister(collectors.NewCRIMetricsCollector(context.TODO(), s.host.ListPodSandboxMetrics, s.host.ListMetricDescriptors))
+	} else {
+		cadvisorOpts := cadvisorv2.RequestOptions{
+			IdType:    cadvisorv2.TypeName,
+			Count:     1,
+			Recursive: true,
+		}
+		r.RawMustRegister(metrics.NewPrometheusCollector(prometheusHostAdapter{s.host}, containerPrometheusLabelsFunc(s.host), includedMetrics, clock.RealClock{}, cadvisorOpts))
+	}
 	s.restfulCont.Handle(cadvisorMetricsPath,
 		compbasemetrics.HandlerFor(r, compbasemetrics.HandlerOpts{ErrorHandling: compbasemetrics.ContinueOnError}),
 	)

pkg/kubelet/server/server_test.go

@@ -156,6 +156,14 @@ func (fk *fakeKubelet) CheckpointContainer(_ context.Context, podUID types.UID,
 	return nil
 }
 
+func (fk *fakeKubelet) ListMetricDescriptors(ctx context.Context) ([]*runtimeapi.MetricDescriptor, error) {
+	return nil, nil
+}
+
+func (fk *fakeKubelet) ListPodSandboxMetrics(ctx context.Context) ([]*runtimeapi.PodSandboxMetrics, error) {
+	return nil, nil
+}
+
 type fakeRuntime struct {
 	execFunc        func(string, []string, io.Reader, io.WriteCloser, io.WriteCloser, bool, <-chan remotecommand.TerminalSize) error
 	attachFunc      func(string, io.Reader, io.WriteCloser, io.WriteCloser, bool, <-chan remotecommand.TerminalSize) error

staging/src/k8s.io/component-base/metrics/value.go

@@ -47,6 +47,16 @@ func NewLazyConstMetric(desc *Desc, valueType ValueType, value float64, labelVal
 	return prometheus.MustNewConstMetric(desc.toPrometheusDesc(), valueType.toPromValueType(), value, labelValues...)
 }
 
+// NewConstMetric is a helper of NewConstMetric.
+//
+// Note: If the metrics described by the desc is hidden, the metrics will not be created.
+func NewConstMetric(desc *Desc, valueType ValueType, value float64, labelValues ...string) (Metric, error) {
+	if desc.IsHidden() {
+		return nil, nil
+	}
+	return prometheus.NewConstMetric(desc.toPrometheusDesc(), valueType.toPromValueType(), value, labelValues...)
+}
+
 // NewLazyMetricWithTimestamp is a helper of NewMetricWithTimestamp.
 //
 // Warning: the Metric 'm' must be the one created by NewLazyConstMetric(),

staging/src/k8s.io/cri-api/pkg/apis/runtime/v1/api.proto

@@ -121,6 +121,16 @@ service RuntimeService {
 
     // GetContainerEvents gets container events from the CRI runtime
     rpc  GetContainerEvents(GetEventsRequest) returns (stream ContainerEventResponse) {}
+
+    // ListMetricDescriptors gets the descriptors for the metrics that will be returned in ListPodSandboxMetrics.
+    // This list should be static at startup: either the client and server restart together when
+    // adding or removing metrics descriptors, or they should not change.
+    // Put differently, if ListPodSandboxMetrics references a name that is not described in the initial
+    // ListMetricDescriptors call, then the metric will not be broadcasted.
+    rpc ListMetricDescriptors(ListMetricDescriptorsRequest) returns (ListMetricDescriptorsResponse) {}
+
+    // ListPodSandboxMetrics gets pod sandbox metrics from CRI Runtime
+    rpc ListPodSandboxMetrics(ListPodSandboxMetricsRequest) returns (ListPodSandboxMetricsResponse) {}
 }
 
 // ImageService defines the public APIs for managing images.
@@ -1716,3 +1726,58 @@ enum ContainerEventType {
     // Container deleted
     CONTAINER_DELETED_EVENT = 3;
 }
+
+message ListMetricDescriptorsRequest {}
+
+message ListMetricDescriptorsResponse {
+    repeated MetricDescriptor descriptors = 1;
+}
+
+message MetricDescriptor {
+    // The name field will be used as a unique identifier of this MetricDescriptor,
+    // and be used in conjunction with the Metric structure to populate the full Metric.
+    string name = 1;
+    string help = 2;
+    // When a metric uses this metric descriptor, it should only define
+    // labels that have previously been declared in label_keys.
+    // It is the responsibility of the runtime to correctly keep sorted the keys and values.
+    // If the two slices have different length, the behavior is undefined.
+    repeated string label_keys = 3;
+}
+
+message ListPodSandboxMetricsRequest {} 
+
+message ListPodSandboxMetricsResponse {
+    repeated PodSandboxMetrics pod_metrics = 1;
+}
+
+message PodSandboxMetrics {
+    string pod_sandbox_id = 1;
+    repeated Metric metrics = 2;
+    repeated ContainerMetrics container_metrics = 3;
+}
+
+message ContainerMetrics {
+    string container_id = 1;
+    repeated Metric metrics = 2;
+}
+
+message Metric {
+    // Name must match a name previously returned in a MetricDescriptors call,
+    // otherwise, it will be ignored.
+    string name = 1;
+    // Timestamp should be 0 if the metric was gathered live.
+    // If it was cached, the Timestamp should reflect the time it was collected.
+    int64 timestamp = 2;
+    MetricType metric_type = 3;
+    // The corresponding LabelValues to the LabelKeys defined in the MetricDescriptor.
+    // It is the responsibility of the runtime to correctly keep sorted the keys and values.
+    // If the two slices have different length, the behavior is undefined.
+    repeated string label_values = 4;
+    UInt64Value value = 5;
+}
+
+enum MetricType {
+    COUNTER = 0;
+    GAUGE = 1;
+}

staging/src/k8s.io/cri-api/pkg/apis/services.go

@@ -97,6 +97,10 @@ type ContainerStatsManager interface {
 	PodSandboxStats(ctx context.Context, podSandboxID string) (*runtimeapi.PodSandboxStats, error)
 	// ListPodSandboxStats returns stats of all running pods.
 	ListPodSandboxStats(ctx context.Context, filter *runtimeapi.PodSandboxStatsFilter) ([]*runtimeapi.PodSandboxStats, error)
+	// ListMetricDescriptors gets the descriptors for the metrics that will be returned in ListPodSandboxMetrics.
+	ListMetricDescriptors(ctx context.Context) ([]*runtimeapi.MetricDescriptor, error)
+	// ListPodSandboxMetrics returns metrics of all running pods.
+	ListPodSandboxMetrics(ctx context.Context) ([]*runtimeapi.PodSandboxMetrics, error)
 }
 
 // RuntimeService interface should be implemented by a container runtime.

staging/src/k8s.io/cri-api/pkg/apis/testing/fake_runtime_service.go

@@ -64,11 +64,14 @@ type FakeRuntimeService struct {
 	Called []string
 	Errors map[string][]error
 
-	FakeStatus          *runtimeapi.RuntimeStatus
+	FakeStatus            *runtimeapi.RuntimeStatus
-	Containers          map[string]*FakeContainer
+	Containers            map[string]*FakeContainer
-	Sandboxes           map[string]*FakePodSandbox
+	Sandboxes             map[string]*FakePodSandbox
-	FakeContainerStats  map[string]*runtimeapi.ContainerStats
+	FakeContainerStats    map[string]*runtimeapi.ContainerStats
-	FakePodSandboxStats map[string]*runtimeapi.PodSandboxStats
+	FakePodSandboxStats   map[string]*runtimeapi.PodSandboxStats
+	FakePodSandboxMetrics map[string]*runtimeapi.PodSandboxMetrics
+	FakeMetricDescriptors map[string]*runtimeapi.MetricDescriptor
+	FakeContainerMetrics  map[string]*runtimeapi.ContainerMetrics
 
 	ErrorOnSandboxCreate bool
 }
@@ -153,12 +156,14 @@ func (r *FakeRuntimeService) popError(f string) error {
 // NewFakeRuntimeService creates a new FakeRuntimeService.
 func NewFakeRuntimeService() *FakeRuntimeService {
 	return &FakeRuntimeService{
-		Called:              make([]string, 0),
+		Called:                make([]string, 0),
-		Errors:              make(map[string][]error),
+		Errors:                make(map[string][]error),
-		Containers:          make(map[string]*FakeContainer),
+		Containers:            make(map[string]*FakeContainer),
-		Sandboxes:           make(map[string]*FakePodSandbox),
+		Sandboxes:             make(map[string]*FakePodSandbox),
-		FakeContainerStats:  make(map[string]*runtimeapi.ContainerStats),
+		FakeContainerStats:    make(map[string]*runtimeapi.ContainerStats),
-		FakePodSandboxStats: make(map[string]*runtimeapi.PodSandboxStats),
+		FakePodSandboxStats:   make(map[string]*runtimeapi.PodSandboxStats),
+		FakePodSandboxMetrics: make(map[string]*runtimeapi.PodSandboxMetrics),
+		FakeContainerMetrics:  make(map[string]*runtimeapi.ContainerMetrics),
 	}
 }
 
@@ -713,3 +718,65 @@ func (r *FakeRuntimeService) CheckpointContainer(_ context.Context, options *run
 func (f *FakeRuntimeService) GetContainerEvents(containerEventsCh chan *runtimeapi.ContainerEventResponse) error {
 	return nil
 }
+
+// SetFakeMetricDescriptors sets the fake metrics descriptors in the FakeRuntimeService.
+func (r *FakeRuntimeService) SetFakeMetricDescriptors(descs []*runtimeapi.MetricDescriptor) {
+	r.Lock()
+	defer r.Unlock()
+
+	r.FakeMetricDescriptors = make(map[string]*runtimeapi.MetricDescriptor)
+	for _, d := range descs {
+		r.FakeMetricDescriptors[d.Name] = d
+	}
+}
+
+// ListMetricDescriptors gets the descriptors for the metrics that will be returned in ListPodSandboxMetrics.
+func (r *FakeRuntimeService) ListMetricDescriptors(_ context.Context) ([]*runtimeapi.MetricDescriptor, error) {
+	r.Lock()
+	defer r.Unlock()
+
+	r.Called = append(r.Called, "ListMetricDescriptors")
+	if err := r.popError("ListMetricDescriptors"); err != nil {
+		return nil, err
+	}
+
+	descs := make([]*runtimeapi.MetricDescriptor, 0, len(r.FakeMetricDescriptors))
+	for _, d := range r.FakeMetricDescriptors {
+		descs = append(descs, d)
+	}
+
+	return descs, nil
+}
+
+// SetFakePodSandboxMetrics sets the fake pod sandbox metrics in the FakeRuntimeService.
+func (r *FakeRuntimeService) SetFakePodSandboxMetrics(podStats []*runtimeapi.PodSandboxMetrics) {
+	r.Lock()
+	defer r.Unlock()
+
+	r.FakePodSandboxMetrics = make(map[string]*runtimeapi.PodSandboxMetrics)
+	for _, s := range podStats {
+		r.FakePodSandboxMetrics[s.PodSandboxId] = s
+	}
+}
+
+// ListPodSandboxMetrics returns the list of all pod sandbox metrics in the FakeRuntimeService.
+func (r *FakeRuntimeService) ListPodSandboxMetrics(_ context.Context) ([]*runtimeapi.PodSandboxMetrics, error) {
+	r.Lock()
+	defer r.Unlock()
+
+	r.Called = append(r.Called, "ListPodSandboxMetrics")
+	if err := r.popError("ListPodSandboxMetrics"); err != nil {
+		return nil, err
+	}
+
+	var result []*runtimeapi.PodSandboxMetrics
+	for _, sb := range r.Sandboxes {
+		s, found := r.FakePodSandboxMetrics[sb.Id]
+		if !found {
+			continue
+		}
+		result = append(result, s)
+	}
+
+	return result, nil
+}