Merge pull request #94272 from RainbowMango/pr_remove_deprecated_endpoint

remove deprecated kubelet endpoint /metrics/resource/v1alpha1

pkg/kubelet/apis/BUILD

@@ -37,7 +37,6 @@ filegroup(
         ":package-srcs",
         "//pkg/kubelet/apis/config:all-srcs",
         "//pkg/kubelet/apis/podresources:all-srcs",
-        "//pkg/kubelet/apis/resourcemetrics/v1alpha1:all-srcs",
         "//pkg/kubelet/apis/stats/v1alpha1:all-srcs",
     ],
     tags = ["automanaged"],

pkg/kubelet/apis/resourcemetrics/v1alpha1/BUILD

@@ -1,22 +0,0 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
-
-go_library(
-    name = "go_default_library",
-    srcs = ["config.go"],
-    importpath = "k8s.io/kubernetes/pkg/kubelet/apis/resourcemetrics/v1alpha1",
-    visibility = ["//visibility:public"],
-)
-
-filegroup(
-    name = "package-srcs",
-    srcs = glob(["**"]),
-    tags = ["automanaged"],
-    visibility = ["//visibility:private"],
-)
-
-filegroup(
-    name = "all-srcs",
-    srcs = [":package-srcs"],
-    tags = ["automanaged"],
-    visibility = ["//visibility:public"],
-)

pkg/kubelet/apis/resourcemetrics/v1alpha1/config.go

@@ -1,23 +0,0 @@
-/*
-Copyright 2019 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package v1alpha1
-
-// TODO(RainbowMango): We don't need to maintain this package anymore.
-//  This package will be remove in release 1.20.0+. More details please refer to https://github.com/kubernetes/kubernetes/pull/86282.
-
-// Version is the string representation of the version of this configuration
-const Version = "v1alpha1"

pkg/kubelet/server/BUILD

@@ -21,7 +21,6 @@ go_library(
         "//pkg/features:go_default_library",
         "//pkg/kubelet/apis/podresources:go_default_library",
         "//pkg/kubelet/apis/podresources/v1alpha1:go_default_library",
-        "//pkg/kubelet/apis/resourcemetrics/v1alpha1:go_default_library",
         "//pkg/kubelet/container:go_default_library",
         "//pkg/kubelet/cri/streaming:go_default_library",
         "//pkg/kubelet/cri/streaming/portforward:go_default_library",

pkg/kubelet/server/auth_test.go

@@ -127,7 +127,6 @@ func AuthzTestCases() []AuthzTestCase {
 		"/metrics":                            "metrics",
 		"/metrics/cadvisor":                   "metrics",
 		"/metrics/probes":                     "metrics",
-		"/metrics/resource/v1alpha1":          "metrics",
 		"/metrics/resource":                   "metrics",
 		"/pods/":                              "proxy",
 		"/portForward/{podNamespace}/{podID}": "proxy",

pkg/kubelet/server/server.go

@@ -25,7 +25,6 @@ import (
 	"net/http"
 	"net/http/pprof"
 	"net/url"
-	"path"
 	"reflect"
 	goruntime "runtime"
 	"strconv"
@@ -67,7 +66,6 @@ import (
 	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/kubelet/apis/podresources"
 	podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1"
-	"k8s.io/kubernetes/pkg/kubelet/apis/resourcemetrics/v1alpha1"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 	"k8s.io/kubernetes/pkg/kubelet/cri/streaming"
 	"k8s.io/kubernetes/pkg/kubelet/cri/streaming/portforward"
@@ -343,7 +341,6 @@ func (s *Server) InstallDefaultHandlers(enableCAdvisorJSONEndpoints bool) {
 	s.addMetricsBucketMatcher("metrics")
 	s.addMetricsBucketMatcher("metrics/cadvisor")
 	s.addMetricsBucketMatcher("metrics/probes")
-	s.addMetricsBucketMatcher("metrics/resource/v1alpha1")
 	s.addMetricsBucketMatcher("metrics/resource")
 	//lint:ignore SA1019 https://github.com/kubernetes/enhancements/issues/1206
 	s.restfulCont.Handle(metricsPath, legacyregistry.Handler())
@@ -378,14 +375,6 @@ func (s *Server) InstallDefaultHandlers(enableCAdvisorJSONEndpoints bool) {
 		compbasemetrics.HandlerFor(r, compbasemetrics.HandlerOpts{ErrorHandling: compbasemetrics.ContinueOnError}),
 	)
 
-	// deprecated endpoint which will be removed in release 1.20.0+.
-	s.addMetricsBucketMatcher("metrics/resource/v1alpha1")
-	v1alpha1ResourceRegistry := compbasemetrics.NewKubeRegistry()
-	v1alpha1ResourceRegistry.CustomMustRegister(stats.NewPrometheusResourceMetricCollector(s.resourceAnalyzer, stats.Config()))
-	s.restfulCont.Handle(path.Join(resourceMetricsPath, v1alpha1.Version),
-		compbasemetrics.HandlerFor(v1alpha1ResourceRegistry, compbasemetrics.HandlerOpts{ErrorHandling: compbasemetrics.ContinueOnError}),
-	)
-
 	s.addMetricsBucketMatcher("metrics/resource")
 	resourceRegistry := compbasemetrics.NewKubeRegistry()
 	resourceRegistry.CustomMustRegister(collectors.NewResourceMetricsCollector(s.resourceAnalyzer))

pkg/kubelet/server/server_test.go

@@ -1507,7 +1507,6 @@ func TestMetricBuckets(t *testing.T) {
 		"metrics":                         {url: "/metrics", bucket: "metrics"},
 		"metrics cadvisor sub":            {url: "/metrics/cadvisor", bucket: "metrics/cadvisor"},
 		"metrics probes sub":              {url: "/metrics/probes", bucket: "metrics/probes"},
-		"metrics resource v1alpha1":       {url: "/metrics/resource/v1alpha1", bucket: "metrics/resource"},
 		"metrics resource sub":            {url: "/metrics/resource", bucket: "metrics/resource"},
 		"pods":                            {url: "/pods/", bucket: "pods"},
 		"portForward":                     {url: "/portForward/podNamespace/podID", bucket: "portForward"},

pkg/kubelet/server/stats/BUILD

@@ -6,7 +6,6 @@ go_library(
         "doc.go",
         "fs_resource_analyzer.go",
         "handler.go",
-        "prometheus_resource_metrics.go",
         "resource_analyzer.go",
         "summary.go",
         "summary_sys_containers.go",
@@ -26,7 +25,6 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
-        "//staging/src/k8s.io/component-base/metrics:go_default_library",
         "//vendor/github.com/emicklei/go-restful:go_default_library",
         "//vendor/github.com/google/cadvisor/info/v1:go_default_library",
         "//vendor/github.com/google/cadvisor/info/v2:go_default_library",
@@ -38,7 +36,6 @@ go_library(
 go_test(
     name = "go_default_test",
     srcs = [
-        "prometheus_resource_metrics_test.go",
         "summary_test.go",
         "summary_windows_test.go",
         "volume_stat_calculator_test.go",
@@ -51,9 +48,7 @@ go_test(
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//staging/src/k8s.io/component-base/metrics/testutil:go_default_library",
         "//vendor/github.com/stretchr/testify/assert:go_default_library",
-        "//vendor/github.com/stretchr/testify/mock:go_default_library",
     ] + select({
         "@io_bazel_rules_go//go/platform:aix": [
             "//pkg/kubelet/cm:go_default_library",

pkg/kubelet/server/stats/prometheus_resource_metrics.go

@@ -1,219 +0,0 @@
-/*
-Copyright 2019 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package stats
-
-import (
-	"time"
-
-	"k8s.io/component-base/metrics"
-	"k8s.io/klog/v2"
-	stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
-)
-
-// This file contains a series of deprecated metrics which we emit them by endpoint `/metrics/resource/v1alpha1`.
-// These metrics have been adapted to new endpoint `/metrics/resource` as well as new `Desc`s.
-// In general, we don't need to maintain these deprecated metrics any more.
-// TODO(RainbowMango): Remove this file in release 1.20.0+.
-
-var (
-	nodeCPUUsageDesc = metrics.NewDesc("node_cpu_usage_seconds_total",
-		"Cumulative cpu time consumed by the node in core-seconds",
-		nil,
-		nil,
-		metrics.ALPHA,
-		"1.18.0")
-
-	nodeMemoryUsageDesc = metrics.NewDesc("node_memory_working_set_bytes",
-		"Current working set of the node in bytes",
-		nil,
-		nil,
-		metrics.ALPHA,
-		"1.18.0")
-
-	containerCPUUsageDesc = metrics.NewDesc("container_cpu_usage_seconds_total",
-		"Cumulative cpu time consumed by the container in core-seconds",
-		[]string{"container", "pod", "namespace"},
-		nil,
-		metrics.ALPHA,
-		"1.18.0")
-
-	containerMemoryUsageDesc = metrics.NewDesc("container_memory_working_set_bytes",
-		"Current working set of the container in bytes",
-		[]string{"container", "pod", "namespace"},
-		nil,
-		metrics.ALPHA,
-		"1.18.0")
-)
-
-// getNodeCPUMetrics returns CPU utilization of a node.
-func getNodeCPUMetrics(s stats.NodeStats) (*float64, time.Time) {
-	if s.CPU == nil {
-		return nil, time.Time{}
-	}
-	v := float64(*s.CPU.UsageCoreNanoSeconds) / float64(time.Second)
-	return &v, s.CPU.Time.Time
-}
-
-// getNodeMemoryMetrics returns memory utilization of a node.
-func getNodeMemoryMetrics(s stats.NodeStats) (*float64, time.Time) {
-	if s.Memory == nil {
-		return nil, time.Time{}
-	}
-	v := float64(*s.Memory.WorkingSetBytes)
-	return &v, s.Memory.Time.Time
-}
-
-// getContainerCPUMetrics returns CPU utilization of a container.
-func getContainerCPUMetrics(s stats.ContainerStats) (*float64, time.Time) {
-	if s.CPU == nil {
-		return nil, time.Time{}
-	}
-	v := float64(*s.CPU.UsageCoreNanoSeconds) / float64(time.Second)
-	return &v, s.CPU.Time.Time
-}
-
-// getContainerMemoryMetrics returns memory utilization of a container.
-func getContainerMemoryMetrics(s stats.ContainerStats) (*float64, time.Time) {
-	if s.Memory == nil {
-		return nil, time.Time{}
-	}
-	v := float64(*s.Memory.WorkingSetBytes)
-	return &v, s.Memory.Time.Time
-}
-
-// NodeResourceMetric describes a metric for the node
-type NodeResourceMetric struct {
-	Desc    *metrics.Desc
-	ValueFn func(stats.NodeStats) (*float64, time.Time)
-}
-
-func (n *NodeResourceMetric) desc() *metrics.Desc {
-	return n.Desc
-}
-
-// ContainerResourceMetric describes a metric for containers
-type ContainerResourceMetric struct {
-	Desc    *metrics.Desc
-	ValueFn func(stats.ContainerStats) (*float64, time.Time)
-}
-
-func (n *ContainerResourceMetric) desc() *metrics.Desc {
-	return n.Desc
-}
-
-// ResourceMetricsConfig specifies which metrics to collect and export
-type ResourceMetricsConfig struct {
-	NodeMetrics      []NodeResourceMetric
-	ContainerMetrics []ContainerResourceMetric
-}
-
-// Config is the v1alpha1 resource metrics definition
-func Config() ResourceMetricsConfig {
-	return ResourceMetricsConfig{
-		NodeMetrics: []NodeResourceMetric{
-			{
-				Desc:    nodeCPUUsageDesc,
-				ValueFn: getNodeCPUMetrics,
-			},
-			{
-				Desc:    nodeMemoryUsageDesc,
-				ValueFn: getNodeMemoryMetrics,
-			},
-		},
-		ContainerMetrics: []ContainerResourceMetric{
-			{
-				Desc:    containerCPUUsageDesc,
-				ValueFn: getContainerCPUMetrics,
-			},
-			{
-				Desc:    containerMemoryUsageDesc,
-				ValueFn: getContainerMemoryMetrics,
-			},
-		},
-	}
-}
-
-// NewPrometheusResourceMetricCollector returns a metrics.StableCollector which exports resource metrics
-func NewPrometheusResourceMetricCollector(provider SummaryProvider, config ResourceMetricsConfig) metrics.StableCollector {
-	return &resourceMetricCollector{
-		provider: provider,
-		config:   config,
-		errors: metrics.NewDesc("scrape_error",
-			"1 if there was an error while getting container metrics, 0 otherwise",
-			nil,
-			nil,
-			metrics.ALPHA,
-			"1.18.0"),
-	}
-}
-
-type resourceMetricCollector struct {
-	metrics.BaseStableCollector
-
-	provider SummaryProvider
-	config   ResourceMetricsConfig
-	errors   *metrics.Desc
-}
-
-var _ metrics.StableCollector = &resourceMetricCollector{}
-
-// DescribeWithStability implements metrics.StableCollector
-func (rc *resourceMetricCollector) DescribeWithStability(ch chan<- *metrics.Desc) {
-	ch <- rc.errors
-
-	for _, metric := range rc.config.NodeMetrics {
-		ch <- metric.desc()
-	}
-	for _, metric := range rc.config.ContainerMetrics {
-		ch <- metric.desc()
-	}
-}
-
-// CollectWithStability implements metrics.StableCollector
-// Since new containers are frequently created and removed, using the Gauge would
-// leak metric collectors for containers or pods that no longer exist.  Instead, implement
-// custom collector in a way that only collects metrics for active containers.
-func (rc *resourceMetricCollector) CollectWithStability(ch chan<- metrics.Metric) {
-	var errorCount float64
-	defer func() {
-		ch <- metrics.NewLazyConstMetric(rc.errors, metrics.GaugeValue, errorCount)
-	}()
-	summary, err := rc.provider.GetCPUAndMemoryStats()
-	if err != nil {
-		errorCount = 1
-		klog.Warningf("Error getting summary for resourceMetric prometheus endpoint: %v", err)
-		return
-	}
-
-	for _, metric := range rc.config.NodeMetrics {
-		if value, timestamp := metric.ValueFn(summary.Node); value != nil {
-			ch <- metrics.NewLazyMetricWithTimestamp(timestamp,
-				metrics.NewLazyConstMetric(metric.desc(), metrics.GaugeValue, *value))
-		}
-	}
-
-	for _, pod := range summary.Pods {
-		for _, container := range pod.Containers {
-			for _, metric := range rc.config.ContainerMetrics {
-				if value, timestamp := metric.ValueFn(container); value != nil {
-					ch <- metrics.NewLazyMetricWithTimestamp(timestamp,
-						metrics.NewLazyConstMetric(metric.desc(), metrics.GaugeValue, *value, container.Name, pod.PodRef.Name, pod.PodRef.Namespace))
-				}
-			}
-		}
-	}
-}

pkg/kubelet/server/stats/prometheus_resource_metrics_test.go

@@ -1,200 +0,0 @@
-/*
-Copyright 2019 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package stats
-
-import (
-	"fmt"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/mock"
-
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/component-base/metrics/testutil"
-	statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
-)
-
-type mockSummaryProvider struct {
-	mock.Mock
-}
-
-func (m *mockSummaryProvider) Get(updateStats bool) (*statsapi.Summary, error) {
-	args := m.Called(updateStats)
-	return args.Get(0).(*statsapi.Summary), args.Error(1)
-}
-
-func (m *mockSummaryProvider) GetCPUAndMemoryStats() (*statsapi.Summary, error) {
-	args := m.Called()
-	return args.Get(0).(*statsapi.Summary), args.Error(1)
-}
-
-func TestCollectResourceMetrics(t *testing.T) {
-	testTime := metav1.NewTime(time.Unix(2, 0)) // a static timestamp: 2000
-
-	tests := []struct {
-		name                 string
-		config               ResourceMetricsConfig
-		summary              *statsapi.Summary
-		summaryErr           error
-		expectedMetricsNames []string
-		expectedMetrics      string
-	}{
-		{
-			name:                 "error getting summary",
-			config:               Config(),
-			summary:              nil,
-			summaryErr:           fmt.Errorf("failed to get summary"),
-			expectedMetricsNames: []string{"scrape_error"},
-			expectedMetrics: `
-				# HELP scrape_error [ALPHA] (Deprecated since 1.18.0) 1 if there was an error while getting container metrics, 0 otherwise
-				# TYPE scrape_error gauge
-				scrape_error 1
-			`,
-		},
-		{
-			name:   "arbitrary node metrics",
-			config: Config(),
-			summary: &statsapi.Summary{
-				Node: statsapi.NodeStats{
-					CPU: &statsapi.CPUStats{
-						Time:                 testTime,
-						UsageCoreNanoSeconds: uint64Ptr(10000000000),
-					},
-					Memory: &statsapi.MemoryStats{
-						Time:            testTime,
-						WorkingSetBytes: uint64Ptr(1000),
-					},
-				},
-			},
-			summaryErr: nil,
-			expectedMetricsNames: []string{
-				"node_cpu_usage_seconds_total",
-				"node_memory_working_set_bytes",
-				"scrape_error",
-			},
-			expectedMetrics: `
-				# HELP node_cpu_usage_seconds_total [ALPHA] (Deprecated since 1.18.0) Cumulative cpu time consumed by the node in core-seconds
-				# TYPE node_cpu_usage_seconds_total gauge
-				node_cpu_usage_seconds_total 10 2000
-				# HELP node_memory_working_set_bytes [ALPHA] (Deprecated since 1.18.0) Current working set of the node in bytes
-				# TYPE node_memory_working_set_bytes gauge
-				node_memory_working_set_bytes 1000 2000
-				# HELP scrape_error [ALPHA] (Deprecated since 1.18.0) 1 if there was an error while getting container metrics, 0 otherwise
-				# TYPE scrape_error gauge
-				scrape_error 0
-			`,
-		},
-		{
-			name:   "arbitrary container metrics for different container, pods and namespaces",
-			config: Config(),
-			summary: &statsapi.Summary{
-				Pods: []statsapi.PodStats{
-					{
-						PodRef: statsapi.PodReference{
-							Name:      "pod_a",
-							Namespace: "namespace_a",
-						},
-						Containers: []statsapi.ContainerStats{
-							{
-								Name: "container_a",
-								CPU: &statsapi.CPUStats{
-									Time:                 testTime,
-									UsageCoreNanoSeconds: uint64Ptr(10000000000),
-								},
-								Memory: &statsapi.MemoryStats{
-									Time:            testTime,
-									WorkingSetBytes: uint64Ptr(1000),
-								},
-							},
-							{
-								Name: "container_b",
-								CPU: &statsapi.CPUStats{
-									Time:                 testTime,
-									UsageCoreNanoSeconds: uint64Ptr(10000000000),
-								},
-								Memory: &statsapi.MemoryStats{
-									Time:            testTime,
-									WorkingSetBytes: uint64Ptr(1000),
-								},
-							},
-						},
-					},
-					{
-						PodRef: statsapi.PodReference{
-							Name:      "pod_b",
-							Namespace: "namespace_b",
-						},
-						Containers: []statsapi.ContainerStats{
-							{
-								Name: "container_a",
-								CPU: &statsapi.CPUStats{
-									Time:                 testTime,
-									UsageCoreNanoSeconds: uint64Ptr(10000000000),
-								},
-								Memory: &statsapi.MemoryStats{
-									Time:            testTime,
-									WorkingSetBytes: uint64Ptr(1000),
-								},
-							},
-						},
-					},
-				},
-			},
-			summaryErr: nil,
-			expectedMetricsNames: []string{
-				"container_cpu_usage_seconds_total",
-				"container_memory_working_set_bytes",
-				"scrape_error",
-			},
-			expectedMetrics: `
-				# HELP scrape_error [ALPHA] (Deprecated since 1.18.0) 1 if there was an error while getting container metrics, 0 otherwise
-				# TYPE scrape_error gauge
-				scrape_error 0
-				# HELP container_cpu_usage_seconds_total [ALPHA] (Deprecated since 1.18.0) Cumulative cpu time consumed by the container in core-seconds
-				# TYPE container_cpu_usage_seconds_total gauge
-				container_cpu_usage_seconds_total{container="container_a",namespace="namespace_a",pod="pod_a"} 10 2000
-				container_cpu_usage_seconds_total{container="container_a",namespace="namespace_b",pod="pod_b"} 10 2000
-				container_cpu_usage_seconds_total{container="container_b",namespace="namespace_a",pod="pod_a"} 10 2000
-				# HELP container_memory_working_set_bytes [ALPHA] (Deprecated since 1.18.0) Current working set of the container in bytes
-				# TYPE container_memory_working_set_bytes gauge
-				container_memory_working_set_bytes{container="container_a",namespace="namespace_a",pod="pod_a"} 1000 2000
-				container_memory_working_set_bytes{container="container_a",namespace="namespace_b",pod="pod_b"} 1000 2000
-				container_memory_working_set_bytes{container="container_b",namespace="namespace_a",pod="pod_a"} 1000 2000
-			`,
-		},
-	}
-
-	for _, test := range tests {
-		tc := test
-		t.Run(tc.name, func(t *testing.T) {
-			provider := &mockSummaryProvider{}
-			provider.On("GetCPUAndMemoryStats").Return(tc.summary, tc.summaryErr)
-			collector := NewPrometheusResourceMetricCollector(provider, tc.config)
-
-			registry := testutil.NewFakeKubeRegistry("1.18.0")
-			registry.CustomMustRegister(collector)
-			if err := testutil.GatherAndCompare(registry, strings.NewReader(tc.expectedMetrics), tc.expectedMetricsNames...); err != nil {
-				t.Fatal(err)
-			}
-		})
-	}
-}
-
-func uint64Ptr(u uint64) *uint64 {
-	return &u
-}