diff --git a/pkg/kubelet/apis/BUILD b/pkg/kubelet/apis/BUILD index 086ad1cc5b5..1118cce69d8 100644 --- a/pkg/kubelet/apis/BUILD +++ b/pkg/kubelet/apis/BUILD @@ -37,7 +37,6 @@ filegroup( ":package-srcs", "//pkg/kubelet/apis/config:all-srcs", "//pkg/kubelet/apis/podresources:all-srcs", - "//pkg/kubelet/apis/resourcemetrics/v1alpha1:all-srcs", "//pkg/kubelet/apis/stats/v1alpha1:all-srcs", ], tags = ["automanaged"], diff --git a/pkg/kubelet/apis/resourcemetrics/v1alpha1/BUILD b/pkg/kubelet/apis/resourcemetrics/v1alpha1/BUILD deleted file mode 100644 index 97ae7c573e0..00000000000 --- a/pkg/kubelet/apis/resourcemetrics/v1alpha1/BUILD +++ /dev/null @@ -1,22 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_library( - name = "go_default_library", - srcs = ["config.go"], - importpath = "k8s.io/kubernetes/pkg/kubelet/apis/resourcemetrics/v1alpha1", - visibility = ["//visibility:public"], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/pkg/kubelet/apis/resourcemetrics/v1alpha1/config.go b/pkg/kubelet/apis/resourcemetrics/v1alpha1/config.go deleted file mode 100644 index 6f0799cfa28..00000000000 --- a/pkg/kubelet/apis/resourcemetrics/v1alpha1/config.go +++ /dev/null @@ -1,23 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha1 - -// TODO(RainbowMango): We don't need to maintain this package anymore. -// This package will be remove in release 1.20.0+. More details please refer to https://github.com/kubernetes/kubernetes/pull/86282. - -// Version is the string representation of the version of this configuration -const Version = "v1alpha1" diff --git a/pkg/kubelet/server/BUILD b/pkg/kubelet/server/BUILD index 29fad991b98..c50ef5e0aa9 100644 --- a/pkg/kubelet/server/BUILD +++ b/pkg/kubelet/server/BUILD @@ -21,7 +21,6 @@ go_library( "//pkg/features:go_default_library", "//pkg/kubelet/apis/podresources:go_default_library", "//pkg/kubelet/apis/podresources/v1alpha1:go_default_library", - "//pkg/kubelet/apis/resourcemetrics/v1alpha1:go_default_library", "//pkg/kubelet/container:go_default_library", "//pkg/kubelet/cri/streaming:go_default_library", "//pkg/kubelet/cri/streaming/portforward:go_default_library", diff --git a/pkg/kubelet/server/auth_test.go b/pkg/kubelet/server/auth_test.go index 638f3ba2bf0..76a86165008 100644 --- a/pkg/kubelet/server/auth_test.go +++ b/pkg/kubelet/server/auth_test.go @@ -127,7 +127,6 @@ func AuthzTestCases() []AuthzTestCase { "/metrics": "metrics", "/metrics/cadvisor": "metrics", "/metrics/probes": "metrics", - "/metrics/resource/v1alpha1": "metrics", "/metrics/resource": "metrics", "/pods/": "proxy", "/portForward/{podNamespace}/{podID}": "proxy", diff --git a/pkg/kubelet/server/server.go b/pkg/kubelet/server/server.go index d4d2c9245b7..d07971e899e 100644 --- a/pkg/kubelet/server/server.go +++ b/pkg/kubelet/server/server.go @@ -25,7 +25,6 @@ import ( "net/http" "net/http/pprof" "net/url" - "path" "reflect" goruntime "runtime" "strconv" @@ -67,7 +66,6 @@ import ( "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/apis/podresources" podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" - "k8s.io/kubernetes/pkg/kubelet/apis/resourcemetrics/v1alpha1" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/cri/streaming" "k8s.io/kubernetes/pkg/kubelet/cri/streaming/portforward" @@ -343,7 +341,6 @@ func (s *Server) InstallDefaultHandlers(enableCAdvisorJSONEndpoints bool) { s.addMetricsBucketMatcher("metrics") s.addMetricsBucketMatcher("metrics/cadvisor") s.addMetricsBucketMatcher("metrics/probes") - s.addMetricsBucketMatcher("metrics/resource/v1alpha1") s.addMetricsBucketMatcher("metrics/resource") //lint:ignore SA1019 https://github.com/kubernetes/enhancements/issues/1206 s.restfulCont.Handle(metricsPath, legacyregistry.Handler()) @@ -378,14 +375,6 @@ func (s *Server) InstallDefaultHandlers(enableCAdvisorJSONEndpoints bool) { compbasemetrics.HandlerFor(r, compbasemetrics.HandlerOpts{ErrorHandling: compbasemetrics.ContinueOnError}), ) - // deprecated endpoint which will be removed in release 1.20.0+. - s.addMetricsBucketMatcher("metrics/resource/v1alpha1") - v1alpha1ResourceRegistry := compbasemetrics.NewKubeRegistry() - v1alpha1ResourceRegistry.CustomMustRegister(stats.NewPrometheusResourceMetricCollector(s.resourceAnalyzer, stats.Config())) - s.restfulCont.Handle(path.Join(resourceMetricsPath, v1alpha1.Version), - compbasemetrics.HandlerFor(v1alpha1ResourceRegistry, compbasemetrics.HandlerOpts{ErrorHandling: compbasemetrics.ContinueOnError}), - ) - s.addMetricsBucketMatcher("metrics/resource") resourceRegistry := compbasemetrics.NewKubeRegistry() resourceRegistry.CustomMustRegister(collectors.NewResourceMetricsCollector(s.resourceAnalyzer)) diff --git a/pkg/kubelet/server/server_test.go b/pkg/kubelet/server/server_test.go index 3a1446e6761..15779d4acc7 100644 --- a/pkg/kubelet/server/server_test.go +++ b/pkg/kubelet/server/server_test.go @@ -1507,7 +1507,6 @@ func TestMetricBuckets(t *testing.T) { "metrics": {url: "/metrics", bucket: "metrics"}, "metrics cadvisor sub": {url: "/metrics/cadvisor", bucket: "metrics/cadvisor"}, "metrics probes sub": {url: "/metrics/probes", bucket: "metrics/probes"}, - "metrics resource v1alpha1": {url: "/metrics/resource/v1alpha1", bucket: "metrics/resource"}, "metrics resource sub": {url: "/metrics/resource", bucket: "metrics/resource"}, "pods": {url: "/pods/", bucket: "pods"}, "portForward": {url: "/portForward/podNamespace/podID", bucket: "portForward"}, diff --git a/pkg/kubelet/server/stats/BUILD b/pkg/kubelet/server/stats/BUILD index 401de4ab181..71ebd5b0136 100644 --- a/pkg/kubelet/server/stats/BUILD +++ b/pkg/kubelet/server/stats/BUILD @@ -6,7 +6,6 @@ go_library( "doc.go", "fs_resource_analyzer.go", "handler.go", - "prometheus_resource_metrics.go", "resource_analyzer.go", "summary.go", "summary_sys_containers.go", @@ -26,7 +25,6 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//staging/src/k8s.io/component-base/metrics:go_default_library", "//vendor/github.com/emicklei/go-restful:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", "//vendor/github.com/google/cadvisor/info/v2:go_default_library", @@ -38,7 +36,6 @@ go_library( go_test( name = "go_default_test", srcs = [ - "prometheus_resource_metrics_test.go", "summary_test.go", "summary_windows_test.go", "volume_stat_calculator_test.go", @@ -51,9 +48,7 @@ go_test( "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/component-base/metrics/testutil:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", - "//vendor/github.com/stretchr/testify/mock:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:aix": [ "//pkg/kubelet/cm:go_default_library", diff --git a/pkg/kubelet/server/stats/prometheus_resource_metrics.go b/pkg/kubelet/server/stats/prometheus_resource_metrics.go deleted file mode 100644 index cb1500158d1..00000000000 --- a/pkg/kubelet/server/stats/prometheus_resource_metrics.go +++ /dev/null @@ -1,219 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package stats - -import ( - "time" - - "k8s.io/component-base/metrics" - "k8s.io/klog/v2" - stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" -) - -// This file contains a series of deprecated metrics which we emit them by endpoint `/metrics/resource/v1alpha1`. -// These metrics have been adapted to new endpoint `/metrics/resource` as well as new `Desc`s. -// In general, we don't need to maintain these deprecated metrics any more. -// TODO(RainbowMango): Remove this file in release 1.20.0+. - -var ( - nodeCPUUsageDesc = metrics.NewDesc("node_cpu_usage_seconds_total", - "Cumulative cpu time consumed by the node in core-seconds", - nil, - nil, - metrics.ALPHA, - "1.18.0") - - nodeMemoryUsageDesc = metrics.NewDesc("node_memory_working_set_bytes", - "Current working set of the node in bytes", - nil, - nil, - metrics.ALPHA, - "1.18.0") - - containerCPUUsageDesc = metrics.NewDesc("container_cpu_usage_seconds_total", - "Cumulative cpu time consumed by the container in core-seconds", - []string{"container", "pod", "namespace"}, - nil, - metrics.ALPHA, - "1.18.0") - - containerMemoryUsageDesc = metrics.NewDesc("container_memory_working_set_bytes", - "Current working set of the container in bytes", - []string{"container", "pod", "namespace"}, - nil, - metrics.ALPHA, - "1.18.0") -) - -// getNodeCPUMetrics returns CPU utilization of a node. -func getNodeCPUMetrics(s stats.NodeStats) (*float64, time.Time) { - if s.CPU == nil { - return nil, time.Time{} - } - v := float64(*s.CPU.UsageCoreNanoSeconds) / float64(time.Second) - return &v, s.CPU.Time.Time -} - -// getNodeMemoryMetrics returns memory utilization of a node. -func getNodeMemoryMetrics(s stats.NodeStats) (*float64, time.Time) { - if s.Memory == nil { - return nil, time.Time{} - } - v := float64(*s.Memory.WorkingSetBytes) - return &v, s.Memory.Time.Time -} - -// getContainerCPUMetrics returns CPU utilization of a container. -func getContainerCPUMetrics(s stats.ContainerStats) (*float64, time.Time) { - if s.CPU == nil { - return nil, time.Time{} - } - v := float64(*s.CPU.UsageCoreNanoSeconds) / float64(time.Second) - return &v, s.CPU.Time.Time -} - -// getContainerMemoryMetrics returns memory utilization of a container. -func getContainerMemoryMetrics(s stats.ContainerStats) (*float64, time.Time) { - if s.Memory == nil { - return nil, time.Time{} - } - v := float64(*s.Memory.WorkingSetBytes) - return &v, s.Memory.Time.Time -} - -// NodeResourceMetric describes a metric for the node -type NodeResourceMetric struct { - Desc *metrics.Desc - ValueFn func(stats.NodeStats) (*float64, time.Time) -} - -func (n *NodeResourceMetric) desc() *metrics.Desc { - return n.Desc -} - -// ContainerResourceMetric describes a metric for containers -type ContainerResourceMetric struct { - Desc *metrics.Desc - ValueFn func(stats.ContainerStats) (*float64, time.Time) -} - -func (n *ContainerResourceMetric) desc() *metrics.Desc { - return n.Desc -} - -// ResourceMetricsConfig specifies which metrics to collect and export -type ResourceMetricsConfig struct { - NodeMetrics []NodeResourceMetric - ContainerMetrics []ContainerResourceMetric -} - -// Config is the v1alpha1 resource metrics definition -func Config() ResourceMetricsConfig { - return ResourceMetricsConfig{ - NodeMetrics: []NodeResourceMetric{ - { - Desc: nodeCPUUsageDesc, - ValueFn: getNodeCPUMetrics, - }, - { - Desc: nodeMemoryUsageDesc, - ValueFn: getNodeMemoryMetrics, - }, - }, - ContainerMetrics: []ContainerResourceMetric{ - { - Desc: containerCPUUsageDesc, - ValueFn: getContainerCPUMetrics, - }, - { - Desc: containerMemoryUsageDesc, - ValueFn: getContainerMemoryMetrics, - }, - }, - } -} - -// NewPrometheusResourceMetricCollector returns a metrics.StableCollector which exports resource metrics -func NewPrometheusResourceMetricCollector(provider SummaryProvider, config ResourceMetricsConfig) metrics.StableCollector { - return &resourceMetricCollector{ - provider: provider, - config: config, - errors: metrics.NewDesc("scrape_error", - "1 if there was an error while getting container metrics, 0 otherwise", - nil, - nil, - metrics.ALPHA, - "1.18.0"), - } -} - -type resourceMetricCollector struct { - metrics.BaseStableCollector - - provider SummaryProvider - config ResourceMetricsConfig - errors *metrics.Desc -} - -var _ metrics.StableCollector = &resourceMetricCollector{} - -// DescribeWithStability implements metrics.StableCollector -func (rc *resourceMetricCollector) DescribeWithStability(ch chan<- *metrics.Desc) { - ch <- rc.errors - - for _, metric := range rc.config.NodeMetrics { - ch <- metric.desc() - } - for _, metric := range rc.config.ContainerMetrics { - ch <- metric.desc() - } -} - -// CollectWithStability implements metrics.StableCollector -// Since new containers are frequently created and removed, using the Gauge would -// leak metric collectors for containers or pods that no longer exist. Instead, implement -// custom collector in a way that only collects metrics for active containers. -func (rc *resourceMetricCollector) CollectWithStability(ch chan<- metrics.Metric) { - var errorCount float64 - defer func() { - ch <- metrics.NewLazyConstMetric(rc.errors, metrics.GaugeValue, errorCount) - }() - summary, err := rc.provider.GetCPUAndMemoryStats() - if err != nil { - errorCount = 1 - klog.Warningf("Error getting summary for resourceMetric prometheus endpoint: %v", err) - return - } - - for _, metric := range rc.config.NodeMetrics { - if value, timestamp := metric.ValueFn(summary.Node); value != nil { - ch <- metrics.NewLazyMetricWithTimestamp(timestamp, - metrics.NewLazyConstMetric(metric.desc(), metrics.GaugeValue, *value)) - } - } - - for _, pod := range summary.Pods { - for _, container := range pod.Containers { - for _, metric := range rc.config.ContainerMetrics { - if value, timestamp := metric.ValueFn(container); value != nil { - ch <- metrics.NewLazyMetricWithTimestamp(timestamp, - metrics.NewLazyConstMetric(metric.desc(), metrics.GaugeValue, *value, container.Name, pod.PodRef.Name, pod.PodRef.Namespace)) - } - } - } - } -} diff --git a/pkg/kubelet/server/stats/prometheus_resource_metrics_test.go b/pkg/kubelet/server/stats/prometheus_resource_metrics_test.go deleted file mode 100644 index 3b88c511a23..00000000000 --- a/pkg/kubelet/server/stats/prometheus_resource_metrics_test.go +++ /dev/null @@ -1,200 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package stats - -import ( - "fmt" - "strings" - "testing" - "time" - - "github.com/stretchr/testify/mock" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/component-base/metrics/testutil" - statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" -) - -type mockSummaryProvider struct { - mock.Mock -} - -func (m *mockSummaryProvider) Get(updateStats bool) (*statsapi.Summary, error) { - args := m.Called(updateStats) - return args.Get(0).(*statsapi.Summary), args.Error(1) -} - -func (m *mockSummaryProvider) GetCPUAndMemoryStats() (*statsapi.Summary, error) { - args := m.Called() - return args.Get(0).(*statsapi.Summary), args.Error(1) -} - -func TestCollectResourceMetrics(t *testing.T) { - testTime := metav1.NewTime(time.Unix(2, 0)) // a static timestamp: 2000 - - tests := []struct { - name string - config ResourceMetricsConfig - summary *statsapi.Summary - summaryErr error - expectedMetricsNames []string - expectedMetrics string - }{ - { - name: "error getting summary", - config: Config(), - summary: nil, - summaryErr: fmt.Errorf("failed to get summary"), - expectedMetricsNames: []string{"scrape_error"}, - expectedMetrics: ` - # HELP scrape_error [ALPHA] (Deprecated since 1.18.0) 1 if there was an error while getting container metrics, 0 otherwise - # TYPE scrape_error gauge - scrape_error 1 - `, - }, - { - name: "arbitrary node metrics", - config: Config(), - summary: &statsapi.Summary{ - Node: statsapi.NodeStats{ - CPU: &statsapi.CPUStats{ - Time: testTime, - UsageCoreNanoSeconds: uint64Ptr(10000000000), - }, - Memory: &statsapi.MemoryStats{ - Time: testTime, - WorkingSetBytes: uint64Ptr(1000), - }, - }, - }, - summaryErr: nil, - expectedMetricsNames: []string{ - "node_cpu_usage_seconds_total", - "node_memory_working_set_bytes", - "scrape_error", - }, - expectedMetrics: ` - # HELP node_cpu_usage_seconds_total [ALPHA] (Deprecated since 1.18.0) Cumulative cpu time consumed by the node in core-seconds - # TYPE node_cpu_usage_seconds_total gauge - node_cpu_usage_seconds_total 10 2000 - # HELP node_memory_working_set_bytes [ALPHA] (Deprecated since 1.18.0) Current working set of the node in bytes - # TYPE node_memory_working_set_bytes gauge - node_memory_working_set_bytes 1000 2000 - # HELP scrape_error [ALPHA] (Deprecated since 1.18.0) 1 if there was an error while getting container metrics, 0 otherwise - # TYPE scrape_error gauge - scrape_error 0 - `, - }, - { - name: "arbitrary container metrics for different container, pods and namespaces", - config: Config(), - summary: &statsapi.Summary{ - Pods: []statsapi.PodStats{ - { - PodRef: statsapi.PodReference{ - Name: "pod_a", - Namespace: "namespace_a", - }, - Containers: []statsapi.ContainerStats{ - { - Name: "container_a", - CPU: &statsapi.CPUStats{ - Time: testTime, - UsageCoreNanoSeconds: uint64Ptr(10000000000), - }, - Memory: &statsapi.MemoryStats{ - Time: testTime, - WorkingSetBytes: uint64Ptr(1000), - }, - }, - { - Name: "container_b", - CPU: &statsapi.CPUStats{ - Time: testTime, - UsageCoreNanoSeconds: uint64Ptr(10000000000), - }, - Memory: &statsapi.MemoryStats{ - Time: testTime, - WorkingSetBytes: uint64Ptr(1000), - }, - }, - }, - }, - { - PodRef: statsapi.PodReference{ - Name: "pod_b", - Namespace: "namespace_b", - }, - Containers: []statsapi.ContainerStats{ - { - Name: "container_a", - CPU: &statsapi.CPUStats{ - Time: testTime, - UsageCoreNanoSeconds: uint64Ptr(10000000000), - }, - Memory: &statsapi.MemoryStats{ - Time: testTime, - WorkingSetBytes: uint64Ptr(1000), - }, - }, - }, - }, - }, - }, - summaryErr: nil, - expectedMetricsNames: []string{ - "container_cpu_usage_seconds_total", - "container_memory_working_set_bytes", - "scrape_error", - }, - expectedMetrics: ` - # HELP scrape_error [ALPHA] (Deprecated since 1.18.0) 1 if there was an error while getting container metrics, 0 otherwise - # TYPE scrape_error gauge - scrape_error 0 - # HELP container_cpu_usage_seconds_total [ALPHA] (Deprecated since 1.18.0) Cumulative cpu time consumed by the container in core-seconds - # TYPE container_cpu_usage_seconds_total gauge - container_cpu_usage_seconds_total{container="container_a",namespace="namespace_a",pod="pod_a"} 10 2000 - container_cpu_usage_seconds_total{container="container_a",namespace="namespace_b",pod="pod_b"} 10 2000 - container_cpu_usage_seconds_total{container="container_b",namespace="namespace_a",pod="pod_a"} 10 2000 - # HELP container_memory_working_set_bytes [ALPHA] (Deprecated since 1.18.0) Current working set of the container in bytes - # TYPE container_memory_working_set_bytes gauge - container_memory_working_set_bytes{container="container_a",namespace="namespace_a",pod="pod_a"} 1000 2000 - container_memory_working_set_bytes{container="container_a",namespace="namespace_b",pod="pod_b"} 1000 2000 - container_memory_working_set_bytes{container="container_b",namespace="namespace_a",pod="pod_a"} 1000 2000 - `, - }, - } - - for _, test := range tests { - tc := test - t.Run(tc.name, func(t *testing.T) { - provider := &mockSummaryProvider{} - provider.On("GetCPUAndMemoryStats").Return(tc.summary, tc.summaryErr) - collector := NewPrometheusResourceMetricCollector(provider, tc.config) - - registry := testutil.NewFakeKubeRegistry("1.18.0") - registry.CustomMustRegister(collector) - if err := testutil.GatherAndCompare(registry, strings.NewReader(tc.expectedMetrics), tc.expectedMetricsNames...); err != nil { - t.Fatal(err) - } - }) - } -} - -func uint64Ptr(u uint64) *uint64 { - return &u -}