From 782a7a54269cba12bd16f90347127c9efd41fa36 Mon Sep 17 00:00:00 2001 From: David Eads Date: Thu, 7 Jan 2021 11:25:43 -0500 Subject: [PATCH] add metrics for insecure backend proxy --- pkg/registry/core/pod/rest/BUILD | 4 ++ pkg/registry/core/pod/rest/log.go | 39 +++++++++-- pkg/registry/core/pod/rest/metrics.go | 67 +++++++++++++++++++ .../pkg/registry/generic/rest/streamer.go | 11 +++ 4 files changed, 115 insertions(+), 6 deletions(-) create mode 100644 pkg/registry/core/pod/rest/metrics.go diff --git a/pkg/registry/core/pod/rest/BUILD b/pkg/registry/core/pod/rest/BUILD index e61a2ff13cf..4b73a3234ba 100644 --- a/pkg/registry/core/pod/rest/BUILD +++ b/pkg/registry/core/pod/rest/BUILD @@ -10,6 +10,7 @@ go_library( name = "go_default_library", srcs = [ "log.go", + "metrics.go", "subresources.go", ], importpath = "k8s.io/kubernetes/pkg/registry/core/pod/rest", @@ -25,11 +26,14 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/proxy:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apiserver/pkg/features:go_default_library", "//staging/src/k8s.io/apiserver/pkg/registry/generic/registry:go_default_library", "//staging/src/k8s.io/apiserver/pkg/registry/generic/rest:go_default_library", "//staging/src/k8s.io/apiserver/pkg/registry/rest:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", + "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library", ], ) diff --git a/pkg/registry/core/pod/rest/log.go b/pkg/registry/core/pod/rest/log.go index 9eb14ec8bac..8a98e5ae987 100644 --- a/pkg/registry/core/pod/rest/log.go +++ b/pkg/registry/core/pod/rest/log.go @@ -20,6 +20,8 @@ import ( "context" "fmt" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" @@ -70,10 +72,16 @@ func (r *LogREST) ProducesObject(verb string) interface{} { // Get retrieves a runtime.Object that will stream the contents of the pod log func (r *LogREST) Get(ctx context.Context, name string, opts runtime.Object) (runtime.Object, error) { + // register the metrics if the context is used. This assumes sync.Once is fast. If it's not, it could be an init block. + registerMetrics() + logOpts, ok := opts.(*api.PodLogOptions) if !ok { return nil, fmt.Errorf("invalid options object: %#v", opts) } + + // we must do this before forcing the insecure flag if the feature is disabled + countSkipTLSMetric(logOpts.InsecureSkipTLSVerifyBackend) if !utilfeature.DefaultFeatureGate.Enabled(features.AllowInsecureBackendProxy) { logOpts.InsecureSkipTLSVerifyBackend = false } @@ -86,15 +94,34 @@ func (r *LogREST) Get(ctx context.Context, name string, opts runtime.Object) (ru return nil, err } return &genericrest.LocationStreamer{ - Location: location, - Transport: transport, - ContentType: "text/plain", - Flush: logOpts.Follow, - ResponseChecker: genericrest.NewGenericHttpResponseChecker(api.Resource("pods/log"), name), - RedirectChecker: genericrest.PreventRedirects, + Location: location, + Transport: transport, + ContentType: "text/plain", + Flush: logOpts.Follow, + ResponseChecker: genericrest.NewGenericHttpResponseChecker(api.Resource("pods/log"), name), + RedirectChecker: genericrest.PreventRedirects, + TLSVerificationErrorCounter: podLogsTLSFailure, }, nil } +func countSkipTLSMetric(insecureSkipTLSVerifyBackend bool) { + usageType := usageEnforce + if insecureSkipTLSVerifyBackend { + if utilfeature.DefaultFeatureGate.Enabled(features.AllowInsecureBackendProxy) { + usageType = usageSkipAllowed + } else { + usageType = usageSkipDenied + } + } + + counter, err := podLogsUsage.GetMetricWithLabelValues(usageType) + if err != nil { + utilruntime.HandleError(err) + return + } + counter.Inc() +} + // NewGetOptions creates a new options object func (r *LogREST) NewGetOptions() (runtime.Object, bool, string) { return &api.PodLogOptions{}, false, "" diff --git a/pkg/registry/core/pod/rest/metrics.go b/pkg/registry/core/pod/rest/metrics.go new file mode 100644 index 00000000000..7286c4d7ab6 --- /dev/null +++ b/pkg/registry/core/pod/rest/metrics.go @@ -0,0 +1,67 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rest + +import ( + "sync" + + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" +) + +const ( + namespace = "kube_apiserver" + subsystem = "pod_logs" + + usageEnforce = "enforce_tls" + usageSkipAllowed = "skip_tls_allowed" + usageSkipDenied = "skip_tls_denied" +) + +var ( + // podLogsUsage counts and categorizes how the insecure backend skip TLS option is used and allowed. + podLogsUsage = metrics.NewCounterVec( + &metrics.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "pods_logs_insecure_backend_total", + Help: "Total number of requests for pods/logs sliced by usage type: enforce_tls, skip_tls_allowed, skip_tls_denied", + StabilityLevel: metrics.ALPHA, + }, + []string{"usage"}, + ) + + // podLogsTLSFailure counts how many attempts to get pod logs fail on tls verification + podLogsTLSFailure = metrics.NewCounter( + &metrics.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "pods_logs_backend_tls_failure_total", + Help: "Total number of requests for pods/logs that failed due to kubelet server TLS verification", + StabilityLevel: metrics.ALPHA, + }, + ) +) + +var registerMetricsOnce sync.Once + +func registerMetrics() { + registerMetricsOnce.Do(func() { + legacyregistry.MustRegister(podLogsUsage) + legacyregistry.MustRegister(podLogsTLSFailure) + }) +} diff --git a/staging/src/k8s.io/apiserver/pkg/registry/generic/rest/streamer.go b/staging/src/k8s.io/apiserver/pkg/registry/generic/rest/streamer.go index be911fab2f1..cb9e4898cd5 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/generic/rest/streamer.go +++ b/staging/src/k8s.io/apiserver/pkg/registry/generic/rest/streamer.go @@ -30,6 +30,10 @@ import ( "k8s.io/apiserver/pkg/registry/rest" ) +type CounterMetric interface { + Inc() +} + // LocationStreamer is a resource that streams the contents of a particular // location URL. type LocationStreamer struct { @@ -39,6 +43,9 @@ type LocationStreamer struct { Flush bool ResponseChecker HttpResponseChecker RedirectChecker func(req *http.Request, via []*http.Request) error + // TLSVerificationErrorCounter is an optional value that will Inc every time a TLS error is encountered. This can + // be wired a single prometheus counter instance to get counts overall. + TLSVerificationErrorCounter CounterMetric } // a LocationStreamer must implement a rest.ResourceStreamer @@ -77,6 +84,10 @@ func (s *LocationStreamer) InputStream(ctx context.Context, apiVersion, acceptHe resp, err := client.Do(req) if err != nil { + // TODO prefer segregate TLS errors more reliably, but we do want to increment a count + if strings.Contains(err.Error(), "x509:") && s.TLSVerificationErrorCounter != nil { + s.TLSVerificationErrorCounter.Inc() + } return nil, false, "", err }