Merge pull request #97814 from deads2k/podlogs-metrics

add metrics for insecure backend proxy

pkg/registry/core/pod/rest/BUILD

@@ -10,6 +10,7 @@ go_library(
     name = "go_default_library",
     srcs = [
         "log.go",
+        "metrics.go",
         "subresources.go",
     ],
     importpath = "k8s.io/kubernetes/pkg/registry/core/pod/rest",
@@ -25,11 +26,14 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/proxy:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/features:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/registry/generic/registry:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/registry/generic/rest:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/registry/rest:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//staging/src/k8s.io/component-base/metrics:go_default_library",
+        "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library",
     ],
 )
 

pkg/registry/core/pod/rest/log.go

@@ -20,6 +20,8 @@ import (
 	"context"
 	"fmt"
 
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	genericregistry "k8s.io/apiserver/pkg/registry/generic/registry"
@@ -70,10 +72,16 @@ func (r *LogREST) ProducesObject(verb string) interface{} {
 
 // Get retrieves a runtime.Object that will stream the contents of the pod log
 func (r *LogREST) Get(ctx context.Context, name string, opts runtime.Object) (runtime.Object, error) {
+	// register the metrics if the context is used.  This assumes sync.Once is fast.  If it's not, it could be an init block.
+	registerMetrics()
+
 	logOpts, ok := opts.(*api.PodLogOptions)
 	if !ok {
 		return nil, fmt.Errorf("invalid options object: %#v", opts)
 	}
+
+	// we must do this before forcing the insecure flag if the feature is disabled
+	countSkipTLSMetric(logOpts.InsecureSkipTLSVerifyBackend)
 	if !utilfeature.DefaultFeatureGate.Enabled(features.AllowInsecureBackendProxy) {
 		logOpts.InsecureSkipTLSVerifyBackend = false
 	}
@@ -86,15 +94,34 @@ func (r *LogREST) Get(ctx context.Context, name string, opts runtime.Object) (ru
 		return nil, err
 	}
 	return &genericrest.LocationStreamer{
-		Location:        location,
-		Transport:       transport,
-		ContentType:     "text/plain",
-		Flush:           logOpts.Follow,
-		ResponseChecker: genericrest.NewGenericHttpResponseChecker(api.Resource("pods/log"), name),
-		RedirectChecker: genericrest.PreventRedirects,
+		Location:                    location,
+		Transport:                   transport,
+		ContentType:                 "text/plain",
+		Flush:                       logOpts.Follow,
+		ResponseChecker:             genericrest.NewGenericHttpResponseChecker(api.Resource("pods/log"), name),
+		RedirectChecker:             genericrest.PreventRedirects,
+		TLSVerificationErrorCounter: podLogsTLSFailure,
 	}, nil
 }
 
+func countSkipTLSMetric(insecureSkipTLSVerifyBackend bool) {
+	usageType := usageEnforce
+	if insecureSkipTLSVerifyBackend {
+		if utilfeature.DefaultFeatureGate.Enabled(features.AllowInsecureBackendProxy) {
+			usageType = usageSkipAllowed
+		} else {
+			usageType = usageSkipDenied
+		}
+	}
+
+	counter, err := podLogsUsage.GetMetricWithLabelValues(usageType)
+	if err != nil {
+		utilruntime.HandleError(err)
+		return
+	}
+	counter.Inc()
+}
+
 // NewGetOptions creates a new options object
 func (r *LogREST) NewGetOptions() (runtime.Object, bool, string) {
 	return &api.PodLogOptions{}, false, ""

pkg/registry/core/pod/rest/metrics.go

@@ -0,0 +1,67 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rest
+
+import (
+	"sync"
+
+	"k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+)
+
+const (
+	namespace = "kube_apiserver"
+	subsystem = "pod_logs"
+
+	usageEnforce     = "enforce_tls"
+	usageSkipAllowed = "skip_tls_allowed"
+	usageSkipDenied  = "skip_tls_denied"
+)
+
+var (
+	// podLogsUsage counts and categorizes how the insecure backend skip TLS option is used and allowed.
+	podLogsUsage = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "pods_logs_insecure_backend_total",
+			Help:           "Total number of requests for pods/logs sliced by usage type: enforce_tls, skip_tls_allowed, skip_tls_denied",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"usage"},
+	)
+
+	// podLogsTLSFailure counts how many attempts to get pod logs fail on tls verification
+	podLogsTLSFailure = metrics.NewCounter(
+		&metrics.CounterOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "pods_logs_backend_tls_failure_total",
+			Help:           "Total number of requests for pods/logs that failed due to kubelet server TLS verification",
+			StabilityLevel: metrics.ALPHA,
+		},
+	)
+)
+
+var registerMetricsOnce sync.Once
+
+func registerMetrics() {
+	registerMetricsOnce.Do(func() {
+		legacyregistry.MustRegister(podLogsUsage)
+		legacyregistry.MustRegister(podLogsTLSFailure)
+	})
+}

staging/src/k8s.io/apiserver/pkg/registry/generic/rest/streamer.go

@@ -30,6 +30,10 @@ import (
 	"k8s.io/apiserver/pkg/registry/rest"
 )
 
+type CounterMetric interface {
+	Inc()
+}
+
 // LocationStreamer is a resource that streams the contents of a particular
 // location URL.
 type LocationStreamer struct {
@@ -39,6 +43,9 @@ type LocationStreamer struct {
 	Flush           bool
 	ResponseChecker HttpResponseChecker
 	RedirectChecker func(req *http.Request, via []*http.Request) error
+	// TLSVerificationErrorCounter is an optional value that will Inc every time a TLS error is encountered.  This can
+	// be wired a single prometheus counter instance to get counts overall.
+	TLSVerificationErrorCounter CounterMetric
 }
 
 // a LocationStreamer must implement a rest.ResourceStreamer
@@ -77,6 +84,10 @@ func (s *LocationStreamer) InputStream(ctx context.Context, apiVersion, acceptHe
 
 	resp, err := client.Do(req)
 	if err != nil {
+		// TODO prefer segregate TLS errors more reliably, but we do want to increment a count
+		if strings.Contains(err.Error(), "x509:") && s.TLSVerificationErrorCounter != nil {
+			s.TLSVerificationErrorCounter.Inc()
+		}
 		return nil, false, "", err
 	}