Added rest client metrics for client TTL and rot. (#84382)

* Added rest client metrics for client TTL and rot.

* Fixed foo bar comment, added nil checks

* Moved rotation observation inside of old cert nil check block

* Fixed rotation age logic.

* fixed BUILD for exec plugin package

* fixed null pointer dereference in exec.go

* Updated metric name, bucket, used oldest cert.

staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/BUILD

@@ -2,7 +2,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
-    srcs = ["exec.go"],
+    srcs = [
+        "exec.go",
+        "metrics.go",
+    ],
     importmap = "k8s.io/kubernetes/vendor/k8s.io/client-go/plugin/pkg/client/auth/exec",
     importpath = "k8s.io/client-go/plugin/pkg/client/auth/exec",
     visibility = ["//visibility:public"],
@@ -16,6 +19,7 @@ go_library(
         "//staging/src/k8s.io/client-go/pkg/apis/clientauthentication/v1alpha1:go_default_library",
         "//staging/src/k8s.io/client-go/pkg/apis/clientauthentication/v1beta1:go_default_library",
         "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/metrics:go_default_library",
         "//staging/src/k8s.io/client-go/transport:go_default_library",
         "//staging/src/k8s.io/client-go/util/connrotation:go_default_library",
         "//vendor/github.com/davecgh/go-spew/spew:go_default_library",
@@ -26,7 +30,10 @@ go_library(
 
 go_test(
     name = "go_default_test",
-    srcs = ["exec_test.go"],
+    srcs = [
+        "exec_test.go",
+        "metrics_test.go",
+    ],
     data = glob(["testdata/**"]),
     embed = [":go_default_library"],
     deps = [
@@ -34,6 +41,7 @@ go_test(
         "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//staging/src/k8s.io/client-go/pkg/apis/clientauthentication:go_default_library",
         "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/metrics:go_default_library",
         "//staging/src/k8s.io/client-go/transport:go_default_library",
     ],
 )

staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/exec.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"context"
 	"crypto/tls"
+	"crypto/x509"
 	"errors"
 	"fmt"
 	"io"
@@ -42,6 +43,7 @@ import (
 	"k8s.io/client-go/pkg/apis/clientauthentication/v1alpha1"
 	"k8s.io/client-go/pkg/apis/clientauthentication/v1beta1"
 	"k8s.io/client-go/tools/clientcmd/api"
+	"k8s.io/client-go/tools/metrics"
 	"k8s.io/client-go/transport"
 	"k8s.io/client-go/util/connrotation"
 	"k8s.io/klog"
@@ -260,6 +262,8 @@ func (a *Authenticator) cert() (*tls.Certificate, error) {
 func (a *Authenticator) getCreds() (*credentials, error) {
 	a.mu.Lock()
 	defer a.mu.Unlock()
+	defer expirationMetrics.report(time.Now)
+
 	if a.cachedCreds != nil && !a.credsExpired() {
 		return a.cachedCreds, nil
 	}
@@ -267,6 +271,7 @@ func (a *Authenticator) getCreds() (*credentials, error) {
 	if err := a.refreshCredsLocked(nil); err != nil {
 		return nil, err
 	}
+
 	return a.cachedCreds, nil
 }
 
@@ -355,6 +360,17 @@ func (a *Authenticator) refreshCredsLocked(r *clientauthentication.Response) err
 		if err != nil {
 			return fmt.Errorf("failed parsing client key/certificate: %v", err)
 		}
+
+		// Leaf is initialized to be nil:
+		//  https://golang.org/pkg/crypto/tls/#X509KeyPair
+		// Leaf certificate is the first certificate:
+		//  https://golang.org/pkg/crypto/tls/#Certificate
+		// Populating leaf is useful for quickly accessing the underlying x509
+		// certificate values.
+		cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])
+		if err != nil {
+			return fmt.Errorf("failed parsing client leaf certificate: %v", err)
+		}
 		newCreds.cert = &cert
 	}
 
@@ -362,10 +378,20 @@ func (a *Authenticator) refreshCredsLocked(r *clientauthentication.Response) err
 	a.cachedCreds = newCreds
 	// Only close all connections when TLS cert rotates. Token rotation doesn't
 	// need the extra noise.
-	if len(a.onRotateList) > 0 && oldCreds != nil && !reflect.DeepEqual(oldCreds.cert, a.cachedCreds.cert) {
+	if oldCreds != nil && !reflect.DeepEqual(oldCreds.cert, a.cachedCreds.cert) {
+		// Can be nil if the exec auth plugin only returned token auth.
+		if oldCreds.cert != nil && oldCreds.cert.Leaf != nil {
+			metrics.ClientCertRotationAge.Observe(time.Now().Sub(oldCreds.cert.Leaf.NotBefore))
+		}
 		for _, onRotate := range a.onRotateList {
 			onRotate()
 		}
 	}
+
+	expiry := time.Time{}
+	if a.cachedCreds.cert != nil && a.cachedCreds.cert.Leaf != nil {
+		expiry = a.cachedCreds.cert.Leaf.NotAfter
+	}
+	expirationMetrics.set(a, expiry)
 	return nil
 }

staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/exec_test.go

@@ -97,6 +97,10 @@ func init() {
 	if err != nil {
 		panic(err)
 	}
+	cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])
+	if err != nil {
+		panic(err)
+	}
 	validCert = &cert
 }
 
@@ -760,7 +764,7 @@ func TestConcurrentUpdateTransportConfig(t *testing.T) {
 }
 
 // genClientCert generates an x509 certificate for testing. Certificate and key
-// are returned in PEM encoding.
+// are returned in PEM encoding. The generated cert expires in 24 hours.
 func genClientCert(t *testing.T) ([]byte, []byte) {
 	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
 	if err != nil {

staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/metrics.go

@@ -0,0 +1,64 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package exec
+
+import (
+	"sync"
+	"time"
+
+	"k8s.io/client-go/tools/metrics"
+)
+
+type certificateExpirationTracker struct {
+	mu       sync.RWMutex
+	m        map[*Authenticator]time.Time
+	earliest time.Time
+}
+
+var expirationMetrics = &certificateExpirationTracker{m: map[*Authenticator]time.Time{}}
+
+// set stores the given expiration time and updates the updates earliest.
+func (c *certificateExpirationTracker) set(a *Authenticator, t time.Time) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.m[a] = t
+
+	// update earliest
+	earliest := time.Time{}
+	for _, t := range c.m {
+		if t.IsZero() {
+			continue
+		}
+		if earliest.IsZero() || earliest.After(t) {
+			earliest = t
+		}
+	}
+	c.earliest = earliest
+}
+
+// report reports the ttl to the earliest reported expiration time.
+// If no Authenticators have reported a certificate expiration, this reports nil.
+func (c *certificateExpirationTracker) report(now func() time.Time) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	if c.earliest.IsZero() {
+		metrics.ClientCertTTL.Set(nil)
+	} else {
+		ttl := c.earliest.Sub(now())
+		metrics.ClientCertTTL.Set(&ttl)
+	}
+}

staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/metrics_test.go

@@ -0,0 +1,106 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package exec
+
+import (
+	"testing"
+	"time"
+
+	"k8s.io/client-go/tools/metrics"
+)
+
+type mockTTLGauge struct {
+	v *time.Duration
+}
+
+func (m *mockTTLGauge) Set(d *time.Duration) {
+	m.v = d
+}
+
+func ptr(d time.Duration) *time.Duration {
+	return &d
+}
+
+func TestCertificateExpirationTracker(t *testing.T) {
+	now := time.Now()
+	nowFn := func() time.Time { return now }
+	mockMetric := &mockTTLGauge{}
+	realMetric := metrics.ClientCertTTL
+	metrics.ClientCertTTL = mockMetric
+	defer func() {
+		metrics.ClientCertTTL = realMetric
+	}()
+
+	tracker := &certificateExpirationTracker{m: map[*Authenticator]time.Time{}}
+	tracker.report(nowFn)
+	if mockMetric.v != nil {
+		t.Error("empty tracker should record nil value")
+	}
+
+	firstAuthenticator := &Authenticator{}
+	secondAuthenticator := &Authenticator{}
+	for _, tc := range []struct {
+		desc string
+		auth *Authenticator
+		time time.Time
+		want *time.Duration
+	}{
+		{
+			desc: "ttl for one authenticator",
+			auth: firstAuthenticator,
+			time: now.Add(time.Minute * 10),
+			want: ptr(time.Minute * 10),
+		},
+		{
+			desc: "second authenticator shorter ttl",
+			auth: secondAuthenticator,
+			time: now.Add(time.Minute * 5),
+			want: ptr(time.Minute * 5),
+		},
+		{
+			desc: "update shorter to be longer",
+			auth: secondAuthenticator,
+			time: now.Add(time.Minute * 15),
+			want: ptr(time.Minute * 10),
+		},
+		{
+			desc: "update shorter to be zero time",
+			auth: firstAuthenticator,
+			time: time.Time{},
+			want: ptr(time.Minute * 15),
+		},
+		{
+			desc: "update last to be zero time records nil",
+			auth: secondAuthenticator,
+			time: time.Time{},
+			want: nil,
+		},
+	} {
+		// Must run in series as the tests build off each other.
+		t.Run(tc.desc, func(t *testing.T) {
+			tracker.set(tc.auth, tc.time)
+			tracker.report(nowFn)
+			if mockMetric.v != nil && tc.want != nil {
+				if mockMetric.v.Seconds() != tc.want.Seconds() {
+					t.Errorf("got: %v; want: %v", mockMetric.v, tc.want)
+				}
+			} else if mockMetric.v != tc.want {
+				t.Errorf("got: %v; want: %v", mockMetric.v, tc.want)
+			}
+		})
+	}
+}

staging/src/k8s.io/client-go/tools/metrics/metrics.go

@@ -26,6 +26,16 @@ import (
 
 var registerMetrics sync.Once
 
+// DurationMetric is a measurement of some amount of time.
+type DurationMetric interface {
+	Observe(duration time.Duration)
+}
+
+// TTLMetric sets the time to live of something.
+type TTLMetric interface {
+	Set(ttl *time.Duration)
+}
+
 // LatencyMetric observes client latency partitioned by verb and url.
 type LatencyMetric interface {
 	Observe(verb string, u url.URL, latency time.Duration)
@@ -37,21 +47,51 @@ type ResultMetric interface {
 }
 
 var (
+	// ClientCertTTL is the time to live of a client certificate
+	ClientCertTTL TTLMetric = noopTTL{}
+	// ClientCertRotationAge is the age of a certificate that has just been rotated.
+	ClientCertRotationAge DurationMetric = noopDuration{}
 	// RequestLatency is the latency metric that rest clients will update.
 	RequestLatency LatencyMetric = noopLatency{}
 	// RequestResult is the result metric that rest clients will update.
 	RequestResult ResultMetric = noopResult{}
 )
 
+// RegisterOpts contains all the metrics to register. Metrics may be nil.
+type RegisterOpts struct {
+	ClientCertTTL         TTLMetric
+	ClientCertRotationAge DurationMetric
+	RequestLatency        LatencyMetric
+	RequestResult         ResultMetric
+}
+
 // Register registers metrics for the rest client to use. This can
 // only be called once.
-func Register(lm LatencyMetric, rm ResultMetric) {
+func Register(opts RegisterOpts) {
 	registerMetrics.Do(func() {
-		RequestLatency = lm
+		if opts.ClientCertTTL != nil {
-		RequestResult = rm
+			ClientCertTTL = opts.ClientCertTTL
+		}
+		if opts.ClientCertRotationAge != nil {
+			ClientCertRotationAge = opts.ClientCertRotationAge
+		}
+		if opts.RequestLatency != nil {
+			RequestLatency = opts.RequestLatency
+		}
+		if opts.RequestResult != nil {
+			RequestResult = opts.RequestResult
+		}
 	})
 }
 
+type noopDuration struct{}
+
+func (noopDuration) Observe(time.Duration) {}
+
+type noopTTL struct{}
+
+func (noopTTL) Set(*time.Duration) {}
+
 type noopLatency struct{}
 
 func (noopLatency) Observe(string, url.URL, time.Duration) {}

staging/src/k8s.io/component-base/metrics/prometheus/restclient/metrics.go

@@ -17,6 +17,7 @@ limitations under the License.
 package restclient
 
 import (
+	"math"
 	"net/url"
 	"time"
 
@@ -55,13 +56,62 @@ var (
 		},
 		[]string{"code", "method", "host"},
 	)
+
+	execPluginCertTTL = k8smetrics.NewGauge(
+		&k8smetrics.GaugeOpts{
+			Name: "rest_client_exec_plugin_ttl_seconds",
+			Help: "Gauge of the shortest TTL (time-to-live) of the client " +
+				"certificate(s) managed by the auth exec plugin. The value " +
+				"is in seconds until certificate expiry. If auth exec " +
+				"plugins are unused or manage no TLS certificates, the " +
+				"value will be +INF.",
+		},
+	)
+
+	execPluginCertRotation = k8smetrics.NewHistogram(
+		&k8smetrics.HistogramOpts{
+			Name: "rest_client_exec_plugin_certificate_rotation_age",
+			Help: "Histogram of the number of seconds the last auth exec " +
+				"plugin client certificate lived before being rotated. " +
+				"If auth exec plugin client certificates are unused, " +
+				"histogram will contain no data.",
+			// There are three sets of ranges these buckets intend to capture:
+			//   - 10-60 minutes: captures a rotation cadence which is
+			//     happening too quickly.
+			//   - 4 hours - 1 month: captures an ideal rotation cadence.
+			//   - 3 months - 4 years: captures a rotation cadence which is
+			//     is probably too slow or much too slow.
+			Buckets: []float64{
+				600,       // 10 minutes
+				1800,      // 30 minutes
+				3600,      // 1  hour
+				14400,     // 4  hours
+				86400,     // 1  day
+				604800,    // 1  week
+				2592000,   // 1  month
+				7776000,   // 3  months
+				15552000,  // 6  months
+				31104000,  // 1  year
+				124416000, // 4  years
+			},
+		},
+	)
 )
 
 func init() {
+	execPluginCertTTL.Set(math.Inf(1)) // Initialize TTL to +INF
+
 	legacyregistry.MustRegister(requestLatency)
 	legacyregistry.MustRegister(deprecatedRequestLatency)
 	legacyregistry.MustRegister(requestResult)
-	metrics.Register(&latencyAdapter{m: requestLatency, dm: deprecatedRequestLatency}, &resultAdapter{requestResult})
+	legacyregistry.MustRegister(execPluginCertTTL)
+	legacyregistry.MustRegister(execPluginCertRotation)
+	metrics.Register(metrics.RegisterOpts{
+		ClientCertTTL:         &ttlAdapter{m: execPluginCertTTL},
+		ClientCertRotationAge: &rotationAdapter{m: execPluginCertRotation},
+		RequestLatency:        &latencyAdapter{m: requestLatency, dm: deprecatedRequestLatency},
+		RequestResult:         &resultAdapter{requestResult},
+	})
 }
 
 type latencyAdapter struct {
@@ -81,3 +131,23 @@ type resultAdapter struct {
 func (r *resultAdapter) Increment(code, method, host string) {
 	r.m.WithLabelValues(code, method, host).Inc()
 }
+
+type ttlAdapter struct {
+	m *k8smetrics.Gauge
+}
+
+func (e *ttlAdapter) Set(ttl *time.Duration) {
+	if ttl == nil {
+		e.m.Set(math.Inf(1))
+	} else {
+		e.m.Set(float64(ttl.Seconds()))
+	}
+}
+
+type rotationAdapter struct {
+	m *k8smetrics.Histogram
+}
+
+func (r *rotationAdapter) Observe(d time.Duration) {
+	r.m.Observe(d.Seconds())
+}