Added rest client metrics for client TTL and rot. (#84382)

* Added rest client metrics for client TTL and rot.

* Fixed foo bar comment, added nil checks

* Moved rotation observation inside of old cert nil check block

* Fixed rotation age logic.

* fixed BUILD for exec plugin package

* fixed null pointer dereference in exec.go

* Updated metric name, bucket, used oldest cert.

Kubernetes-commit: 9dcb3bfcff2024e4690d70262cad14668b1f2507

plugin/pkg/client/auth/exec/exec.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"context"
 	"crypto/tls"
+	"crypto/x509"
 	"errors"
 	"fmt"
 	"io"
@@ -42,6 +43,7 @@ import (
 	"k8s.io/client-go/pkg/apis/clientauthentication/v1alpha1"
 	"k8s.io/client-go/pkg/apis/clientauthentication/v1beta1"
 	"k8s.io/client-go/tools/clientcmd/api"
+	"k8s.io/client-go/tools/metrics"
 	"k8s.io/client-go/transport"
 	"k8s.io/client-go/util/connrotation"
 	"k8s.io/klog"
@@ -260,6 +262,8 @@ func (a *Authenticator) cert() (*tls.Certificate, error) {
 func (a *Authenticator) getCreds() (*credentials, error) {
 	a.mu.Lock()
 	defer a.mu.Unlock()
+	defer expirationMetrics.report(time.Now)
+
 	if a.cachedCreds != nil && !a.credsExpired() {
 		return a.cachedCreds, nil
 	}
@@ -267,6 +271,7 @@ func (a *Authenticator) getCreds() (*credentials, error) {
 	if err := a.refreshCredsLocked(nil); err != nil {
 		return nil, err
 	}
+
 	return a.cachedCreds, nil
 }
 
@@ -355,6 +360,17 @@ func (a *Authenticator) refreshCredsLocked(r *clientauthentication.Response) err
 		if err != nil {
 			return fmt.Errorf("failed parsing client key/certificate: %v", err)
 		}
+
+		// Leaf is initialized to be nil:
+		//  https://golang.org/pkg/crypto/tls/#X509KeyPair
+		// Leaf certificate is the first certificate:
+		//  https://golang.org/pkg/crypto/tls/#Certificate
+		// Populating leaf is useful for quickly accessing the underlying x509
+		// certificate values.
+		cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])
+		if err != nil {
+			return fmt.Errorf("failed parsing client leaf certificate: %v", err)
+		}
 		newCreds.cert = &cert
 	}
 
@@ -362,10 +378,20 @@ func (a *Authenticator) refreshCredsLocked(r *clientauthentication.Response) err
 	a.cachedCreds = newCreds
 	// Only close all connections when TLS cert rotates. Token rotation doesn't
 	// need the extra noise.
-	if len(a.onRotateList) > 0 && oldCreds != nil && !reflect.DeepEqual(oldCreds.cert, a.cachedCreds.cert) {
+	if oldCreds != nil && !reflect.DeepEqual(oldCreds.cert, a.cachedCreds.cert) {
+		// Can be nil if the exec auth plugin only returned token auth.
+		if oldCreds.cert != nil && oldCreds.cert.Leaf != nil {
+			metrics.ClientCertRotationAge.Observe(time.Now().Sub(oldCreds.cert.Leaf.NotBefore))
+		}
 		for _, onRotate := range a.onRotateList {
 			onRotate()
 		}
 	}
+
+	expiry := time.Time{}
+	if a.cachedCreds.cert != nil && a.cachedCreds.cert.Leaf != nil {
+		expiry = a.cachedCreds.cert.Leaf.NotAfter
+	}
+	expirationMetrics.set(a, expiry)
 	return nil
 }

plugin/pkg/client/auth/exec/exec_test.go

@@ -97,6 +97,10 @@ func init() {
 	if err != nil {
 		panic(err)
 	}
+	cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])
+	if err != nil {
+		panic(err)
+	}
 	validCert = &cert
 }
 
@@ -760,7 +764,7 @@ func TestConcurrentUpdateTransportConfig(t *testing.T) {
 }
 
 // genClientCert generates an x509 certificate for testing. Certificate and key
-// are returned in PEM encoding.
+// are returned in PEM encoding. The generated cert expires in 24 hours.
 func genClientCert(t *testing.T) ([]byte, []byte) {
 	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
 	if err != nil {

plugin/pkg/client/auth/exec/metrics.go

@@ -0,0 +1,64 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package exec
+
+import (
+	"sync"
+	"time"
+
+	"k8s.io/client-go/tools/metrics"
+)
+
+type certificateExpirationTracker struct {
+	mu       sync.RWMutex
+	m        map[*Authenticator]time.Time
+	earliest time.Time
+}
+
+var expirationMetrics = &certificateExpirationTracker{m: map[*Authenticator]time.Time{}}
+
+// set stores the given expiration time and updates the updates earliest.
+func (c *certificateExpirationTracker) set(a *Authenticator, t time.Time) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.m[a] = t
+
+	// update earliest
+	earliest := time.Time{}
+	for _, t := range c.m {
+		if t.IsZero() {
+			continue
+		}
+		if earliest.IsZero() || earliest.After(t) {
+			earliest = t
+		}
+	}
+	c.earliest = earliest
+}
+
+// report reports the ttl to the earliest reported expiration time.
+// If no Authenticators have reported a certificate expiration, this reports nil.
+func (c *certificateExpirationTracker) report(now func() time.Time) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	if c.earliest.IsZero() {
+		metrics.ClientCertTTL.Set(nil)
+	} else {
+		ttl := c.earliest.Sub(now())
+		metrics.ClientCertTTL.Set(&ttl)
+	}
+}

plugin/pkg/client/auth/exec/metrics_test.go

@@ -0,0 +1,106 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package exec
+
+import (
+	"testing"
+	"time"
+
+	"k8s.io/client-go/tools/metrics"
+)
+
+type mockTTLGauge struct {
+	v *time.Duration
+}
+
+func (m *mockTTLGauge) Set(d *time.Duration) {
+	m.v = d
+}
+
+func ptr(d time.Duration) *time.Duration {
+	return &d
+}
+
+func TestCertificateExpirationTracker(t *testing.T) {
+	now := time.Now()
+	nowFn := func() time.Time { return now }
+	mockMetric := &mockTTLGauge{}
+	realMetric := metrics.ClientCertTTL
+	metrics.ClientCertTTL = mockMetric
+	defer func() {
+		metrics.ClientCertTTL = realMetric
+	}()
+
+	tracker := &certificateExpirationTracker{m: map[*Authenticator]time.Time{}}
+	tracker.report(nowFn)
+	if mockMetric.v != nil {
+		t.Error("empty tracker should record nil value")
+	}
+
+	firstAuthenticator := &Authenticator{}
+	secondAuthenticator := &Authenticator{}
+	for _, tc := range []struct {
+		desc string
+		auth *Authenticator
+		time time.Time
+		want *time.Duration
+	}{
+		{
+			desc: "ttl for one authenticator",
+			auth: firstAuthenticator,
+			time: now.Add(time.Minute * 10),
+			want: ptr(time.Minute * 10),
+		},
+		{
+			desc: "second authenticator shorter ttl",
+			auth: secondAuthenticator,
+			time: now.Add(time.Minute * 5),
+			want: ptr(time.Minute * 5),
+		},
+		{
+			desc: "update shorter to be longer",
+			auth: secondAuthenticator,
+			time: now.Add(time.Minute * 15),
+			want: ptr(time.Minute * 10),
+		},
+		{
+			desc: "update shorter to be zero time",
+			auth: firstAuthenticator,
+			time: time.Time{},
+			want: ptr(time.Minute * 15),
+		},
+		{
+			desc: "update last to be zero time records nil",
+			auth: secondAuthenticator,
+			time: time.Time{},
+			want: nil,
+		},
+	} {
+		// Must run in series as the tests build off each other.
+		t.Run(tc.desc, func(t *testing.T) {
+			tracker.set(tc.auth, tc.time)
+			tracker.report(nowFn)
+			if mockMetric.v != nil && tc.want != nil {
+				if mockMetric.v.Seconds() != tc.want.Seconds() {
+					t.Errorf("got: %v; want: %v", mockMetric.v, tc.want)
+				}
+			} else if mockMetric.v != tc.want {
+				t.Errorf("got: %v; want: %v", mockMetric.v, tc.want)
+			}
+		})
+	}
+}

tools/metrics/metrics.go

@@ -26,6 +26,16 @@ import (
 
 var registerMetrics sync.Once
 
+// DurationMetric is a measurement of some amount of time.
+type DurationMetric interface {
+	Observe(duration time.Duration)
+}
+
+// TTLMetric sets the time to live of something.
+type TTLMetric interface {
+	Set(ttl *time.Duration)
+}
+
 // LatencyMetric observes client latency partitioned by verb and url.
 type LatencyMetric interface {
 	Observe(verb string, u url.URL, latency time.Duration)
@@ -37,21 +47,51 @@ type ResultMetric interface {
 }
 
 var (
+	// ClientCertTTL is the time to live of a client certificate
+	ClientCertTTL TTLMetric = noopTTL{}
+	// ClientCertRotationAge is the age of a certificate that has just been rotated.
+	ClientCertRotationAge DurationMetric = noopDuration{}
 	// RequestLatency is the latency metric that rest clients will update.
 	RequestLatency LatencyMetric = noopLatency{}
 	// RequestResult is the result metric that rest clients will update.
 	RequestResult ResultMetric = noopResult{}
 )
 
+// RegisterOpts contains all the metrics to register. Metrics may be nil.
+type RegisterOpts struct {
+	ClientCertTTL         TTLMetric
+	ClientCertRotationAge DurationMetric
+	RequestLatency        LatencyMetric
+	RequestResult         ResultMetric
+}
+
 // Register registers metrics for the rest client to use. This can
 // only be called once.
-func Register(lm LatencyMetric, rm ResultMetric) {
+func Register(opts RegisterOpts) {
 	registerMetrics.Do(func() {
-		RequestLatency = lm
+		if opts.ClientCertTTL != nil {
-		RequestResult = rm
+			ClientCertTTL = opts.ClientCertTTL
+		}
+		if opts.ClientCertRotationAge != nil {
+			ClientCertRotationAge = opts.ClientCertRotationAge
+		}
+		if opts.RequestLatency != nil {
+			RequestLatency = opts.RequestLatency
+		}
+		if opts.RequestResult != nil {
+			RequestResult = opts.RequestResult
+		}
 	})
 }
 
+type noopDuration struct{}
+
+func (noopDuration) Observe(time.Duration) {}
+
+type noopTTL struct{}
+
+func (noopTTL) Set(*time.Duration) {}
+
 type noopLatency struct{}
 
 func (noopLatency) Observe(string, url.URL, time.Duration) {}