Merge pull request #85874 from sambdavidson/ttlFunc

Kubelet cert TTL via GaugeFunc

pkg/kubelet/certificate/kubelet.go

@@ -21,8 +21,10 @@ import (
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"fmt"
+	"math"
 	"net"
 	"sort"
+	"time"
 
 	certificates "k8s.io/api/certificates/v1beta1"
 	v1 "k8s.io/api/core/v1"
@@ -52,15 +54,6 @@ func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialize server certificate store: %v", err)
 	}
-	certificateExpiration := compbasemetrics.NewGauge(
-		&compbasemetrics.GaugeOpts{
-			Subsystem:      metrics.KubeletSubsystem,
-			Name:           "certificate_manager_server_expiration_seconds",
-			Help:           "Gauge of the lifetime of a certificate. The value is the date the certificate will expire in seconds since January 1, 1970 UTC.",
-			StabilityLevel: compbasemetrics.ALPHA,
-		},
-	)
-	legacyregistry.MustRegister(certificateExpiration)
 	var certificateRenewFailure = compbasemetrics.NewCounter(
 		&compbasemetrics.CounterOpts{
 			Subsystem:      metrics.KubeletSubsystem,
@@ -129,13 +122,30 @@ func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg
 			certificates.UsageServerAuth,
 		},
 		CertificateStore:        certificateStore,
-		CertificateExpiration:   certificateExpiration,
 		CertificateRotation:     certificateRotationAge,
 		CertificateRenewFailure: certificateRenewFailure,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialize server certificate manager: %v", err)
 	}
+	legacyregistry.RawMustRegister(compbasemetrics.NewGaugeFunc(
+		compbasemetrics.GaugeOpts{
+			Subsystem: metrics.KubeletSubsystem,
+			Name:      "certificate_manager_server_ttl_seconds",
+			Help: "Gauge of the shortest TTL (time-to-live) of " +
+				"the Kubelet's serving certificate. The value is in seconds " +
+				"until certificate expiry (negative if already expired). If " +
+				"serving certificate is invalid or unused, the value will " +
+				"be +INF.",
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		func() float64 {
+			if c := m.Current(); c != nil && c.Leaf != nil {
+				return c.Leaf.NotAfter.Sub(time.Now()).Seconds()
+			}
+			return math.Inf(1)
+		},
+	))
 	return m, nil
 }
 
@@ -252,7 +262,6 @@ func NewKubeletClientCertificateManager(
 		BootstrapKeyPEM:         bootstrapKeyData,
 
 		CertificateStore:        certificateStore,
-		CertificateExpiration:   certificateExpiration,
 		CertificateRenewFailure: certificateRenewFailure,
 	})
 	if err != nil {

staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/BUILD

@@ -41,7 +41,6 @@ go_test(
         "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//staging/src/k8s.io/client-go/pkg/apis/clientauthentication:go_default_library",
         "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library",
-        "//staging/src/k8s.io/client-go/tools/metrics:go_default_library",
         "//staging/src/k8s.io/client-go/transport:go_default_library",
     ],
 )

staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/exec.go

@@ -262,7 +262,6 @@ func (a *Authenticator) cert() (*tls.Certificate, error) {
 func (a *Authenticator) getCreds() (*credentials, error) {
 	a.mu.Lock()
 	defer a.mu.Unlock()
-	defer expirationMetrics.report(time.Now)
 
 	if a.cachedCreds != nil && !a.credsExpired() {
 		return a.cachedCreds, nil

staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/metrics.go

@@ -24,20 +24,25 @@ import (
 )
 
 type certificateExpirationTracker struct {
-	mu       sync.RWMutex
+	mu        sync.RWMutex
-	m        map[*Authenticator]time.Time
+	m         map[*Authenticator]time.Time
-	earliest time.Time
+	metricSet func(*time.Time)
 }
 
-var expirationMetrics = &certificateExpirationTracker{m: map[*Authenticator]time.Time{}}
+var expirationMetrics = &certificateExpirationTracker{
+	m: map[*Authenticator]time.Time{},
+	metricSet: func(e *time.Time) {
+		metrics.ClientCertExpiry.Set(e)
+	},
+}
 
-// set stores the given expiration time and updates the updates earliest.
+// set stores the given expiration time and updates the updates the certificate
+// expiry metric to the earliest expiration time.
 func (c *certificateExpirationTracker) set(a *Authenticator, t time.Time) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	c.m[a] = t
 
-	// update earliest
 	earliest := time.Time{}
 	for _, t := range c.m {
 		if t.IsZero() {
@@ -47,18 +52,9 @@ func (c *certificateExpirationTracker) set(a *Authenticator, t time.Time) {
 			earliest = t
 		}
 	}
-	c.earliest = earliest
+	if earliest.IsZero() {
-}
+		c.metricSet(nil)
-
-// report reports the ttl to the earliest reported expiration time.
-// If no Authenticators have reported a certificate expiration, this reports nil.
-func (c *certificateExpirationTracker) report(now func() time.Time) {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-	if c.earliest.IsZero() {
-		metrics.ClientCertTTL.Set(nil)
 	} else {
-		ttl := c.earliest.Sub(now())
+		c.metricSet(&earliest)
-		metrics.ClientCertTTL.Set(&ttl)
 	}
 }

staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/metrics_test.go

@@ -19,36 +19,27 @@ package exec
 import (
 	"testing"
 	"time"
-
-	"k8s.io/client-go/tools/metrics"
 )
 
-type mockTTLGauge struct {
+type mockExpiryGauge struct {
-	v *time.Duration
+	v *time.Time
 }
 
-func (m *mockTTLGauge) Set(d *time.Duration) {
+func (m *mockExpiryGauge) Set(t *time.Time) {
-	m.v = d
+	m.v = t
 }
 
-func ptr(d time.Duration) *time.Duration {
+func ptr(t time.Time) *time.Time {
-	return &d
+	return &t
 }
 
 func TestCertificateExpirationTracker(t *testing.T) {
 	now := time.Now()
-	nowFn := func() time.Time { return now }
+	mockMetric := &mockExpiryGauge{}
-	mockMetric := &mockTTLGauge{}
-	realMetric := metrics.ClientCertTTL
-	metrics.ClientCertTTL = mockMetric
-	defer func() {
-		metrics.ClientCertTTL = realMetric
-	}()
 
-	tracker := &certificateExpirationTracker{m: map[*Authenticator]time.Time{}}
+	tracker := &certificateExpirationTracker{
-	tracker.report(nowFn)
+		m:         map[*Authenticator]time.Time{},
-	if mockMetric.v != nil {
+		metricSet: mockMetric.Set,
-		t.Error("empty tracker should record nil value")
 	}
 
 	firstAuthenticator := &Authenticator{}
@@ -57,31 +48,31 @@ func TestCertificateExpirationTracker(t *testing.T) {
 		desc string
 		auth *Authenticator
 		time time.Time
-		want *time.Duration
+		want *time.Time
 	}{
 		{
 			desc: "ttl for one authenticator",
 			auth: firstAuthenticator,
 			time: now.Add(time.Minute * 10),
-			want: ptr(time.Minute * 10),
+			want: ptr(now.Add(time.Minute * 10)),
 		},
 		{
 			desc: "second authenticator shorter ttl",
 			auth: secondAuthenticator,
 			time: now.Add(time.Minute * 5),
-			want: ptr(time.Minute * 5),
+			want: ptr(now.Add(time.Minute * 5)),
 		},
 		{
 			desc: "update shorter to be longer",
 			auth: secondAuthenticator,
 			time: now.Add(time.Minute * 15),
-			want: ptr(time.Minute * 10),
+			want: ptr(now.Add(time.Minute * 10)),
 		},
 		{
 			desc: "update shorter to be zero time",
 			auth: firstAuthenticator,
 			time: time.Time{},
-			want: ptr(time.Minute * 15),
+			want: ptr(now.Add(time.Minute * 15)),
 		},
 		{
 			desc: "update last to be zero time records nil",
@@ -93,13 +84,12 @@ func TestCertificateExpirationTracker(t *testing.T) {
 		// Must run in series as the tests build off each other.
 		t.Run(tc.desc, func(t *testing.T) {
 			tracker.set(tc.auth, tc.time)
-			tracker.report(nowFn)
 			if mockMetric.v != nil && tc.want != nil {
-				if mockMetric.v.Seconds() != tc.want.Seconds() {
+				if !mockMetric.v.Equal(*tc.want) {
-					t.Errorf("got: %v; want: %v", mockMetric.v, tc.want)
+					t.Errorf("got: %s; want: %s", mockMetric.v, tc.want)
 				}
 			} else if mockMetric.v != tc.want {
-				t.Errorf("got: %v; want: %v", mockMetric.v, tc.want)
+				t.Errorf("got: %s; want: %s", mockMetric.v, tc.want)
 			}
 		})
 	}

staging/src/k8s.io/client-go/tools/metrics/metrics.go

@@ -31,9 +31,9 @@ type DurationMetric interface {
 	Observe(duration time.Duration)
 }
 
-// TTLMetric sets the time to live of something.
+// ExpiryMetric sets some time of expiry. If nil, assume not relevant.
-type TTLMetric interface {
+type ExpiryMetric interface {
-	Set(ttl *time.Duration)
+	Set(expiry *time.Time)
 }
 
 // LatencyMetric observes client latency partitioned by verb and url.
@@ -47,8 +47,8 @@ type ResultMetric interface {
 }
 
 var (
-	// ClientCertTTL is the time to live of a client certificate
+	// ClientCertExpiry is the expiry time of a client certificate
-	ClientCertTTL TTLMetric = noopTTL{}
+	ClientCertExpiry ExpiryMetric = noopExpiry{}
 	// ClientCertRotationAge is the age of a certificate that has just been rotated.
 	ClientCertRotationAge DurationMetric = noopDuration{}
 	// RequestLatency is the latency metric that rest clients will update.
@@ -59,7 +59,7 @@ var (
 
 // RegisterOpts contains all the metrics to register. Metrics may be nil.
 type RegisterOpts struct {
-	ClientCertTTL         TTLMetric
+	ClientCertExpiry      ExpiryMetric
 	ClientCertRotationAge DurationMetric
 	RequestLatency        LatencyMetric
 	RequestResult         ResultMetric
@@ -69,8 +69,8 @@ type RegisterOpts struct {
 // only be called once.
 func Register(opts RegisterOpts) {
 	registerMetrics.Do(func() {
-		if opts.ClientCertTTL != nil {
+		if opts.ClientCertExpiry != nil {
-			ClientCertTTL = opts.ClientCertTTL
+			ClientCertExpiry = opts.ClientCertExpiry
 		}
 		if opts.ClientCertRotationAge != nil {
 			ClientCertRotationAge = opts.ClientCertRotationAge
@@ -88,9 +88,9 @@ type noopDuration struct{}
 
 func (noopDuration) Observe(time.Duration) {}
 
-type noopTTL struct{}
+type noopExpiry struct{}
 
-func (noopTTL) Set(*time.Duration) {}
+func (noopExpiry) Set(*time.Time) {}
 
 type noopLatency struct{}
 

staging/src/k8s.io/client-go/util/certificate/certificate_manager.go

@@ -112,12 +112,6 @@ type Config struct {
 	// initialized using a generic, multi-use cert/key pair which will be
 	// quickly replaced with a unique cert/key pair.
 	BootstrapKeyPEM []byte
-	// CertificateExpiration will record a metric that shows the remaining
-	// lifetime of the certificate. This metric is a gauge because only the
-	// current cert expiry time is really useful. Reading this metric at any
-	// time simply gives the next expiration date, no need to keep some
-	// history (histogram) of all previous expiry dates.
-	CertificateExpiration Gauge
 	// CertificateRotation will record a metric showing the time in seconds
 	// that certificates lived before being rotated. This metric is a histogram
 	// because there is value in keeping a history of rotation cadences. It
@@ -185,7 +179,6 @@ type manager struct {
 
 	certStore Store
 
-	certificateExpiration   Gauge
 	certificateRotation     Histogram
 	certificateRenewFailure Counter
 
@@ -230,7 +223,6 @@ func NewManager(config *Config) (Manager, error) {
 		certStore:               config.CertificateStore,
 		cert:                    cert,
 		forceRotation:           forceRotation,
-		certificateExpiration:   config.CertificateExpiration,
 		certificateRotation:     config.CertificateRotation,
 		certificateRenewFailure: config.CertificateRenewFailure,
 		now:                     time.Now,
@@ -554,9 +546,6 @@ func (m *manager) nextRotationDeadline() time.Time {
 	deadline := m.cert.Leaf.NotBefore.Add(jitteryDuration(totalDuration))
 
 	klog.V(2).Infof("Certificate expiration is %v, rotation deadline is %v", notAfter, deadline)
-	if m.certificateExpiration != nil {
-		m.certificateExpiration.Set(float64(notAfter.Unix()))
-	}
 	return deadline
 }
 

staging/src/k8s.io/client-go/util/certificate/certificate_manager_test.go

@@ -200,7 +200,6 @@ func TestSetRotationDeadline(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			g := metricMock{}
 			m := manager{
 				cert: &tls.Certificate{
 					Leaf: &x509.Certificate{
@@ -208,10 +207,9 @@ func TestSetRotationDeadline(t *testing.T) {
 						NotAfter:  tc.notAfter,
 					},
 				},
-				getTemplate:           func() *x509.CertificateRequest { return &x509.CertificateRequest{} },
+				getTemplate: func() *x509.CertificateRequest { return &x509.CertificateRequest{} },
-				usages:                []certificates.KeyUsage{},
+				usages:      []certificates.KeyUsage{},
-				certificateExpiration: &g,
+				now:         func() time.Time { return now },
-				now:                   func() time.Time { return now },
 			}
 			jitteryDuration = func(float64) time.Duration { return time.Duration(float64(tc.notAfter.Sub(tc.notBefore)) * 0.7) }
 			lowerBound := tc.notBefore.Add(time.Duration(float64(tc.notAfter.Sub(tc.notBefore)) * 0.7))
@@ -225,12 +223,6 @@ func TestSetRotationDeadline(t *testing.T) {
 					deadline,
 					lowerBound)
 			}
-			if g.calls != 1 {
-				t.Errorf("%d metrics were recorded, wanted %d", g.calls, 1)
-			}
-			if g.lastValue != float64(tc.notAfter.Unix()) {
-				t.Errorf("%f value for metric was recorded, wanted %d", g.lastValue, tc.notAfter.Unix())
-			}
 		})
 	}
 }

staging/src/k8s.io/component-base/metrics/prometheus/restclient/metrics.go

@@ -57,14 +57,23 @@ var (
 		[]string{"code", "method", "host"},
 	)
 
-	execPluginCertTTL = k8smetrics.NewGauge(
+	execPluginCertTTLAdapter = &expiryToTTLAdapter{}
-		&k8smetrics.GaugeOpts{
+
+	execPluginCertTTL = k8smetrics.NewGaugeFunc(
+		k8smetrics.GaugeOpts{
 			Name: "rest_client_exec_plugin_ttl_seconds",
 			Help: "Gauge of the shortest TTL (time-to-live) of the client " +
 				"certificate(s) managed by the auth exec plugin. The value " +
-				"is in seconds until certificate expiry. If auth exec " +
+				"is in seconds until certificate expiry (negative if " +
-				"plugins are unused or manage no TLS certificates, the " +
+				"already expired). If auth exec plugins are unused or manage no " +
-				"value will be +INF.",
+				"TLS certificates, the value will be +INF.",
+			StabilityLevel: k8smetrics.ALPHA,
+		},
+		func() float64 {
+			if execPluginCertTTLAdapter.e == nil {
+				return math.Inf(1)
+			}
+			return execPluginCertTTLAdapter.e.Sub(time.Now()).Seconds()
 		},
 	)
 
@@ -99,15 +108,14 @@ var (
 )
 
 func init() {
-	execPluginCertTTL.Set(math.Inf(1)) // Initialize TTL to +INF
 
 	legacyregistry.MustRegister(requestLatency)
 	legacyregistry.MustRegister(deprecatedRequestLatency)
 	legacyregistry.MustRegister(requestResult)
-	legacyregistry.MustRegister(execPluginCertTTL)
+	legacyregistry.RawMustRegister(execPluginCertTTL)
 	legacyregistry.MustRegister(execPluginCertRotation)
 	metrics.Register(metrics.RegisterOpts{
-		ClientCertTTL:         &ttlAdapter{m: execPluginCertTTL},
+		ClientCertExpiry:      execPluginCertTTLAdapter,
 		ClientCertRotationAge: &rotationAdapter{m: execPluginCertRotation},
 		RequestLatency:        &latencyAdapter{m: requestLatency, dm: deprecatedRequestLatency},
 		RequestResult:         &resultAdapter{requestResult},
@@ -132,16 +140,12 @@ func (r *resultAdapter) Increment(code, method, host string) {
 	r.m.WithLabelValues(code, method, host).Inc()
 }
 
-type ttlAdapter struct {
+type expiryToTTLAdapter struct {
-	m *k8smetrics.Gauge
+	e *time.Time
 }
 
-func (e *ttlAdapter) Set(ttl *time.Duration) {
+func (e *expiryToTTLAdapter) Set(expiry *time.Time) {
-	if ttl == nil {
+	e.e = expiry
-		e.m.Set(math.Inf(1))
-	} else {
-		e.m.Set(float64(ttl.Seconds()))
-	}
 }
 
 type rotationAdapter struct {