diff --git a/pkg/kubelet/certificate/kubelet.go b/pkg/kubelet/certificate/kubelet.go index 414fb2e54ac..667285774fd 100644 --- a/pkg/kubelet/certificate/kubelet.go +++ b/pkg/kubelet/certificate/kubelet.go @@ -21,8 +21,10 @@ import ( "crypto/x509" "crypto/x509/pkix" "fmt" + "math" "net" "sort" + "time" certificates "k8s.io/api/certificates/v1beta1" v1 "k8s.io/api/core/v1" @@ -52,15 +54,6 @@ func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg if err != nil { return nil, fmt.Errorf("failed to initialize server certificate store: %v", err) } - certificateExpiration := compbasemetrics.NewGauge( - &compbasemetrics.GaugeOpts{ - Subsystem: metrics.KubeletSubsystem, - Name: "certificate_manager_server_expiration_seconds", - Help: "Gauge of the lifetime of a certificate. The value is the date the certificate will expire in seconds since January 1, 1970 UTC.", - StabilityLevel: compbasemetrics.ALPHA, - }, - ) - legacyregistry.MustRegister(certificateExpiration) var certificateRenewFailure = compbasemetrics.NewCounter( &compbasemetrics.CounterOpts{ Subsystem: metrics.KubeletSubsystem, @@ -129,13 +122,30 @@ func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg certificates.UsageServerAuth, }, CertificateStore: certificateStore, - CertificateExpiration: certificateExpiration, CertificateRotation: certificateRotationAge, CertificateRenewFailure: certificateRenewFailure, }) if err != nil { return nil, fmt.Errorf("failed to initialize server certificate manager: %v", err) } + legacyregistry.RawMustRegister(compbasemetrics.NewGaugeFunc( + compbasemetrics.GaugeOpts{ + Subsystem: metrics.KubeletSubsystem, + Name: "certificate_manager_server_ttl_seconds", + Help: "Gauge of the shortest TTL (time-to-live) of " + + "the Kubelet's serving certificate. The value is in seconds " + + "until certificate expiry (negative if already expired). If " + + "serving certificate is invalid or unused, the value will " + + "be +INF.", + StabilityLevel: compbasemetrics.ALPHA, + }, + func() float64 { + if c := m.Current(); c != nil && c.Leaf != nil { + return c.Leaf.NotAfter.Sub(time.Now()).Seconds() + } + return math.Inf(1) + }, + )) return m, nil } @@ -252,7 +262,6 @@ func NewKubeletClientCertificateManager( BootstrapKeyPEM: bootstrapKeyData, CertificateStore: certificateStore, - CertificateExpiration: certificateExpiration, CertificateRenewFailure: certificateRenewFailure, }) if err != nil { diff --git a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/BUILD b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/BUILD index a34513df7f5..82676fdebbf 100644 --- a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/BUILD +++ b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/BUILD @@ -41,7 +41,6 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/client-go/pkg/apis/clientauthentication:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", - "//staging/src/k8s.io/client-go/tools/metrics:go_default_library", "//staging/src/k8s.io/client-go/transport:go_default_library", ], ) diff --git a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/exec.go b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/exec.go index 9f029ee0a97..71ed045acd9 100644 --- a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/exec.go +++ b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/exec.go @@ -262,7 +262,6 @@ func (a *Authenticator) cert() (*tls.Certificate, error) { func (a *Authenticator) getCreds() (*credentials, error) { a.mu.Lock() defer a.mu.Unlock() - defer expirationMetrics.report(time.Now) if a.cachedCreds != nil && !a.credsExpired() { return a.cachedCreds, nil diff --git a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/metrics.go b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/metrics.go index 6ec6556c073..caf0cca3e43 100644 --- a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/metrics.go +++ b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/metrics.go @@ -24,20 +24,25 @@ import ( ) type certificateExpirationTracker struct { - mu sync.RWMutex - m map[*Authenticator]time.Time - earliest time.Time + mu sync.RWMutex + m map[*Authenticator]time.Time + metricSet func(*time.Time) } -var expirationMetrics = &certificateExpirationTracker{m: map[*Authenticator]time.Time{}} +var expirationMetrics = &certificateExpirationTracker{ + m: map[*Authenticator]time.Time{}, + metricSet: func(e *time.Time) { + metrics.ClientCertExpiry.Set(e) + }, +} -// set stores the given expiration time and updates the updates earliest. +// set stores the given expiration time and updates the updates the certificate +// expiry metric to the earliest expiration time. func (c *certificateExpirationTracker) set(a *Authenticator, t time.Time) { c.mu.Lock() defer c.mu.Unlock() c.m[a] = t - // update earliest earliest := time.Time{} for _, t := range c.m { if t.IsZero() { @@ -47,18 +52,9 @@ func (c *certificateExpirationTracker) set(a *Authenticator, t time.Time) { earliest = t } } - c.earliest = earliest -} - -// report reports the ttl to the earliest reported expiration time. -// If no Authenticators have reported a certificate expiration, this reports nil. -func (c *certificateExpirationTracker) report(now func() time.Time) { - c.mu.RLock() - defer c.mu.RUnlock() - if c.earliest.IsZero() { - metrics.ClientCertTTL.Set(nil) + if earliest.IsZero() { + c.metricSet(nil) } else { - ttl := c.earliest.Sub(now()) - metrics.ClientCertTTL.Set(&ttl) + c.metricSet(&earliest) } } diff --git a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/metrics_test.go b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/metrics_test.go index 974f433c856..dae90f5da89 100644 --- a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/metrics_test.go +++ b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/metrics_test.go @@ -19,36 +19,27 @@ package exec import ( "testing" "time" - - "k8s.io/client-go/tools/metrics" ) -type mockTTLGauge struct { - v *time.Duration +type mockExpiryGauge struct { + v *time.Time } -func (m *mockTTLGauge) Set(d *time.Duration) { - m.v = d +func (m *mockExpiryGauge) Set(t *time.Time) { + m.v = t } -func ptr(d time.Duration) *time.Duration { - return &d +func ptr(t time.Time) *time.Time { + return &t } func TestCertificateExpirationTracker(t *testing.T) { now := time.Now() - nowFn := func() time.Time { return now } - mockMetric := &mockTTLGauge{} - realMetric := metrics.ClientCertTTL - metrics.ClientCertTTL = mockMetric - defer func() { - metrics.ClientCertTTL = realMetric - }() + mockMetric := &mockExpiryGauge{} - tracker := &certificateExpirationTracker{m: map[*Authenticator]time.Time{}} - tracker.report(nowFn) - if mockMetric.v != nil { - t.Error("empty tracker should record nil value") + tracker := &certificateExpirationTracker{ + m: map[*Authenticator]time.Time{}, + metricSet: mockMetric.Set, } firstAuthenticator := &Authenticator{} @@ -57,31 +48,31 @@ func TestCertificateExpirationTracker(t *testing.T) { desc string auth *Authenticator time time.Time - want *time.Duration + want *time.Time }{ { desc: "ttl for one authenticator", auth: firstAuthenticator, time: now.Add(time.Minute * 10), - want: ptr(time.Minute * 10), + want: ptr(now.Add(time.Minute * 10)), }, { desc: "second authenticator shorter ttl", auth: secondAuthenticator, time: now.Add(time.Minute * 5), - want: ptr(time.Minute * 5), + want: ptr(now.Add(time.Minute * 5)), }, { desc: "update shorter to be longer", auth: secondAuthenticator, time: now.Add(time.Minute * 15), - want: ptr(time.Minute * 10), + want: ptr(now.Add(time.Minute * 10)), }, { desc: "update shorter to be zero time", auth: firstAuthenticator, time: time.Time{}, - want: ptr(time.Minute * 15), + want: ptr(now.Add(time.Minute * 15)), }, { desc: "update last to be zero time records nil", @@ -93,13 +84,12 @@ func TestCertificateExpirationTracker(t *testing.T) { // Must run in series as the tests build off each other. t.Run(tc.desc, func(t *testing.T) { tracker.set(tc.auth, tc.time) - tracker.report(nowFn) if mockMetric.v != nil && tc.want != nil { - if mockMetric.v.Seconds() != tc.want.Seconds() { - t.Errorf("got: %v; want: %v", mockMetric.v, tc.want) + if !mockMetric.v.Equal(*tc.want) { + t.Errorf("got: %s; want: %s", mockMetric.v, tc.want) } } else if mockMetric.v != tc.want { - t.Errorf("got: %v; want: %v", mockMetric.v, tc.want) + t.Errorf("got: %s; want: %s", mockMetric.v, tc.want) } }) } diff --git a/staging/src/k8s.io/client-go/tools/metrics/metrics.go b/staging/src/k8s.io/client-go/tools/metrics/metrics.go index ee6800f31ef..6a8f25a9445 100644 --- a/staging/src/k8s.io/client-go/tools/metrics/metrics.go +++ b/staging/src/k8s.io/client-go/tools/metrics/metrics.go @@ -31,9 +31,9 @@ type DurationMetric interface { Observe(duration time.Duration) } -// TTLMetric sets the time to live of something. -type TTLMetric interface { - Set(ttl *time.Duration) +// ExpiryMetric sets some time of expiry. If nil, assume not relevant. +type ExpiryMetric interface { + Set(expiry *time.Time) } // LatencyMetric observes client latency partitioned by verb and url. @@ -47,8 +47,8 @@ type ResultMetric interface { } var ( - // ClientCertTTL is the time to live of a client certificate - ClientCertTTL TTLMetric = noopTTL{} + // ClientCertExpiry is the expiry time of a client certificate + ClientCertExpiry ExpiryMetric = noopExpiry{} // ClientCertRotationAge is the age of a certificate that has just been rotated. ClientCertRotationAge DurationMetric = noopDuration{} // RequestLatency is the latency metric that rest clients will update. @@ -59,7 +59,7 @@ var ( // RegisterOpts contains all the metrics to register. Metrics may be nil. type RegisterOpts struct { - ClientCertTTL TTLMetric + ClientCertExpiry ExpiryMetric ClientCertRotationAge DurationMetric RequestLatency LatencyMetric RequestResult ResultMetric @@ -69,8 +69,8 @@ type RegisterOpts struct { // only be called once. func Register(opts RegisterOpts) { registerMetrics.Do(func() { - if opts.ClientCertTTL != nil { - ClientCertTTL = opts.ClientCertTTL + if opts.ClientCertExpiry != nil { + ClientCertExpiry = opts.ClientCertExpiry } if opts.ClientCertRotationAge != nil { ClientCertRotationAge = opts.ClientCertRotationAge @@ -88,9 +88,9 @@ type noopDuration struct{} func (noopDuration) Observe(time.Duration) {} -type noopTTL struct{} +type noopExpiry struct{} -func (noopTTL) Set(*time.Duration) {} +func (noopExpiry) Set(*time.Time) {} type noopLatency struct{} diff --git a/staging/src/k8s.io/client-go/util/certificate/certificate_manager.go b/staging/src/k8s.io/client-go/util/certificate/certificate_manager.go index e432aed7782..1af46abaa4e 100644 --- a/staging/src/k8s.io/client-go/util/certificate/certificate_manager.go +++ b/staging/src/k8s.io/client-go/util/certificate/certificate_manager.go @@ -112,12 +112,6 @@ type Config struct { // initialized using a generic, multi-use cert/key pair which will be // quickly replaced with a unique cert/key pair. BootstrapKeyPEM []byte - // CertificateExpiration will record a metric that shows the remaining - // lifetime of the certificate. This metric is a gauge because only the - // current cert expiry time is really useful. Reading this metric at any - // time simply gives the next expiration date, no need to keep some - // history (histogram) of all previous expiry dates. - CertificateExpiration Gauge // CertificateRotation will record a metric showing the time in seconds // that certificates lived before being rotated. This metric is a histogram // because there is value in keeping a history of rotation cadences. It @@ -185,7 +179,6 @@ type manager struct { certStore Store - certificateExpiration Gauge certificateRotation Histogram certificateRenewFailure Counter @@ -230,7 +223,6 @@ func NewManager(config *Config) (Manager, error) { certStore: config.CertificateStore, cert: cert, forceRotation: forceRotation, - certificateExpiration: config.CertificateExpiration, certificateRotation: config.CertificateRotation, certificateRenewFailure: config.CertificateRenewFailure, now: time.Now, @@ -554,9 +546,6 @@ func (m *manager) nextRotationDeadline() time.Time { deadline := m.cert.Leaf.NotBefore.Add(jitteryDuration(totalDuration)) klog.V(2).Infof("Certificate expiration is %v, rotation deadline is %v", notAfter, deadline) - if m.certificateExpiration != nil { - m.certificateExpiration.Set(float64(notAfter.Unix())) - } return deadline } diff --git a/staging/src/k8s.io/client-go/util/certificate/certificate_manager_test.go b/staging/src/k8s.io/client-go/util/certificate/certificate_manager_test.go index 85969857eb1..96650adcdc5 100644 --- a/staging/src/k8s.io/client-go/util/certificate/certificate_manager_test.go +++ b/staging/src/k8s.io/client-go/util/certificate/certificate_manager_test.go @@ -200,7 +200,6 @@ func TestSetRotationDeadline(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - g := metricMock{} m := manager{ cert: &tls.Certificate{ Leaf: &x509.Certificate{ @@ -208,10 +207,9 @@ func TestSetRotationDeadline(t *testing.T) { NotAfter: tc.notAfter, }, }, - getTemplate: func() *x509.CertificateRequest { return &x509.CertificateRequest{} }, - usages: []certificates.KeyUsage{}, - certificateExpiration: &g, - now: func() time.Time { return now }, + getTemplate: func() *x509.CertificateRequest { return &x509.CertificateRequest{} }, + usages: []certificates.KeyUsage{}, + now: func() time.Time { return now }, } jitteryDuration = func(float64) time.Duration { return time.Duration(float64(tc.notAfter.Sub(tc.notBefore)) * 0.7) } lowerBound := tc.notBefore.Add(time.Duration(float64(tc.notAfter.Sub(tc.notBefore)) * 0.7)) @@ -225,12 +223,6 @@ func TestSetRotationDeadline(t *testing.T) { deadline, lowerBound) } - if g.calls != 1 { - t.Errorf("%d metrics were recorded, wanted %d", g.calls, 1) - } - if g.lastValue != float64(tc.notAfter.Unix()) { - t.Errorf("%f value for metric was recorded, wanted %d", g.lastValue, tc.notAfter.Unix()) - } }) } } diff --git a/staging/src/k8s.io/component-base/metrics/prometheus/restclient/metrics.go b/staging/src/k8s.io/component-base/metrics/prometheus/restclient/metrics.go index 51888a640ec..ccbc1d2ffa0 100644 --- a/staging/src/k8s.io/component-base/metrics/prometheus/restclient/metrics.go +++ b/staging/src/k8s.io/component-base/metrics/prometheus/restclient/metrics.go @@ -57,14 +57,23 @@ var ( []string{"code", "method", "host"}, ) - execPluginCertTTL = k8smetrics.NewGauge( - &k8smetrics.GaugeOpts{ + execPluginCertTTLAdapter = &expiryToTTLAdapter{} + + execPluginCertTTL = k8smetrics.NewGaugeFunc( + k8smetrics.GaugeOpts{ Name: "rest_client_exec_plugin_ttl_seconds", Help: "Gauge of the shortest TTL (time-to-live) of the client " + "certificate(s) managed by the auth exec plugin. The value " + - "is in seconds until certificate expiry. If auth exec " + - "plugins are unused or manage no TLS certificates, the " + - "value will be +INF.", + "is in seconds until certificate expiry (negative if " + + "already expired). If auth exec plugins are unused or manage no " + + "TLS certificates, the value will be +INF.", + StabilityLevel: k8smetrics.ALPHA, + }, + func() float64 { + if execPluginCertTTLAdapter.e == nil { + return math.Inf(1) + } + return execPluginCertTTLAdapter.e.Sub(time.Now()).Seconds() }, ) @@ -99,15 +108,14 @@ var ( ) func init() { - execPluginCertTTL.Set(math.Inf(1)) // Initialize TTL to +INF legacyregistry.MustRegister(requestLatency) legacyregistry.MustRegister(deprecatedRequestLatency) legacyregistry.MustRegister(requestResult) - legacyregistry.MustRegister(execPluginCertTTL) + legacyregistry.RawMustRegister(execPluginCertTTL) legacyregistry.MustRegister(execPluginCertRotation) metrics.Register(metrics.RegisterOpts{ - ClientCertTTL: &ttlAdapter{m: execPluginCertTTL}, + ClientCertExpiry: execPluginCertTTLAdapter, ClientCertRotationAge: &rotationAdapter{m: execPluginCertRotation}, RequestLatency: &latencyAdapter{m: requestLatency, dm: deprecatedRequestLatency}, RequestResult: &resultAdapter{requestResult}, @@ -132,16 +140,12 @@ func (r *resultAdapter) Increment(code, method, host string) { r.m.WithLabelValues(code, method, host).Inc() } -type ttlAdapter struct { - m *k8smetrics.Gauge +type expiryToTTLAdapter struct { + e *time.Time } -func (e *ttlAdapter) Set(ttl *time.Duration) { - if ttl == nil { - e.m.Set(math.Inf(1)) - } else { - e.m.Set(float64(ttl.Seconds())) - } +func (e *expiryToTTLAdapter) Set(expiry *time.Time) { + e.e = expiry } type rotationAdapter struct {