diff --git a/util/certificate/certificate_manager.go b/util/certificate/certificate_manager.go index 55f5f5cb..e432aed7 100644 --- a/util/certificate/certificate_manager.go +++ b/util/certificate/certificate_manager.go @@ -124,6 +124,9 @@ type Config struct { // allows one to setup monitoring and alerting of unexpected rotation // behavior and track trends in rotation frequency. CertificateRotation Histogram + // CertifcateRenewFailure will record a metric that keeps track of + // certificate renewal failures. + CertificateRenewFailure Counter } // Store is responsible for getting and updating the current certificate. @@ -154,6 +157,11 @@ type Histogram interface { Observe(float64) } +// Counter will wrap a counter with labels +type Counter interface { + Inc() +} + // NoCertKeyError indicates there is no cert/key currently available. type NoCertKeyError string @@ -177,8 +185,9 @@ type manager struct { certStore Store - certificateExpiration Gauge - certificateRotation Histogram + certificateExpiration Gauge + certificateRotation Histogram + certificateRenewFailure Counter // the following variables must only be accessed under certAccessLock certAccessLock sync.RWMutex @@ -213,17 +222,18 @@ func NewManager(config *Config) (Manager, error) { } m := manager{ - stopCh: make(chan struct{}), - clientFn: config.ClientFn, - getTemplate: getTemplate, - dynamicTemplate: config.GetTemplate != nil, - usages: config.Usages, - certStore: config.CertificateStore, - cert: cert, - forceRotation: forceRotation, - certificateExpiration: config.CertificateExpiration, - certificateRotation: config.CertificateRotation, - now: time.Now, + stopCh: make(chan struct{}), + clientFn: config.ClientFn, + getTemplate: getTemplate, + dynamicTemplate: config.GetTemplate != nil, + usages: config.Usages, + certStore: config.CertificateStore, + cert: cert, + forceRotation: forceRotation, + certificateExpiration: config.CertificateExpiration, + certificateRotation: config.CertificateRotation, + certificateRenewFailure: config.CertificateRenewFailure, + now: time.Now, } return &m, nil @@ -404,6 +414,9 @@ func (m *manager) rotateCerts() (bool, error) { template, csrPEM, keyPEM, privateKey, err := m.generateCSR() if err != nil { utilruntime.HandleError(fmt.Errorf("Unable to generate a certificate signing request: %v", err)) + if m.certificateRenewFailure != nil { + m.certificateRenewFailure.Inc() + } return false, nil } @@ -411,6 +424,9 @@ func (m *manager) rotateCerts() (bool, error) { client, err := m.getClient() if err != nil { utilruntime.HandleError(fmt.Errorf("Unable to load a client to request certificates: %v", err)) + if m.certificateRenewFailure != nil { + m.certificateRenewFailure.Inc() + } return false, nil } @@ -419,6 +435,9 @@ func (m *manager) rotateCerts() (bool, error) { req, err := csr.RequestCertificate(client, csrPEM, "", m.usages, privateKey) if err != nil { utilruntime.HandleError(fmt.Errorf("Failed while requesting a signed certificate from the master: %v", err)) + if m.certificateRenewFailure != nil { + m.certificateRenewFailure.Inc() + } return false, m.updateServerError(err) } @@ -433,12 +452,18 @@ func (m *manager) rotateCerts() (bool, error) { crtPEM, err := csr.WaitForCertificate(ctx, client, req) if err != nil { utilruntime.HandleError(fmt.Errorf("certificate request was not signed: %v", err)) + if m.certificateRenewFailure != nil { + m.certificateRenewFailure.Inc() + } return false, nil } cert, err := m.certStore.Update(crtPEM, keyPEM) if err != nil { utilruntime.HandleError(fmt.Errorf("Unable to store the new cert/key pair: %v", err)) + if m.certificateRenewFailure != nil { + m.certificateRenewFailure.Inc() + } return false, nil }