kubelet: add certificate rotation error metric

Kubernetes-commit: 8e50c55e6bf715d9067376a9e7f136ffacb0a3ee

util/certificate/certificate_manager.go

@@ -124,6 +124,9 @@ type Config struct {
 	// allows one to setup monitoring and alerting of unexpected rotation
 	// behavior and track trends in rotation frequency.
 	CertificateRotation Histogram
+	// CertifcateRenewFailure will record a metric that keeps track of
+	// certificate renewal failures.
+	CertificateRenewFailure Counter
 }
 
 // Store is responsible for getting and updating the current certificate.
@@ -154,6 +157,11 @@ type Histogram interface {
 	Observe(float64)
 }
 
+// Counter will wrap a counter with labels
+type Counter interface {
+	Inc()
+}
+
 // NoCertKeyError indicates there is no cert/key currently available.
 type NoCertKeyError string
 
@@ -177,8 +185,9 @@ type manager struct {
 
 	certStore Store
 
-	certificateExpiration Gauge
-	certificateRotation   Histogram
+	certificateExpiration   Gauge
+	certificateRotation     Histogram
+	certificateRenewFailure Counter
 
 	// the following variables must only be accessed under certAccessLock
 	certAccessLock sync.RWMutex
@@ -213,17 +222,18 @@ func NewManager(config *Config) (Manager, error) {
 	}
 
 	m := manager{
-		stopCh:                make(chan struct{}),
-		clientFn:              config.ClientFn,
-		getTemplate:           getTemplate,
-		dynamicTemplate:       config.GetTemplate != nil,
-		usages:                config.Usages,
-		certStore:             config.CertificateStore,
-		cert:                  cert,
-		forceRotation:         forceRotation,
-		certificateExpiration: config.CertificateExpiration,
-		certificateRotation:   config.CertificateRotation,
-		now:                   time.Now,
+		stopCh:                  make(chan struct{}),
+		clientFn:                config.ClientFn,
+		getTemplate:             getTemplate,
+		dynamicTemplate:         config.GetTemplate != nil,
+		usages:                  config.Usages,
+		certStore:               config.CertificateStore,
+		cert:                    cert,
+		forceRotation:           forceRotation,
+		certificateExpiration:   config.CertificateExpiration,
+		certificateRotation:     config.CertificateRotation,
+		certificateRenewFailure: config.CertificateRenewFailure,
+		now:                     time.Now,
 	}
 
 	return &m, nil
@@ -404,6 +414,9 @@ func (m *manager) rotateCerts() (bool, error) {
 	template, csrPEM, keyPEM, privateKey, err := m.generateCSR()
 	if err != nil {
 		utilruntime.HandleError(fmt.Errorf("Unable to generate a certificate signing request: %v", err))
+		if m.certificateRenewFailure != nil {
+			m.certificateRenewFailure.Inc()
+		}
 		return false, nil
 	}
 
@@ -411,6 +424,9 @@ func (m *manager) rotateCerts() (bool, error) {
 	client, err := m.getClient()
 	if err != nil {
 		utilruntime.HandleError(fmt.Errorf("Unable to load a client to request certificates: %v", err))
+		if m.certificateRenewFailure != nil {
+			m.certificateRenewFailure.Inc()
+		}
 		return false, nil
 	}
 
@@ -419,6 +435,9 @@ func (m *manager) rotateCerts() (bool, error) {
 	req, err := csr.RequestCertificate(client, csrPEM, "", m.usages, privateKey)
 	if err != nil {
 		utilruntime.HandleError(fmt.Errorf("Failed while requesting a signed certificate from the master: %v", err))
+		if m.certificateRenewFailure != nil {
+			m.certificateRenewFailure.Inc()
+		}
 		return false, m.updateServerError(err)
 	}
 
@@ -433,12 +452,18 @@ func (m *manager) rotateCerts() (bool, error) {
 	crtPEM, err := csr.WaitForCertificate(ctx, client, req)
 	if err != nil {
 		utilruntime.HandleError(fmt.Errorf("certificate request was not signed: %v", err))
+		if m.certificateRenewFailure != nil {
+			m.certificateRenewFailure.Inc()
+		}
 		return false, nil
 	}
 
 	cert, err := m.certStore.Update(crtPEM, keyPEM)
 	if err != nil {
 		utilruntime.HandleError(fmt.Errorf("Unable to store the new cert/key pair: %v", err))
+		if m.certificateRenewFailure != nil {
+			m.certificateRenewFailure.Inc()
+		}
 		return false, nil
 	}