kubelet: add certificate rotation error metric

Kubernetes-commit: 8e50c55e6bf715d9067376a9e7f136ffacb0a3ee

util/certificate/certificate_manager.go

@@ -124,6 +124,9 @@ type Config struct {
 	// allows one to setup monitoring and alerting of unexpected rotation
 	// behavior and track trends in rotation frequency.
 	CertificateRotation Histogram
+	// CertifcateRenewFailure will record a metric that keeps track of
+	// certificate renewal failures.
+	CertificateRenewFailure Counter
 }
 
 // Store is responsible for getting and updating the current certificate.
@@ -154,6 +157,11 @@ type Histogram interface {
 	Observe(float64)
 }
 
+// Counter will wrap a counter with labels
+type Counter interface {
+	Inc()
+}
+
 // NoCertKeyError indicates there is no cert/key currently available.
 type NoCertKeyError string
 
@@ -179,6 +187,7 @@ type manager struct {
 
 	certificateExpiration   Gauge
 	certificateRotation     Histogram
+	certificateRenewFailure Counter
 
 	// the following variables must only be accessed under certAccessLock
 	certAccessLock sync.RWMutex
@@ -223,6 +232,7 @@ func NewManager(config *Config) (Manager, error) {
 		forceRotation:           forceRotation,
 		certificateExpiration:   config.CertificateExpiration,
 		certificateRotation:     config.CertificateRotation,
+		certificateRenewFailure: config.CertificateRenewFailure,
 		now:                     time.Now,
 	}
 
@@ -404,6 +414,9 @@ func (m *manager) rotateCerts() (bool, error) {
 	template, csrPEM, keyPEM, privateKey, err := m.generateCSR()
 	if err != nil {
 		utilruntime.HandleError(fmt.Errorf("Unable to generate a certificate signing request: %v", err))
+		if m.certificateRenewFailure != nil {
+			m.certificateRenewFailure.Inc()
+		}
 		return false, nil
 	}
 
@@ -411,6 +424,9 @@ func (m *manager) rotateCerts() (bool, error) {
 	client, err := m.getClient()
 	if err != nil {
 		utilruntime.HandleError(fmt.Errorf("Unable to load a client to request certificates: %v", err))
+		if m.certificateRenewFailure != nil {
+			m.certificateRenewFailure.Inc()
+		}
 		return false, nil
 	}
 
@@ -419,6 +435,9 @@ func (m *manager) rotateCerts() (bool, error) {
 	req, err := csr.RequestCertificate(client, csrPEM, "", m.usages, privateKey)
 	if err != nil {
 		utilruntime.HandleError(fmt.Errorf("Failed while requesting a signed certificate from the master: %v", err))
+		if m.certificateRenewFailure != nil {
+			m.certificateRenewFailure.Inc()
+		}
 		return false, m.updateServerError(err)
 	}
 
@@ -433,12 +452,18 @@ func (m *manager) rotateCerts() (bool, error) {
 	crtPEM, err := csr.WaitForCertificate(ctx, client, req)
 	if err != nil {
 		utilruntime.HandleError(fmt.Errorf("certificate request was not signed: %v", err))
+		if m.certificateRenewFailure != nil {
+			m.certificateRenewFailure.Inc()
+		}
 		return false, nil
 	}
 
 	cert, err := m.certStore.Update(crtPEM, keyPEM)
 	if err != nil {
 		utilruntime.HandleError(fmt.Errorf("Unable to store the new cert/key pair: %v", err))
+		if m.certificateRenewFailure != nil {
+			m.certificateRenewFailure.Inc()
+		}
 		return false, nil
 	}