From f1fef11b3762ba79df7aa3a23c49ff0b099b050f Mon Sep 17 00:00:00 2001 From: Jacob Simpson Date: Wed, 16 Aug 2017 09:18:01 -0700 Subject: [PATCH] Add a kubelet metric to track certificate expiration. --- pkg/kubelet/certificate/BUILD | 3 ++ .../certificate/certificate_manager.go | 30 +++++++++++++++++-- .../certificate/certificate_manager_test.go | 21 +++++++++++-- pkg/kubelet/certificate/kubelet.go | 2 ++ 4 files changed, 52 insertions(+), 4 deletions(-) diff --git a/pkg/kubelet/certificate/BUILD b/pkg/kubelet/certificate/BUILD index a9df3be8c7d..9dfc95f814b 100644 --- a/pkg/kubelet/certificate/BUILD +++ b/pkg/kubelet/certificate/BUILD @@ -16,8 +16,10 @@ go_library( ], deps = [ "//pkg/kubelet/apis/kubeletconfig:go_default_library", + "//pkg/kubelet/metrics:go_default_library", "//pkg/util/file:go_default_library", "//vendor/github.com/golang/glog:go_default_library", + "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/k8s.io/api/certificates/v1beta1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library", @@ -41,6 +43,7 @@ go_test( ], library = ":go_default_library", deps = [ + "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/k8s.io/api/certificates/v1beta1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", diff --git a/pkg/kubelet/certificate/certificate_manager.go b/pkg/kubelet/certificate/certificate_manager.go index dcaa1a37749..200268126fa 100644 --- a/pkg/kubelet/certificate/certificate_manager.go +++ b/pkg/kubelet/certificate/certificate_manager.go @@ -28,6 +28,7 @@ import ( "time" "github.com/golang/glog" + "github.com/prometheus/client_golang/prometheus" certificates "k8s.io/api/certificates/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -36,10 +37,13 @@ import ( "k8s.io/apimachinery/pkg/watch" certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" "k8s.io/client-go/util/cert" + "k8s.io/kubernetes/pkg/kubelet/metrics" ) const ( - syncPeriod = 1 * time.Hour + syncPeriod = 1 * time.Hour + certificateManagerSubsystem = "certificate_manager" + certificateExpirationKey = "expiration_seconds" ) // Manager maintains and updates the certificates in use by this certificate @@ -59,6 +63,10 @@ type Manager interface { // Config is the set of configuration parameters available for a new Manager. type Config struct { + // Name is a name describing the certificate being managed by this + // certificate manager. It will be used for recording metrics relevant to + // the certificate. + Name string // CertificateSigningRequestClient will be used for signing new certificate // requests generated when a key rotation occurs. It must be set either at // initialization or by using CertificateSigningRequestClient before @@ -128,12 +136,17 @@ type manager struct { cert *tls.Certificate rotationDeadline time.Time forceRotation bool + certificateExpiration prometheus.Gauge } // NewManager returns a new certificate manager. A certificate manager is // responsible for being the authoritative source of certificates in the // Kubelet and handling updates due to rotation. func NewManager(config *Config) (Manager, error) { + if config.Name == "" { + return nil, fmt.Errorf("the 'Name' is required to disambiguate metric values of different certificate manager instances") + } + cert, forceRotation, err := getCurrentCertificateOrBootstrap( config.CertificateStore, config.BootstrapCertificatePEM, @@ -142,6 +155,17 @@ func NewManager(config *Config) (Manager, error) { return nil, err } + var certificateExpiration = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: metrics.KubeletSubsystem, + Subsystem: certificateManagerSubsystem, + Name: fmt.Sprintf("%s_%s", config.Name, certificateExpirationKey), + Help: "Gauge of the lifetime of a certificate. The value is the date the certificate will expire in seconds since January 1, 1970 UTC.", + }, + ) + + prometheus.MustRegister(certificateExpiration) + m := manager{ certSigningRequestClient: config.CertificateSigningRequestClient, template: config.Template, @@ -149,6 +173,7 @@ func NewManager(config *Config) (Manager, error) { certStore: config.CertificateStore, cert: cert, forceRotation: forceRotation, + certificateExpiration: certificateExpiration, } return &m, nil @@ -319,7 +344,8 @@ func (m *manager) setRotationDeadline() { jitteryDuration := wait.Jitter(time.Duration(totalDuration), 0.2) - time.Duration(totalDuration*0.3) m.rotationDeadline = m.cert.Leaf.NotBefore.Add(jitteryDuration) - glog.V(2).Infof("Certificate rotation deadline is %v", m.rotationDeadline) + glog.V(2).Infof("Certificate expiration is %v, rotation deadline is %v", notAfter, m.rotationDeadline) + m.certificateExpiration.Set(float64(notAfter.Unix())) } func (m *manager) updateCached(cert *tls.Certificate) { diff --git a/pkg/kubelet/certificate/certificate_manager_test.go b/pkg/kubelet/certificate/certificate_manager_test.go index b671ad4aba2..e2f35706bda 100644 --- a/pkg/kubelet/certificate/certificate_manager_test.go +++ b/pkg/kubelet/certificate/certificate_manager_test.go @@ -26,6 +26,8 @@ import ( "testing" "time" + "github.com/prometheus/client_golang/prometheus" + certificates "k8s.io/api/certificates/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" watch "k8s.io/apimachinery/pkg/watch" @@ -135,6 +137,7 @@ func TestNewManagerNoRotation(t *testing.T) { cert: storeCertData.certificate, } if _, err := NewManager(&Config{ + Name: "test_no_rotation", Template: &x509.CertificateRequest{}, Usages: []certificates.KeyUsage{}, CertificateStore: store, @@ -170,6 +173,11 @@ func TestShouldRotate(t *testing.T) { }, template: &x509.CertificateRequest{}, usages: []certificates.KeyUsage{}, + certificateExpiration: prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "test_gauge_name", + }, + ), } m.setRotationDeadline() if m.shouldRotate() != test.shouldRotate { @@ -212,6 +220,11 @@ func TestSetRotationDeadline(t *testing.T) { }, template: &x509.CertificateRequest{}, usages: []certificates.KeyUsage{}, + certificateExpiration: prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "test_gauge_name", + }, + ), } lowerBound := tc.notBefore.Add(time.Duration(float64(tc.notAfter.Sub(tc.notBefore)) * 0.7)) upperBound := tc.notBefore.Add(time.Duration(float64(tc.notAfter.Sub(tc.notBefore)) * 0.9)) @@ -282,6 +295,7 @@ func TestNewManagerBootstrap(t *testing.T) { var cm Manager cm, err := NewManager(&Config{ + Name: "test_bootstrap", Template: &x509.CertificateRequest{}, Usages: []certificates.KeyUsage{}, CertificateStore: store, @@ -319,6 +333,7 @@ func TestNewManagerNoBootstrap(t *testing.T) { } cm, err := NewManager(&Config{ + Name: "test_no_bootstrap", Template: &x509.CertificateRequest{}, Usages: []certificates.KeyUsage{}, CertificateStore: store, @@ -454,13 +469,14 @@ func TestInitializeCertificateSigningRequestClient(t *testing.T) { }, } - for _, tc := range testCases { + for i, tc := range testCases { t.Run(tc.description, func(t *testing.T) { certificateStore := &fakeStore{ cert: tc.storeCert.certificate, } certificateManager, err := NewManager(&Config{ + Name: fmt.Sprintf("test_initialize_client_%d", i), Template: &x509.CertificateRequest{ Subject: pkix.Name{ Organization: []string{"system:nodes"}, @@ -555,13 +571,14 @@ func TestInitializeOtherRESTClients(t *testing.T) { }, } - for _, tc := range testCases { + for i, tc := range testCases { t.Run(tc.description, func(t *testing.T) { certificateStore := &fakeStore{ cert: tc.storeCert.certificate, } certificateManager, err := NewManager(&Config{ + Name: fmt.Sprintf("test_initialize_other_rest_clients_%d", i), Template: &x509.CertificateRequest{ Subject: pkix.Name{ Organization: []string{"system:nodes"}, diff --git a/pkg/kubelet/certificate/kubelet.go b/pkg/kubelet/certificate/kubelet.go index b7459868150..c346bcb4f8e 100644 --- a/pkg/kubelet/certificate/kubelet.go +++ b/pkg/kubelet/certificate/kubelet.go @@ -46,6 +46,7 @@ func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg return nil, fmt.Errorf("failed to initialize server certificate store: %v", err) } m, err := NewManager(&Config{ + Name: "server", CertificateSigningRequestClient: certSigningRequestClient, Template: &x509.CertificateRequest{ Subject: pkix.Name{ @@ -92,6 +93,7 @@ func NewKubeletClientCertificateManager(certDirectory string, nodeName types.Nod return nil, fmt.Errorf("failed to initialize client certificate store: %v", err) } m, err := NewManager(&Config{ + Name: "client", Template: &x509.CertificateRequest{ Subject: pkix.Name{ CommonName: fmt.Sprintf("system:node:%s", nodeName),