From 49a19c6011e05363a8baf8e99c917d11a9496568 Mon Sep 17 00:00:00 2001 From: Jacob Simpson Date: Wed, 9 Aug 2017 10:50:10 -0700 Subject: [PATCH] Add metric for remaining life of authenticating certificates When incoming requests to the API server are authenticated by a certificate, the expiration of the certificate can affect the validity of the authentication. With auto rotation of certificates, which is starting with kubelet certificates, the goal is to use shorter lifetimes and let the kubelet renew the certificate as desired. Monitoring certificates which are approaching expiration and not renewing would be an early warning sign that nodes are about to stop participating in the cluster. --- .../pkg/authentication/request/x509/BUILD | 1 + .../pkg/authentication/request/x509/x509.go | 26 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/staging/src/k8s.io/apiserver/pkg/authentication/request/x509/BUILD b/staging/src/k8s.io/apiserver/pkg/authentication/request/x509/BUILD index 269620ca105..2303669e62a 100644 --- a/staging/src/k8s.io/apiserver/pkg/authentication/request/x509/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/authentication/request/x509/BUILD @@ -35,6 +35,7 @@ go_library( tags = ["automanaged"], deps = [ "//vendor/github.com/golang/glog:go_default_library", + "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library", diff --git a/staging/src/k8s.io/apiserver/pkg/authentication/request/x509/x509.go b/staging/src/k8s.io/apiserver/pkg/authentication/request/x509/x509.go index ce00219af7b..d583a321846 100644 --- a/staging/src/k8s.io/apiserver/pkg/authentication/request/x509/x509.go +++ b/staging/src/k8s.io/apiserver/pkg/authentication/request/x509/x509.go @@ -22,8 +22,10 @@ import ( "encoding/asn1" "fmt" "net/http" + "time" "github.com/golang/glog" + "github.com/prometheus/client_golang/prometheus" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" @@ -31,6 +33,28 @@ import ( "k8s.io/apiserver/pkg/authentication/user" ) +var clientCertificateExpirationHistogram = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Namespace: "apiserver", + Subsystem: "client", + Name: "certificate_expiration_seconds", + Help: "Distribution of the remaining lifetime on the certificate used to authenticate a request.", + Buckets: []float64{ + 0, + (6 * time.Hour).Seconds(), + (12 * time.Hour).Seconds(), + (24 * time.Hour).Seconds(), + (2 * 24 * time.Hour).Seconds(), + (4 * 24 * time.Hour).Seconds(), + (7 * 24 * time.Hour).Seconds(), + }, + }, +) + +func init() { + prometheus.MustRegister(clientCertificateExpirationHistogram) +} + // UserConversion defines an interface for extracting user info from a client certificate chain type UserConversion interface { User(chain []*x509.Certificate) (user.Info, bool, error) @@ -71,6 +95,8 @@ func (a *Authenticator) AuthenticateRequest(req *http.Request) (user.Info, bool, } } + remaining := req.TLS.PeerCertificates[0].NotAfter.Sub(time.Now()) + clientCertificateExpirationHistogram.Observe(remaining.Seconds()) chains, err := req.TLS.PeerCertificates[0].Verify(optsCopy) if err != nil { return nil, false, err