Add metric for remaining life of authenticating certificates

When incoming requests to the API server are authenticated by a
certificate, the expiration of the certificate can affect the validity
of the authentication. With auto rotation of certificates, which is
starting with kubelet certificates, the goal is to use shorter lifetimes
and let the kubelet renew the certificate as desired. Monitoring
certificates which are approaching expiration and not renewing would be
an early warning sign that nodes are about to stop participating in the
cluster.
This commit is contained in:
Jacob Simpson 2017-08-09 10:50:10 -07:00
parent 868fef189c
commit 49a19c6011
2 changed files with 27 additions and 0 deletions

View File

@ -35,6 +35,7 @@ go_library(
tags = ["automanaged"], tags = ["automanaged"],
deps = [ deps = [
"//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/golang/glog:go_default_library",
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library", "//vendor/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library",

View File

@ -22,8 +22,10 @@ import (
"encoding/asn1" "encoding/asn1"
"fmt" "fmt"
"net/http" "net/http"
"time"
"github.com/golang/glog" "github.com/golang/glog"
"github.com/prometheus/client_golang/prometheus"
utilerrors "k8s.io/apimachinery/pkg/util/errors" utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/sets"
@ -31,6 +33,28 @@ import (
"k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/authentication/user"
) )
var clientCertificateExpirationHistogram = prometheus.NewHistogram(
prometheus.HistogramOpts{
Namespace: "apiserver",
Subsystem: "client",
Name: "certificate_expiration_seconds",
Help: "Distribution of the remaining lifetime on the certificate used to authenticate a request.",
Buckets: []float64{
0,
(6 * time.Hour).Seconds(),
(12 * time.Hour).Seconds(),
(24 * time.Hour).Seconds(),
(2 * 24 * time.Hour).Seconds(),
(4 * 24 * time.Hour).Seconds(),
(7 * 24 * time.Hour).Seconds(),
},
},
)
func init() {
prometheus.MustRegister(clientCertificateExpirationHistogram)
}
// UserConversion defines an interface for extracting user info from a client certificate chain // UserConversion defines an interface for extracting user info from a client certificate chain
type UserConversion interface { type UserConversion interface {
User(chain []*x509.Certificate) (user.Info, bool, error) User(chain []*x509.Certificate) (user.Info, bool, error)
@ -71,6 +95,8 @@ func (a *Authenticator) AuthenticateRequest(req *http.Request) (user.Info, bool,
} }
} }
remaining := req.TLS.PeerCertificates[0].NotAfter.Sub(time.Now())
clientCertificateExpirationHistogram.Observe(remaining.Seconds())
chains, err := req.TLS.PeerCertificates[0].Verify(optsCopy) chains, err := req.TLS.PeerCertificates[0].Verify(optsCopy)
if err != nil { if err != nil {
return nil, false, err return nil, false, err