From 3bf34c8ff92760ec6499bb7b019ac4c2dac154d8 Mon Sep 17 00:00:00 2001 From: Max Sokolovsky Date: Tue, 4 Jan 2022 11:53:23 -0500 Subject: [PATCH] Fix listenAndServe certificate expiration by preloading certs Signed-off-by: Brad Davidson (cherry picked from commit 284cc004e84101e4a5b9fa50c00e101095db83f6) --- cert/cert.go | 4 ++ factory/cert_utils.go | 3 ++ factory/gen.go | 45 ++++++++++++++------ go.mod | 4 +- go.sum | 10 +++-- listener.go | 46 +++++++++++++++------ storage/file/file.go | 5 ++- storage/kubernetes/controller.go | 71 +++++++++++++++++++++----------- 8 files changed, 132 insertions(+), 56 deletions(-) diff --git a/cert/cert.go b/cert/cert.go index c82e2d5..2ef70da 100644 --- a/cert/cert.go +++ b/cert/cert.go @@ -90,6 +90,10 @@ func NewSelfSignedCACert(cfg Config, key crypto.Signer) (*x509.Certificate, erro if err != nil { return nil, err } + + logrus.Infof("generated self-signed CA certificate %s: notBefore=%s notAfter=%s", + tmpl.Subject, tmpl.NotBefore, tmpl.NotAfter) + return x509.ParseCertificate(certDERBytes) } diff --git a/factory/cert_utils.go b/factory/cert_utils.go index e2aa4af..1f21593 100644 --- a/factory/cert_utils.go +++ b/factory/cert_utils.go @@ -43,6 +43,9 @@ func NewSelfSignedCACert(key crypto.Signer, cn string, org ...string) (*x509.Cer return nil, err } + logrus.Infof("generated self-signed CA certificate %s: notBefore=%s notAfter=%s", + tmpl.Subject, tmpl.NotBefore, tmpl.NotAfter) + return x509.ParseCertificate(certDERBytes) } diff --git a/factory/gen.go b/factory/gen.go index 5bad159..d677b01 100644 --- a/factory/gen.go +++ b/factory/gen.go @@ -68,20 +68,41 @@ func collectCNs(secret *v1.Secret) (domains []string, ips []net.IP, err error) { return } -// Merge combines the SAN lists from the target and additional Secrets, and returns a potentially modified Secret, -// along with a bool indicating if the returned Secret has been updated or not. If the two SAN lists alread matched -// and no merging was necessary, but the Secrets' certificate fingerprints differed, the second secret is returned -// and the updated bool is set to true despite neither certificate having actually been modified. This is required -// to support handling certificate renewal within the kubernetes storage provider. +// Merge combines the SAN lists from the target and additional Secrets, and +// returns a potentially modified Secret, along with a bool indicating if the +// returned Secret is not the same as the target Secret. +// +// If the merge would not add any CNs to the additional Secret, the additional +// Secret is returned, to allow for certificate rotation/regeneration. +// +// If the merge would not add any CNs to the target Secret, the target Secret is +// returned; no merging is necessary. +// +// If neither certificate is acceptable as-is, a new certificate containing +// the union of the two lists is generated, using the private key from the +// first Secret. The returned Secret will contain the updated cert. func (t *TLS) Merge(target, additional *v1.Secret) (*v1.Secret, bool, error) { - secret, updated, err := t.AddCN(target, cns(additional)...) - if !updated { - if target.Annotations[fingerprint] != additional.Annotations[fingerprint] { - secret = additional - updated = true - } + // static secrets can't be altered, don't bother trying + if IsStatic(target) { + return target, false, nil } - return secret, updated, err + + mergedCNs := append(cns(target), cns(additional)...) + + // if the additional secret already has all the CNs, use it in preference to the + // current one. This behavior is required to allow for renewal or regeneration. + if !NeedsUpdate(0, additional, mergedCNs...) { + return additional, true, nil + } + + // if the target secret already has all the CNs, continue using it. The additional + // cert had only a subset of the current CNs, so nothing needs to be added. + if !NeedsUpdate(0, target, mergedCNs...) { + return target, false, nil + } + + // neither cert currently has all the necessary CNs; generate a new one. + return t.generateCert(target, mergedCNs...) } // Renew returns a copy of the given certificate that has been re-signed diff --git a/go.mod b/go.mod index d0420c1..a70fff5 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,9 @@ module github.com/rancher/dynamiclistener -go 1.16 +go 1.12 require ( - github.com/rancher/wrangler v0.8.3 + github.com/rancher/wrangler v0.8.9 github.com/sirupsen/logrus v1.4.2 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 k8s.io/api v0.18.8 diff --git a/go.sum b/go.sum index 9a359c6..38084cb 100644 --- a/go.sum +++ b/go.sum @@ -86,6 +86,7 @@ github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -307,8 +308,8 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/qri-io/starlib v0.4.2-0.20200213133954-ff2e8cd5ef8d/go.mod h1:7DPO4domFU579Ga6E61sB9VFNaniPVwJP5C4bBCu3wA= github.com/rancher/lasso v0.0.0-20210616224652-fc3ebd901c08 h1:NxR8Fh0eE7/5/5Zvlog9B5NVjWKqBSb1WYMUF7/IE5c= github.com/rancher/lasso v0.0.0-20210616224652-fc3ebd901c08/go.mod h1:9qZd/S8DqWzfKtjKGgSoHqGEByYmUE3qRaBaaAHwfEM= -github.com/rancher/wrangler v0.8.3 h1:m3d5ChOQj2Pdozy6nkGiSzAgQxlQlXRis2zSRwaO83k= -github.com/rancher/wrangler v0.8.3/go.mod h1:dKEaHNB4izxmPUtpq1Hvr3z3Oh+9k5pCZyFO9sUhlaY= +github.com/rancher/wrangler v0.8.9 h1:qNHBUw7jHdQKBVX4ksmY9ckth6oZaL2tRUKMwtERZw8= +github.com/rancher/wrangler v0.8.9/go.mod h1:Lte9WjPtGYxYacIWeiS9qawvu2R4NujFU9xuXWJvc/0= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= @@ -343,8 +344,9 @@ github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRci github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.3-0.20181224173747-660f15d67dbb/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -534,6 +536,8 @@ gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200121175148-a6ecf24a6d71/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/listener.go b/listener.go index 096145c..072fa3e 100644 --- a/listener.go +++ b/listener.go @@ -7,6 +7,7 @@ import ( "crypto/x509" "net" "net/http" + "runtime" "strings" "sync" "time" @@ -162,7 +163,10 @@ type listener struct { func (l *listener) WrapExpiration(days int) net.Listener { ctx, cancel := context.WithCancel(context.Background()) go func() { - time.Sleep(30 * time.Second) + // busy-wait for certificate preload to complete + for l.cert == nil { + runtime.Gosched() + } for { wait := 6 * time.Hour @@ -258,7 +262,13 @@ func (l *listener) checkExpiration(days int) error { func (l *listener) Accept() (net.Conn, error) { l.init.Do(func() { if len(l.sans) > 0 { - l.updateCert(l.sans...) + if err := l.updateCert(l.sans...); err != nil { + logrus.Errorf("failed to update cert with configured SANs: %v", err) + return + } + if _, err := l.loadCert(nil); err != nil { + logrus.Errorf("failed to preload certificate: %v", err) + } } }) @@ -280,7 +290,7 @@ func (l *listener) Accept() (net.Conn, error) { if !strings.Contains(host, ":") { if err := l.updateCert(host); err != nil { - logrus.Infof("failed to create TLS cert for: %s, %v", host, err) + logrus.Errorf("failed to update cert with listener address: %v", err) } } @@ -308,8 +318,9 @@ func (l *listener) wrap(conn net.Conn) net.Conn { type closeWrapper struct { net.Conn - id int - l *listener + id int + l *listener + ready bool } func (c *closeWrapper) close() error { @@ -324,13 +335,15 @@ func (c *closeWrapper) Close() error { } func (l *listener) getCertificate(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { + newConn := hello.Conn if hello.ServerName != "" { if err := l.updateCert(hello.ServerName); err != nil { + logrus.Errorf("failed to update cert with TLS ServerName: %v", err) return nil, err } } - return l.loadCert() + return l.loadCert(newConn) } func (l *listener) updateCert(cn ...string) error { @@ -347,7 +360,7 @@ func (l *listener) updateCert(cn ...string) error { return err } - if !factory.IsStatic(secret) && !factory.NeedsUpdate(l.maxSANs, secret, cn...) { + if factory.IsStatic(secret) || !factory.NeedsUpdate(l.maxSANs, secret, cn...) { return nil } @@ -365,14 +378,16 @@ func (l *listener) updateCert(cn ...string) error { if err := l.storage.Update(secret); err != nil { return err } - // clear version to force cert reload + // Clear version to force cert reload next time loadCert is called by TLSConfig's + // GetCertificate hook to provide a certificate for a new connection. Note that this + // means the old certificate stays in l.cert until a new connection is made. l.version = "" } return nil } -func (l *listener) loadCert() (*tls.Certificate, error) { +func (l *listener) loadCert(currentConn net.Conn) (*tls.Certificate, error) { l.RLock() defer l.RUnlock() @@ -402,12 +417,17 @@ func (l *listener) loadCert() (*tls.Certificate, error) { return nil, err } - // cert has changed, close closeWrapper wrapped connections - if l.conns != nil { + // cert has changed, close closeWrapper wrapped connections if this isn't the first load + if currentConn != nil && l.conns != nil && l.cert != nil { l.connLock.Lock() for _, conn := range l.conns { + // Don't close a connection that's in the middle of completing a TLS handshake + if !conn.ready { + continue + } _ = conn.close() } + l.conns[currentConn.(*closeWrapper).id].ready = true l.connLock.Unlock() } @@ -431,7 +451,9 @@ func (l *listener) cacheHandler() http.Handler { } } - l.updateCert(h) + if err := l.updateCert(h); err != nil { + logrus.Errorf("failed to update cert with HTTP request Host header: %v", err) + } } }) } diff --git a/storage/file/file.go b/storage/file/file.go index 0672944..a7f1376 100644 --- a/storage/file/file.go +++ b/storage/file/file.go @@ -2,9 +2,10 @@ package file import ( "encoding/json" - "github.com/rancher/dynamiclistener" - "k8s.io/api/core/v1" "os" + + "github.com/rancher/dynamiclistener" + v1 "k8s.io/api/core/v1" ) func New(file string) dynamiclistener.TLSStorage { diff --git a/storage/kubernetes/controller.go b/storage/kubernetes/controller.go index 2e82e96..b1a9e40 100644 --- a/storage/kubernetes/controller.go +++ b/storage/kubernetes/controller.go @@ -42,7 +42,7 @@ func New(ctx context.Context, core CoreGetter, namespace, name string, backing d core := core() if core != nil { storage.init(core.Core().V1().Secret()) - start.All(ctx, 5, core) + _ = start.All(ctx, 5, core) return } @@ -89,18 +89,31 @@ func (s *storage) init(secrets v1controller.SecretController) { }) s.secrets = secrets - if secret, err := s.storage.Get(); err == nil && secret != nil && len(secret.Data) > 0 { - // just ensure there is a secret in k3s - if _, err := s.secrets.Get(s.namespace, s.name, metav1.GetOptions{}); errors.IsNotFound(err) { - _, _ = s.secrets.Create(&v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: s.name, - Namespace: s.namespace, - Annotations: secret.Annotations, - }, - Type: v1.SecretTypeTLS, - Data: secret.Data, - }) + secret, err := s.storage.Get() + if err == nil && secret != nil && len(secret.Data) > 0 { + // local storage had a cached secret, ensure that it exists in Kubernetes + _, err := s.secrets.Create(&v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: s.name, + Namespace: s.namespace, + Annotations: secret.Annotations, + }, + Type: v1.SecretTypeTLS, + Data: secret.Data, + }) + if err != nil && !errors.IsAlreadyExists(err) { + logrus.Warnf("Failed to create Kubernetes secret: %v", err) + } + } else { + // local storage was empty, try to populate it + secret, err := s.secrets.Get(s.namespace, s.name, metav1.GetOptions{}) + if err != nil { + logrus.Warnf("Failed to init Kubernetes secret: %v", err) + return + } + + if err := s.storage.Update(secret); err != nil { + logrus.Warnf("Failed to init backing storage secret: %v", err) } } } @@ -130,23 +143,31 @@ func (s *storage) saveInK8s(secret *v1.Secret) (*v1.Secret, error) { return secret, nil } - if existing, err := s.storage.Get(); err == nil && s.tls != nil { - if newSecret, updated, err := s.tls.Merge(existing, secret); err == nil && updated { - secret = newSecret - } - } - targetSecret, err := s.targetSecret() if err != nil { return nil, err } - if newSecret, updated, err := s.tls.Merge(targetSecret, secret); err != nil { - return nil, err - } else if !updated { - return newSecret, nil - } else { - secret = newSecret + // if we don't have a TLS factory we can't create certs, so don't bother trying to merge anything, + // in favor of just blindly replacing the fields on the Kubernetes secret. + if s.tls != nil { + // merge new secret with secret from backing storage, if one exists + if existing, err := s.storage.Get(); err == nil && existing != nil && len(existing.Data) > 0 { + if newSecret, updated, err := s.tls.Merge(existing, secret); err == nil && updated { + secret = newSecret + } + } + + // merge new secret with existing secret from Kubernetes, if one exists + if len(targetSecret.Data) > 0 { + if newSecret, updated, err := s.tls.Merge(targetSecret, secret); err != nil { + return nil, err + } else if !updated { + return newSecret, nil + } else { + secret = newSecret + } + } } targetSecret.Annotations = secret.Annotations