rancher/dynamiclistener
1
0
Fork 0
forked from github/dynamiclistener
Code Issues Pull Requests Projects Releases Wiki Activity

Fix listenAndServe certificate expiration by preloading certs

Browse Source 
Signed-off-by: Brad Davidson <brad.davidson@rancher.com>
(cherry picked from commit 284cc004e8)
This commit is contained in:
Max Sokolovsky 2022-01-04 11:53:23 -05:00
parent 097ec29ed8
commit 3bf34c8ff9
8 changed files with 132 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cert/cert.go
View File

@ -90,6 +90,10 @@ func NewSelfSignedCACert(cfg Config, key crypto.Signer) (*x509.Certificate, erro
if err != nil {
return nil, err
}
logrus.Infof("generated self-signed CA certificate %s: notBefore=%s notAfter=%s",
tmpl.Subject, tmpl.NotBefore, tmpl.NotAfter)
return x509.ParseCertificate(certDERBytes)
}

3
factory/cert_utils.go
View File

@ -43,6 +43,9 @@ func NewSelfSignedCACert(key crypto.Signer, cn string, org ...string) (*x509.Cer
return nil, err
}
logrus.Infof("generated self-signed CA certificate %s: notBefore=%s notAfter=%s",
tmpl.Subject, tmpl.NotBefore, tmpl.NotAfter)
return x509.ParseCertificate(certDERBytes)
}

45
factory/gen.go
View File

@ -68,20 +68,41 @@ func collectCNs(secret *v1.Secret) (domains []string, ips []net.IP, err error) {
return
}
// Merge combines the SAN lists from the target and additional Secrets, and returns a potentially modified Secret,
// along with a bool indicating if the returned Secret has been updated or not. If the two SAN lists alread matched
// and no merging was necessary, but the Secrets' certificate fingerprints differed, the second secret is returned
// and the updated bool is set to true despite neither certificate having actually been modified. This is required
// to support handling certificate renewal within the kubernetes storage provider.
// Merge combines the SAN lists from the target and additional Secrets, and
// returns a potentially modified Secret, along with a bool indicating if the
// returned Secret is not the same as the target Secret.
//
// If the merge would not add any CNs to the additional Secret, the additional
// Secret is returned, to allow for certificate rotation/regeneration.
//
// If the merge would not add any CNs to the target Secret, the target Secret is
// returned; no merging is necessary.
//
// If neither certificate is acceptable as-is, a new certificate containing
// the union of the two lists is generated, using the private key from the
// first Secret. The returned Secret will contain the updated cert.
func (t *TLS) Merge(target, additional *v1.Secret) (*v1.Secret, bool, error) {
secret, updated, err := t.AddCN(target, cns(additional)...)
if !updated {
if target.Annotations[fingerprint] != additional.Annotations[fingerprint] {
secret = additional
updated = true
}
// static secrets can't be altered, don't bother trying
if IsStatic(target) {
return target, false, nil
}
return secret, updated, err
mergedCNs := append(cns(target), cns(additional)...)
// if the additional secret already has all the CNs, use it in preference to the
// current one. This behavior is required to allow for renewal or regeneration.
if !NeedsUpdate(0, additional, mergedCNs...) {
return additional, true, nil
}
// if the target secret already has all the CNs, continue using it. The additional
// cert had only a subset of the current CNs, so nothing needs to be added.
if !NeedsUpdate(0, target, mergedCNs...) {
return target, false, nil
}
// neither cert currently has all the necessary CNs; generate a new one.
return t.generateCert(target, mergedCNs...)
}
// Renew returns a copy of the given certificate that has been re-signed

4
go.mod
View File

@ -1,9 +1,9 @@
module github.com/rancher/dynamiclistener
go 1.16
go 1.12
require (
github.com/rancher/wrangler v0.8.3
github.com/rancher/wrangler v0.8.9
github.com/sirupsen/logrus v1.4.2
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
k8s.io/api v0.18.8

10
go.sum
View File

@ -86,6 +86,7 @@ github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi
github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
@ -307,8 +308,8 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T
github.com/qri-io/starlib v0.4.2-0.20200213133954-ff2e8cd5ef8d/go.mod h1:7DPO4domFU579Ga6E61sB9VFNaniPVwJP5C4bBCu3wA=
github.com/rancher/lasso v0.0.0-20210616224652-fc3ebd901c08 h1:NxR8Fh0eE7/5/5Zvlog9B5NVjWKqBSb1WYMUF7/IE5c=
github.com/rancher/lasso v0.0.0-20210616224652-fc3ebd901c08/go.mod h1:9qZd/S8DqWzfKtjKGgSoHqGEByYmUE3qRaBaaAHwfEM=
github.com/rancher/wrangler v0.8.3 h1:m3d5ChOQj2Pdozy6nkGiSzAgQxlQlXRis2zSRwaO83k=
github.com/rancher/wrangler v0.8.3/go.mod h1:dKEaHNB4izxmPUtpq1Hvr3z3Oh+9k5pCZyFO9sUhlaY=
github.com/rancher/wrangler v0.8.9 h1:qNHBUw7jHdQKBVX4ksmY9ckth6oZaL2tRUKMwtERZw8=
github.com/rancher/wrangler v0.8.9/go.mod h1:Lte9WjPtGYxYacIWeiS9qawvu2R4NujFU9xuXWJvc/0=
github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
@ -343,8 +344,9 @@ github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRci
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.2.3-0.20181224173747-660f15d67dbb/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@ -534,6 +536,8 @@ gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200121175148-a6ecf24a6d71/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

46
listener.go
View File

@ -7,6 +7,7 @@ import (
"crypto/x509"
"net"
"net/http"
"runtime"
"strings"
"sync"
"time"
@ -162,7 +163,10 @@ type listener struct {
func (l *listener) WrapExpiration(days int) net.Listener {
ctx, cancel := context.WithCancel(context.Background())
go func() {
time.Sleep(30 * time.Second)
// busy-wait for certificate preload to complete
for l.cert == nil {
runtime.Gosched()
}
for {
wait := 6 * time.Hour
@ -258,7 +262,13 @@ func (l *listener) checkExpiration(days int) error {
func (l *listener) Accept() (net.Conn, error) {
l.init.Do(func() {
if len(l.sans) > 0 {
l.updateCert(l.sans...)
if err := l.updateCert(l.sans...); err != nil {
logrus.Errorf("failed to update cert with configured SANs: %v", err)
return
}
if _, err := l.loadCert(nil); err != nil {
logrus.Errorf("failed to preload certificate: %v", err)
}
}
})
@ -280,7 +290,7 @@ func (l *listener) Accept() (net.Conn, error) {
if !strings.Contains(host, ":") {
if err := l.updateCert(host); err != nil {
logrus.Infof("failed to create TLS cert for: %s, %v", host, err)
logrus.Errorf("failed to update cert with listener address: %v", err)
}
}
@ -308,8 +318,9 @@ func (l *listener) wrap(conn net.Conn) net.Conn {
type closeWrapper struct {
net.Conn
id int
l *listener
id int
l *listener
ready bool
}
func (c *closeWrapper) close() error {
@ -324,13 +335,15 @@ func (c *closeWrapper) Close() error {
}
func (l *listener) getCertificate(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
newConn := hello.Conn
if hello.ServerName != "" {
if err := l.updateCert(hello.ServerName); err != nil {
logrus.Errorf("failed to update cert with TLS ServerName: %v", err)
return nil, err
}
}
return l.loadCert()
return l.loadCert(newConn)
}
func (l *listener) updateCert(cn ...string) error {
@ -347,7 +360,7 @@ func (l *listener) updateCert(cn ...string) error {
return err
}
if !factory.IsStatic(secret) && !factory.NeedsUpdate(l.maxSANs, secret, cn...) {
if factory.IsStatic(secret) || !factory.NeedsUpdate(l.maxSANs, secret, cn...) {
return nil
}
@ -365,14 +378,16 @@ func (l *listener) updateCert(cn ...string) error {
if err := l.storage.Update(secret); err != nil {
return err
}
// clear version to force cert reload
// Clear version to force cert reload next time loadCert is called by TLSConfig's
// GetCertificate hook to provide a certificate for a new connection. Note that this
// means the old certificate stays in l.cert until a new connection is made.
l.version = ""
}
return nil
}
func (l *listener) loadCert() (*tls.Certificate, error) {
func (l *listener) loadCert(currentConn net.Conn) (*tls.Certificate, error) {
l.RLock()
defer l.RUnlock()
@ -402,12 +417,17 @@ func (l *listener) loadCert() (*tls.Certificate, error) {
return nil, err
}
// cert has changed, close closeWrapper wrapped connections
if l.conns != nil {
// cert has changed, close closeWrapper wrapped connections if this isn't the first load
if currentConn != nil && l.conns != nil && l.cert != nil {
l.connLock.Lock()
for _, conn := range l.conns {
// Don't close a connection that's in the middle of completing a TLS handshake
if !conn.ready {
continue
}
_ = conn.close()
}
l.conns[currentConn.(*closeWrapper).id].ready = true
l.connLock.Unlock()
}
@ -431,7 +451,9 @@ func (l *listener) cacheHandler() http.Handler {
}
}
l.updateCert(h)
if err := l.updateCert(h); err != nil {
logrus.Errorf("failed to update cert with HTTP request Host header: %v", err)
}
}
})
}

5
storage/file/file.go
View File

@ -2,9 +2,10 @@ package file
import (
"encoding/json"
"github.com/rancher/dynamiclistener"
"k8s.io/api/core/v1"
"os"
"github.com/rancher/dynamiclistener"
v1 "k8s.io/api/core/v1"
)
func New(file string) dynamiclistener.TLSStorage {

71
storage/kubernetes/controller.go
View File

@ -42,7 +42,7 @@ func New(ctx context.Context, core CoreGetter, namespace, name string, backing d
core := core()
if core != nil {
storage.init(core.Core().V1().Secret())
start.All(ctx, 5, core)
_ = start.All(ctx, 5, core)
return
}
@ -89,18 +89,31 @@ func (s *storage) init(secrets v1controller.SecretController) {
})
s.secrets = secrets
if secret, err := s.storage.Get(); err == nil && secret != nil && len(secret.Data) > 0 {
// just ensure there is a secret in k3s
if _, err := s.secrets.Get(s.namespace, s.name, metav1.GetOptions{}); errors.IsNotFound(err) {
_, _ = s.secrets.Create(&v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: s.name,
Namespace: s.namespace,
Annotations: secret.Annotations,
},
Type: v1.SecretTypeTLS,
Data: secret.Data,
})
secret, err := s.storage.Get()
if err == nil && secret != nil && len(secret.Data) > 0 {
// local storage had a cached secret, ensure that it exists in Kubernetes
_, err := s.secrets.Create(&v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: s.name,
Namespace: s.namespace,
Annotations: secret.Annotations,
},
Type: v1.SecretTypeTLS,
Data: secret.Data,
})
if err != nil && !errors.IsAlreadyExists(err) {
logrus.Warnf("Failed to create Kubernetes secret: %v", err)
}
} else {
// local storage was empty, try to populate it
secret, err := s.secrets.Get(s.namespace, s.name, metav1.GetOptions{})
if err != nil {
logrus.Warnf("Failed to init Kubernetes secret: %v", err)
return
}
if err := s.storage.Update(secret); err != nil {
logrus.Warnf("Failed to init backing storage secret: %v", err)
}
}
}
@ -130,23 +143,31 @@ func (s *storage) saveInK8s(secret *v1.Secret) (*v1.Secret, error) {
return secret, nil
}
if existing, err := s.storage.Get(); err == nil && s.tls != nil {
if newSecret, updated, err := s.tls.Merge(existing, secret); err == nil && updated {
secret = newSecret
}
}
targetSecret, err := s.targetSecret()
if err != nil {
return nil, err
}
if newSecret, updated, err := s.tls.Merge(targetSecret, secret); err != nil {
return nil, err
} else if !updated {
return newSecret, nil
} else {
secret = newSecret
// if we don't have a TLS factory we can't create certs, so don't bother trying to merge anything,
// in favor of just blindly replacing the fields on the Kubernetes secret.
if s.tls != nil {
// merge new secret with secret from backing storage, if one exists
if existing, err := s.storage.Get(); err == nil && existing != nil && len(existing.Data) > 0 {
if newSecret, updated, err := s.tls.Merge(existing, secret); err == nil && updated {
secret = newSecret
}
}
// merge new secret with existing secret from Kubernetes, if one exists
if len(targetSecret.Data) > 0 {
if newSecret, updated, err := s.tls.Merge(targetSecret, secret); err != nil {
return nil, err
} else if !updated {
return newSecret, nil
} else {
secret = newSecret
}
}
}
targetSecret.Annotations = secret.Annotations