Move Kubernetes Secrets storage update to goroutine

Fixes issue where apiserver outages can block dynamiclistener from accepting new connections. Signed-off-by: Brad Davidson <brad.davidson@rancher.com>
update config to allow for specifying experiation in days (#53 )
2022-05-02 18:48:48 -07:00 · 2021-12-21 15:38:04 -07:00 · 2021-11-23 23:38:49 -08:00 · 2021-11-23 23:38:49 -08:00 · 2021-11-19 14:29:09 -08:00 · 2021-11-19 13:50:48 -05:00
12 changed files with 330 additions and 99 deletions
--- a/README.md
+++ b/README.md
@@ -0,0 +1,10 @@
+# [dynamiclistener](https://github.com/rancher/dynamiclistener)
+
+This `README` is a work in progress; aimed towards providing information for navigating the contents of this repository.
+
+## Changing the Expiration Days for Newly Signed Certificates
+
+By default, a newly signed certificate is set to expire 365 days (1 year) after its creation time and date.
+You can use the `CATTLE_NEW_SIGNED_CERT_EXPIRATION_DAYS` environment variable to change this value.
+
+**Please note:** the value for the aforementioned variable must be a string representing an unsigned integer corresponding to the number of days until expiration (i.e. X509 "NotAfter" value).
--- a/cert/cert.go
+++ b/cert/cert.go
@@ -45,16 +45,15 @@ const (
 	duration365d = time.Hour * 24 * 365
 )

-var (
-	ErrStaticCert = errors.New("cannot renew static certificate")
-)
+var ErrStaticCert = errors.New("cannot renew static certificate")

-// Config contains the basic fields required for creating a certificate
+// Config contains the basic fields required for creating a certificate.
 type Config struct {
 	CommonName   string
 	Organization []string
 	AltNames     AltNames
 	Usages       []x509.ExtKeyUsage
+	ExpiresAt    time.Duration
 }

 // AltNames contains the domain names and IP addresses that will be added
@@ -90,10 +89,15 @@ func NewSelfSignedCACert(cfg Config, key crypto.Signer) (*x509.Certificate, erro
 	if err != nil {
 		return nil, err
 	}
+
+	logrus.Infof("generated self-signed CA certificate %s: notBefore=%s notAfter=%s",
+		tmpl.Subject, tmpl.NotBefore, tmpl.NotAfter)
+
 	return x509.ParseCertificate(certDERBytes)
 }

-// NewSignedCert creates a signed certificate using the given CA certificate and key
+// NewSignedCert creates a signed certificate using the given CA certificate and key based
+// on the given configuration.
 func NewSignedCert(cfg Config, key crypto.Signer, caCert *x509.Certificate, caKey crypto.Signer) (*x509.Certificate, error) {
 	serial, err := rand.Int(rand.Reader, new(big.Int).SetInt64(math.MaxInt64))
 	if err != nil {
@@ -105,6 +109,12 @@ func NewSignedCert(cfg Config, key crypto.Signer, caCert *x509.Certificate, caKe
 	if len(cfg.Usages) == 0 {
 		return nil, errors.New("must specify at least one ExtKeyUsage")
 	}
+	var expiresAt time.Duration
+	if cfg.ExpiresAt > 0 {
+		expiresAt = time.Duration(cfg.ExpiresAt)
+	} else {
+		expiresAt = duration365d
+	}

 	certTmpl := x509.Certificate{
 		Subject: pkix.Name{
@@ -115,7 +125,7 @@ func NewSignedCert(cfg Config, key crypto.Signer, caCert *x509.Certificate, caKe
 		IPAddresses:  cfg.AltNames.IPs,
 		SerialNumber: serial,
 		NotBefore:    caCert.NotBefore,
-		NotAfter:     time.Now().Add(duration365d).UTC(),
+		NotAfter:     time.Now().Add(expiresAt).UTC(),
 		KeyUsage:     x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
 		ExtKeyUsage:  cfg.Usages,
 	}
--- a/cert/secret.go
+++ b/cert/secret.go
@@ -0,0 +1,16 @@
+package cert
+
+import v1 "k8s.io/api/core/v1"
+
+func IsValidTLSSecret(secret *v1.Secret) bool {
+	if secret == nil {
+		return false
+	}
+	if _, ok := secret.Data[v1.TLSCertKey]; !ok {
+		return false
+	}
+	if _, ok := secret.Data[v1.TLSPrivateKeyKey]; !ok {
+		return false
+	}
+	return true
+}
--- a/factory/cert_utils.go
+++ b/factory/cert_utils.go
@@ -10,6 +10,8 @@ import (
 	"math"
 	"math/big"
 	"net"
+	"os"
+	"strconv"
 	"strings"
 	"time"

@@ -17,7 +19,8 @@ import (
 )

 const (
-	CertificateBlockType = "CERTIFICATE"
+	CertificateBlockType               = "CERTIFICATE"
+	defaultNewSignedCertExpirationDays = 365
 )

 func NewSelfSignedCACert(key crypto.Signer, cn string, org ...string) (*x509.Certificate, error) {
@@ -40,6 +43,9 @@ func NewSelfSignedCACert(key crypto.Signer, cn string, org ...string) (*x509.Cer
 		return nil, err
 	}

+	logrus.Infof("generated self-signed CA certificate %s: notBefore=%s notAfter=%s",
+		tmpl.Subject, tmpl.NotBefore, tmpl.NotAfter)
+
 	return x509.ParseCertificate(certDERBytes)
 }

@@ -82,12 +88,22 @@ func NewSignedCert(signer crypto.Signer, caCert *x509.Certificate, caKey crypto.
 		return nil, err
 	}

+	expirationDays := defaultNewSignedCertExpirationDays
+	envExpirationDays := os.Getenv("CATTLE_NEW_SIGNED_CERT_EXPIRATION_DAYS")
+	if envExpirationDays != "" {
+		if envExpirationDaysInt, err := strconv.Atoi(envExpirationDays); err != nil {
+			logrus.Infof("[NewSignedCert] expiration days from ENV (%s) could not be converted to int (falling back to default value: %d)", envExpirationDays, defaultNewSignedCertExpirationDays)
+		} else {
+			expirationDays = envExpirationDaysInt
+		}
+	}
+
 	parent := x509.Certificate{
 		DNSNames:     domains,
 		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
 		IPAddresses:  ips,
 		KeyUsage:     x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
-		NotAfter:     time.Now().Add(time.Hour * 24 * 365).UTC(),
+		NotAfter:     time.Now().Add(time.Hour * 24 * time.Duration(expirationDays)).UTC(),
 		NotBefore:    caCert.NotBefore,
 		SerialNumber: serialNumber,
 		Subject: pkix.Name{
--- a/factory/gen.go
+++ b/factory/gen.go
@@ -6,7 +6,9 @@ import (
 	"crypto/elliptic"
 	"crypto/rand"
 	"crypto/sha1"
+	"crypto/sha256"
 	"crypto/x509"
+	"encoding/hex"
 	"encoding/pem"
 	"fmt"
 	"net"
@@ -26,7 +28,7 @@ const (
 )

 var (
-	cnRegexp = regexp.MustCompile("^([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$")
+	cnRegexp = regexp.MustCompile("^([A-Za-z0-9:][-A-Za-z0-9_.:]*)?[A-Za-z0-9:]$")
 )

 type TLS struct {
@@ -68,20 +70,41 @@ func collectCNs(secret *v1.Secret) (domains []string, ips []net.IP, err error) {
 	return
 }

-// Merge combines the SAN lists from the target and additional Secrets, and returns a potentially modified Secret,
-// along with a bool indicating if the returned Secret has been updated or not. If the two SAN lists alread matched
-// and no merging was necessary, but the Secrets' certificate fingerprints differed, the second secret is returned
-// and the updated bool is set to true despite neither certificate having actually been modified. This is required
-// to support handling certificate renewal within the kubernetes storage provider.
+// Merge combines the SAN lists from the target and additional Secrets, and
+// returns a potentially modified Secret, along with a bool indicating if the
+// returned Secret is not the same as the target Secret.
+//
+// If the merge would not add any CNs to the additional Secret, the additional
+// Secret is returned, to allow for certificate rotation/regeneration.
+//
+// If the merge would not add any CNs to the target Secret, the target Secret is
+// returned; no merging is necessary.
+//
+// If neither certificate is acceptable as-is, a new certificate containing
+// the union of the two lists is generated, using the private key from the
+// first Secret. The returned Secret will contain the updated cert.
 func (t *TLS) Merge(target, additional *v1.Secret) (*v1.Secret, bool, error) {
-	secret, updated, err := t.AddCN(target, cns(additional)...)
-	if !updated {
-		if target.Annotations[fingerprint] != additional.Annotations[fingerprint] {
-			secret = additional
-			updated = true
-		}
+	// static secrets can't be altered, don't bother trying
+	if IsStatic(target) {
+		return target, false, nil
 	}
-	return secret, updated, err
+
+	mergedCNs := append(cns(target), cns(additional)...)
+
+	// if the additional secret already has all the CNs, use it in preference to the
+	// current one. This behavior is required to allow for renewal or regeneration.
+	if !NeedsUpdate(0, additional, mergedCNs...) {
+		return additional, true, nil
+	}
+
+	// if the target secret already has all the CNs, continue using it. The additional
+	// cert had only a subset of the current CNs, so nothing needs to be added.
+	if !NeedsUpdate(0, target, mergedCNs...) {
+		return target, false, nil
+	}
+
+	// neither cert currently has all the necessary CNs; generate a new one.
+	return t.generateCert(target, mergedCNs...)
 }

 // Renew returns a copy of the given certificate that has been re-signed
@@ -119,6 +142,12 @@ func (t *TLS) AddCN(secret *v1.Secret, cn ...string) (*v1.Secret, bool, error) {
 	return t.generateCert(secret, cn...)
 }

+func (t *TLS) Regenerate(secret *v1.Secret) (*v1.Secret, error) {
+	cns := cns(secret)
+	secret, _, err := t.generateCert(nil, cns...)
+	return secret, err
+}
+
 func (t *TLS) generateCert(secret *v1.Secret, cn ...string) (*v1.Secret, bool, error) {
 	secret = secret.DeepCopy()
 	if secret == nil {
@@ -150,6 +179,7 @@ func (t *TLS) generateCert(secret *v1.Secret, cn ...string) (*v1.Secret, bool, e
 	if secret.Data == nil {
 		secret.Data = map[string][]byte{}
 	}
+	secret.Type = v1.SecretTypeTLS
 	secret.Data[v1.TLSCertKey] = certBytes
 	secret.Data[v1.TLSPrivateKeyKey] = keyBytes
 	secret.Annotations[fingerprint] = fmt.Sprintf("SHA1=%X", sha1.Sum(newCert.Raw))
@@ -168,7 +198,7 @@ func populateCN(secret *v1.Secret, cn ...string) *v1.Secret {
 	}
 	for _, cn := range cn {
 		if cnRegexp.MatchString(cn) {
-			secret.Annotations[cnPrefix+cn] = cn
+			secret.Annotations[getAnnotationKey(cn)] = cn
 		} else {
 			logrus.Errorf("dropping invalid CN: %s", cn)
 		}
@@ -179,7 +209,10 @@ func populateCN(secret *v1.Secret, cn ...string) *v1.Secret {
 // IsStatic returns true if the Secret has an attribute indicating that it contains
 // a static (aka user-provided) certificate, which should not be modified.
 func IsStatic(secret *v1.Secret) bool {
-	return secret.Annotations[Static] == "true"
+	if secret != nil && secret.Annotations != nil {
+		return secret.Annotations[Static] == "true"
+	}
+	return false
 }

 // NeedsUpdate returns true if any of the CNs are not currently present on the
@@ -192,7 +225,7 @@ func NeedsUpdate(maxSANs int, secret *v1.Secret, cn ...string) bool {
 	}

 	for _, cn := range cn {
-		if secret.Annotations[cnPrefix+cn] == "" {
+		if secret.Annotations[getAnnotationKey(cn)] == "" {
 			if maxSANs > 0 && len(cns(secret)) >= maxSANs {
 				return false
 			}
@@ -236,3 +269,22 @@ func Marshal(x509Cert *x509.Certificate, privateKey crypto.Signer) ([]byte, []by
 func NewPrivateKey() (crypto.Signer, error) {
 	return ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
 }
+
+// getAnnotationKey return the key to use for a given CN. IPv4 addresses and short hostnames
+// are safe to store as-is, but longer hostnames and IPv6 address must be truncated and/or undergo
+// character replacement in order to be used as an annotation key. If the CN requires modification,
+// a portion of the SHA256 sum of the original value is used as the suffix, to reduce the likelihood
+// of collisions in modified values.
+func getAnnotationKey(cn string) string {
+	cn = cnPrefix + cn
+	cnLen := len(cn)
+	if cnLen < 64 && !strings.ContainsRune(cn, ':') {
+		return cn
+	}
+	digest := sha256.Sum256([]byte(cn))
+	cn = strings.ReplaceAll(cn, ":", "_")
+	if cnLen > 56 {
+		cnLen = 56
+	}
+	return cn[0:cnLen] + "-" + hex.EncodeToString(digest[0:])[0:6]
+}
--- a/go.mod
+++ b/go.mod
@@ -3,9 +3,10 @@ module github.com/rancher/dynamiclistener
 go 1.12

 require (
-	github.com/rancher/wrangler v0.8.0
+	github.com/rancher/wrangler v0.8.9
 	github.com/sirupsen/logrus v1.4.2
 	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
 	k8s.io/api v0.18.8
 	k8s.io/apimachinery v0.18.8
+	k8s.io/client-go v0.18.8
 )
--- a/go.sum
+++ b/go.sum
@@ -153,8 +153,9 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -169,8 +170,9 @@ github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
@@ -198,8 +200,9 @@ github.com/grpc-ecosystem/grpc-gateway v1.3.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpg
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk=
+github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
@@ -218,8 +221,9 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -302,10 +306,10 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/qri-io/starlib v0.4.2-0.20200213133954-ff2e8cd5ef8d/go.mod h1:7DPO4domFU579Ga6E61sB9VFNaniPVwJP5C4bBCu3wA=
-github.com/rancher/lasso v0.0.0-20210408231703-9ddd9378d08d h1:vfjPEF6M7Jf1/zK1xF7z2drLfniooKcgDQdoXO5+U7w=
-github.com/rancher/lasso v0.0.0-20210408231703-9ddd9378d08d/go.mod h1:OhBBBO1pBwYp0hacWdnvSGOj+XE9yMLOLnaypIlic18=
-github.com/rancher/wrangler v0.8.0 h1:jGWr7ES0KwZU/8zB6lte0z4QB7hFcspaXPTvGmrvHok=
-github.com/rancher/wrangler v0.8.0/go.mod h1:zSV5oh3+YZboilwcJmFHO3J6FZba82BTQft1b6ijx2I=
+github.com/rancher/lasso v0.0.0-20210616224652-fc3ebd901c08 h1:NxR8Fh0eE7/5/5Zvlog9B5NVjWKqBSb1WYMUF7/IE5c=
+github.com/rancher/lasso v0.0.0-20210616224652-fc3ebd901c08/go.mod h1:9qZd/S8DqWzfKtjKGgSoHqGEByYmUE3qRaBaaAHwfEM=
+github.com/rancher/wrangler v0.8.9 h1:qNHBUw7jHdQKBVX4ksmY9ckth6oZaL2tRUKMwtERZw8=
+github.com/rancher/wrangler v0.8.9/go.mod h1:Lte9WjPtGYxYacIWeiS9qawvu2R4NujFU9xuXWJvc/0=
 github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
@@ -340,8 +344,9 @@ github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRci
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.3-0.20181224173747-660f15d67dbb/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@@ -376,6 +381,7 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -408,6 +414,7 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -442,6 +449,7 @@ golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -473,10 +481,12 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191017205301-920acffc3e65/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -487,8 +497,9 @@ gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmK
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -525,6 +536,8 @@ gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200121175148-a6ecf24a6d71/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
--- a/listener.go
+++ b/listener.go
@@ -7,6 +7,7 @@ import (
 	"crypto/x509"
 	"net"
 	"net/http"
+	"runtime"
 	"strings"
 	"sync"
 	"time"
@@ -27,6 +28,7 @@ type TLSFactory interface {
 	AddCN(secret *v1.Secret, cn ...string) (*v1.Secret, bool, error)
 	Merge(target *v1.Secret, additional *v1.Secret) (*v1.Secret, bool, error)
 	Filter(cn ...string) []string
+	Regenerate(secret *v1.Secret) (*v1.Secret, error)
 }

 type SetFactory interface {
@@ -74,11 +76,18 @@ func NewListener(l net.Listener, storage TLSStorage, caCert *x509.Certificate, c
 		setter.SetFactory(dynamicListener.factory)
 	}

+	if config.RegenerateCerts != nil && config.RegenerateCerts() {
+		if err := dynamicListener.regenerateCerts(); err != nil {
+			return nil, nil, err
+		}
+	}
+
 	if config.ExpirationDaysCheck == 0 {
 		config.ExpirationDaysCheck = 30
 	}

 	tlsListener := tls.NewListener(dynamicListener.WrapExpiration(config.ExpirationDaysCheck), dynamicListener.tlsConfig)
+
 	return tlsListener, dynamicListener.cacheHandler(), nil
 }

@@ -129,6 +138,7 @@ type Config struct {
 	MaxSANs               int
 	ExpirationDaysCheck   int
 	CloseConnOnCertChange bool
+	RegenerateCerts       func() bool
 	FilterCN              func(...string) []string
 }

@@ -153,7 +163,10 @@ type listener struct {
 func (l *listener) WrapExpiration(days int) net.Listener {
 	ctx, cancel := context.WithCancel(context.Background())
 	go func() {
-		time.Sleep(30 * time.Second)
+		// busy-wait for certificate preload to complete
+		for l.cert == nil {
+			runtime.Gosched()
+		}

 		for {
 			wait := 6 * time.Hour
@@ -180,6 +193,30 @@ func (l *listener) WrapExpiration(days int) net.Listener {
 	}
 }

+// regenerateCerts regenerates the used certificates and
+// updates the secret.
+func (l *listener) regenerateCerts() error {
+	l.Lock()
+	defer l.Unlock()
+
+	secret, err := l.storage.Get()
+	if err != nil {
+		return err
+	}
+
+	newSecret, err := l.factory.Regenerate(secret)
+	if err != nil {
+		return err
+	}
+	if err := l.storage.Update(newSecret); err != nil {
+		return err
+	}
+	// clear version to force cert reload
+	l.version = ""
+
+	return nil
+}
+
 func (l *listener) checkExpiration(days int) error {
 	l.Lock()
 	defer l.Unlock()
@@ -225,7 +262,13 @@ func (l *listener) checkExpiration(days int) error {
 func (l *listener) Accept() (net.Conn, error) {
 	l.init.Do(func() {
 		if len(l.sans) > 0 {
-			l.updateCert(l.sans...)
+			if err := l.updateCert(l.sans...); err != nil {
+				logrus.Errorf("failed to update cert with configured SANs: %v", err)
+				return
+			}
+			if _, err := l.loadCert(nil); err != nil {
+				logrus.Errorf("failed to preload certificate: %v", err)
+			}
 		}
 	})

@@ -247,7 +290,7 @@ func (l *listener) Accept() (net.Conn, error) {

 	if !strings.Contains(host, ":") {
 		if err := l.updateCert(host); err != nil {
-			logrus.Infof("failed to create TLS cert for: %s, %v", host, err)
+			logrus.Errorf("failed to update cert with listener address: %v", err)
 		}
 	}

@@ -275,8 +318,9 @@ func (l *listener) wrap(conn net.Conn) net.Conn {

 type closeWrapper struct {
 	net.Conn
-	id int
-	l  *listener
+	id    int
+	l     *listener
+	ready bool
 }

 func (c *closeWrapper) close() error {
@@ -291,13 +335,15 @@ func (c *closeWrapper) Close() error {
 }

 func (l *listener) getCertificate(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
+	newConn := hello.Conn
 	if hello.ServerName != "" {
 		if err := l.updateCert(hello.ServerName); err != nil {
+			logrus.Errorf("failed to update cert with TLS ServerName: %v", err)
 			return nil, err
 		}
 	}

-	return l.loadCert()
+	return l.loadCert(newConn)
 }

 func (l *listener) updateCert(cn ...string) error {
@@ -314,7 +360,7 @@ func (l *listener) updateCert(cn ...string) error {
 		return err
 	}

-	if !factory.IsStatic(secret) && !factory.NeedsUpdate(l.maxSANs, secret, cn...) {
+	if factory.IsStatic(secret) || !factory.NeedsUpdate(l.maxSANs, secret, cn...) {
 		return nil
 	}

@@ -332,14 +378,16 @@ func (l *listener) updateCert(cn ...string) error {
 		if err := l.storage.Update(secret); err != nil {
 			return err
 		}
-		// clear version to force cert reload
+		// Clear version to force cert reload next time loadCert is called by TLSConfig's
+		// GetCertificate hook to provide a certificate for a new connection. Note that this
+		// means the old certificate stays in l.cert until a new connection is made.
 		l.version = ""
 	}

 	return nil
 }

-func (l *listener) loadCert() (*tls.Certificate, error) {
+func (l *listener) loadCert(currentConn net.Conn) (*tls.Certificate, error) {
 	l.RLock()
 	defer l.RUnlock()

@@ -360,6 +408,9 @@ func (l *listener) loadCert() (*tls.Certificate, error) {
 	if err != nil {
 		return nil, err
 	}
+	if !cert.IsValidTLSSecret(secret) {
+		return l.cert, nil
+	}
 	if l.cert != nil && l.version == secret.ResourceVersion && secret.ResourceVersion != "" {
 		return l.cert, nil
 	}
@@ -369,12 +420,17 @@ func (l *listener) loadCert() (*tls.Certificate, error) {
 		return nil, err
 	}

-	// cert has changed, close closeWrapper wrapped connections
-	if l.conns != nil {
+	// cert has changed, close closeWrapper wrapped connections if this isn't the first load
+	if currentConn != nil && l.conns != nil && l.cert != nil {
 		l.connLock.Lock()
 		for _, conn := range l.conns {
+			// Don't close a connection that's in the middle of completing a TLS handshake
+			if !conn.ready {
+				continue
+			}
 			_ = conn.close()
 		}
+		l.conns[currentConn.(*closeWrapper).id].ready = true
 		l.connLock.Unlock()
 	}

@@ -398,7 +454,9 @@ func (l *listener) cacheHandler() http.Handler {
 				}
 			}

-			l.updateCert(h)
+			if err := l.updateCert(h); err != nil {
+				logrus.Errorf("failed to update cert with HTTP request Host header: %v", err)
+			}
 		}
 	})
 }
--- a/server/server.go
+++ b/server/server.go
@@ -64,7 +64,10 @@ func ListenAndServe(ctx context.Context, httpsPort, httpPort int, handler http.H
 		}

 		tlsServer := http.Server{
-			Handler:  handler,
+			Handler: handler,
+			BaseContext: func(listener net.Listener) context.Context {
+				return ctx
+			},
 			ErrorLog: errorLog,
 		}

@@ -86,6 +89,9 @@ func ListenAndServe(ctx context.Context, httpsPort, httpPort int, handler http.H
 			Addr:     fmt.Sprintf("%s:%d", opts.BindHost, httpPort),
 			Handler:  handler,
 			ErrorLog: errorLog,
+			BaseContext: func(listener net.Listener) context.Context {
+				return ctx
+			},
 		}
 		go func() {
 			logrus.Infof("Listening on %s:%d", opts.BindHost, httpPort)
--- a/storage/file/file.go
+++ b/storage/file/file.go
@@ -2,9 +2,10 @@ package file

 import (
 	"encoding/json"
-	"github.com/rancher/dynamiclistener"
-	"k8s.io/api/core/v1"
 	"os"
+
+	"github.com/rancher/dynamiclistener"
+	v1 "k8s.io/api/core/v1"
 )

 func New(file string) dynamiclistener.TLSStorage {
--- a/storage/kubernetes/controller.go
+++ b/storage/kubernetes/controller.go
@@ -6,6 +6,7 @@ import (
 	"time"

 	"github.com/rancher/dynamiclistener"
+	"github.com/rancher/dynamiclistener/cert"
 	"github.com/rancher/wrangler/pkg/generated/controllers/core"
 	v1controller "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
 	"github.com/rancher/wrangler/pkg/start"
@@ -13,6 +14,7 @@ import (
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/util/retry"
 )

 type CoreGetter func() *core.Factory
@@ -39,10 +41,9 @@ func New(ctx context.Context, core CoreGetter, namespace, name string, backing d
 	// lazy init
 	go func() {
 		for {
-			core := core()
-			if core != nil {
-				storage.init(core.Core().V1().Secret())
-				start.All(ctx, 5, core)
+			if coreFactory := core(); coreFactory != nil {
+				storage.init(coreFactory.Core().V1().Secret())
+				_ = start.All(ctx, 5, coreFactory)
 				return
 			}

@@ -58,11 +59,11 @@ func New(ctx context.Context, core CoreGetter, namespace, name string, backing d
 }

 type storage struct {
-	sync.Mutex
+	sync.RWMutex

 	namespace, name string
 	storage         dynamiclistener.TLSStorage
-	secrets         v1controller.SecretClient
+	secrets         v1controller.SecretController
 	ctx             context.Context
 	tls             dynamiclistener.TLSFactory
 }
@@ -89,30 +90,46 @@ func (s *storage) init(secrets v1controller.SecretController) {
 	})
 	s.secrets = secrets

-	if secret, err := s.storage.Get(); err == nil && secret != nil && len(secret.Data) > 0 {
-		// just ensure there is a secret in k3s
-		if _, err := s.secrets.Get(s.namespace, s.name, metav1.GetOptions{}); errors.IsNotFound(err) {
-			_, _ = s.secrets.Create(&v1.Secret{
-				ObjectMeta: metav1.ObjectMeta{
-					Name:        s.name,
-					Namespace:   s.namespace,
-					Annotations: secret.Annotations,
-				},
-				Type: v1.SecretTypeTLS,
-				Data: secret.Data,
-			})
+	secret, err := s.storage.Get()
+	if err == nil && cert.IsValidTLSSecret(secret) {
+		// local storage had a cached secret, ensure that it exists in Kubernetes
+		_, err := s.secrets.Create(&v1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:        s.name,
+				Namespace:   s.namespace,
+				Annotations: secret.Annotations,
+			},
+			Type: v1.SecretTypeTLS,
+			Data: secret.Data,
+		})
+		if err != nil && !errors.IsAlreadyExists(err) {
+			logrus.Warnf("Failed to create Kubernetes secret: %v", err)
+		}
+	} else {
+		// local storage was empty, try to populate it
+		secret, err := s.secrets.Get(s.namespace, s.name, metav1.GetOptions{})
+		if err != nil {
+			logrus.Warnf("Failed to init Kubernetes secret: %v", err)
+			return
+		}
+
+		if err := s.storage.Update(secret); err != nil {
+			logrus.Warnf("Failed to init backing storage secret: %v", err)
 		}
 	}
 }

 func (s *storage) Get() (*v1.Secret, error) {
-	s.Lock()
-	defer s.Unlock()
+	s.RLock()
+	defer s.RUnlock()

 	return s.storage.Get()
 }

 func (s *storage) targetSecret() (*v1.Secret, error) {
+	s.RLock()
+	defer s.RUnlock()
+
 	existingSecret, err := s.secrets.Get(s.namespace, s.name, metav1.GetOptions{})
 	if errors.IsNotFound(err) {
 		return &v1.Secret{
@@ -120,33 +137,48 @@ func (s *storage) targetSecret() (*v1.Secret, error) {
 				Name:      s.name,
 				Namespace: s.namespace,
 			},
+			Type: v1.SecretTypeTLS,
 		}, nil
 	}
 	return existingSecret, err
 }

 func (s *storage) saveInK8s(secret *v1.Secret) (*v1.Secret, error) {
-	if s.secrets == nil {
+	if !s.controllerHasSynced() {
 		return secret, nil
 	}

-	if existing, err := s.storage.Get(); err == nil && s.tls != nil {
-		if newSecret, updated, err := s.tls.Merge(existing, secret); err == nil && updated {
-			secret = newSecret
-		}
-	}
-
 	targetSecret, err := s.targetSecret()
 	if err != nil {
 		return nil, err
 	}

-	if newSecret, updated, err := s.tls.Merge(targetSecret, secret); err != nil {
-		return nil, err
-	} else if !updated {
-		return newSecret, nil
-	} else {
-		secret = newSecret
+	// if we don't have a TLS factory we can't create certs, so don't bother trying to merge anything,
+	// in favor of just blindly replacing the fields on the Kubernetes secret.
+	if s.tls != nil {
+		// merge new secret with secret from backing storage, if one exists
+		if existing, err := s.Get(); err == nil && cert.IsValidTLSSecret(existing) {
+			if newSecret, updated, err := s.tls.Merge(existing, secret); err == nil && updated {
+				secret = newSecret
+			}
+		}
+
+		// merge new secret with existing secret from Kubernetes, if one exists
+		if cert.IsValidTLSSecret(targetSecret) {
+			if newSecret, updated, err := s.tls.Merge(targetSecret, secret); err != nil {
+				return nil, err
+			} else if !updated {
+				return newSecret, nil
+			} else {
+				secret = newSecret
+			}
+		}
+	}
+
+	// ensure that the merged secret actually contains data before overwriting the existing fields
+	if !cert.IsValidTLSSecret(secret) {
+		logrus.Warnf("Skipping save of TLS secret for %s/%s due to missing certificate data", secret.Namespace, secret.Name)
+		return targetSecret, nil
 	}

 	targetSecret.Annotations = secret.Annotations
@@ -154,31 +186,47 @@ func (s *storage) saveInK8s(secret *v1.Secret) (*v1.Secret, error) {
 	targetSecret.Data = secret.Data

 	if targetSecret.UID == "" {
-		logrus.Infof("Creating new TLS secret for %v (count: %d): %v", targetSecret.Name, len(targetSecret.Annotations)-1, targetSecret.Annotations)
+		logrus.Infof("Creating new TLS secret for %s/%s (count: %d): %v", targetSecret.Namespace, targetSecret.Name, len(targetSecret.Annotations)-1, targetSecret.Annotations)
 		return s.secrets.Create(targetSecret)
 	}
-	logrus.Infof("Updating TLS secret for %v (count: %d): %v", targetSecret.Name, len(targetSecret.Annotations)-1, targetSecret.Annotations)
+	logrus.Infof("Updating TLS secret for %s/%s (count: %d): %v", targetSecret.Namespace, targetSecret.Name, len(targetSecret.Annotations)-1, targetSecret.Annotations)
 	return s.secrets.Update(targetSecret)
 }

-func (s *storage) Update(secret *v1.Secret) (err error) {
-	s.Lock()
-	defer s.Unlock()
-
-	for i := 0; i < 3; i++ {
-		secret, err = s.saveInK8s(secret)
-		if errors.IsConflict(err) {
-			continue
-		} else if err != nil {
-			return err
+func (s *storage) Update(secret *v1.Secret) error {
+	// Asynchronously update the Kubernetes secret, as doing so inline may block the listener from
+	// accepting new connections if the apiserver becomes unavailable after the Secrets controller
+	// has been initialized. We're not passing around any contexts here, nor does the controller
+	// accept any, so there's no good way to soft-fail with a reasonable timeout.
+	go func() {
+		if err := s.update(secret); err != nil {
+			logrus.Errorf("Failed to save TLS secret for %s/%s: %v", secret.Namespace, secret.Name, err)
 		}
-		break
-	}
+	}()
+	return nil
+}
+
+func (s *storage) update(secret *v1.Secret) (err error) {
+	err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		secret, err = s.saveInK8s(secret)
+		return err
+	})

 	if err != nil {
 		return err
 	}

-	// update underlying storage
+	// Only hold the lock while updating underlying storage
+	s.Lock()
+	defer s.Unlock()
 	return s.storage.Update(secret)
 }
+
+func (s *storage) controllerHasSynced() bool {
+	s.RLock()
+	defer s.RUnlock()
+	if s.secrets == nil {
+		return false
+	}
+	return s.secrets.Informer().HasSynced()
+}
--- a/storage/memory/memory.go
+++ b/storage/memory/memory.go
@@ -39,7 +39,7 @@ func (m *memory) Update(secret *v1.Secret) error {
 			}
 		}

-		logrus.Infof("Active TLS secret %s (ver=%s) (count %d): %v", secret.Name, secret.ResourceVersion, len(secret.Annotations)-1, secret.Annotations)
+		logrus.Infof("Active TLS secret %s/%s (ver=%s) (count %d): %v", secret.Namespace, secret.Name, secret.ResourceVersion, len(secret.Annotations)-1, secret.Annotations)
 		m.secret = secret
 	}
 	return nil
Author	SHA1	Message	Date
Brad Davidson	b1d65efb6f	Move Kubernetes Secrets storage update to goroutine Fixes issue where apiserver outages can block dynamiclistener from accepting new connections. Signed-off-by: Brad Davidson <brad.davidson@rancher.com>	2022-05-02 18:48:48 -07:00
Brian Downs	148d38076d	update config to allow for specifying experiation in days (#53 )	2021-12-21 15:38:04 -07:00
Brad Davidson	43f9c3ae0a	Fix handling of IPv6 addresses and long hostnames Signed-off-by: Brad Davidson <brad.davidson@rancher.com>	2021-11-23 23:38:49 -08:00
Brad Davidson	284cc004e8	Fix listenAndServe certificate expiration by preloading certs Signed-off-by: Brad Davidson <brad.davidson@rancher.com>	2021-11-23 23:38:49 -08:00
Kinara Shah	120a37b97a	Merge pull request #51 from nickgerace/quick-fix Add README	2021-11-19 14:29:09 -08:00
Nick Gerace	bbac29e0fa	Add README	2021-11-19 13:50:48 -05:00
Kinara Shah	962b635269	Merge pull request #50 from nickgerace/quick-fix Fix defaultNewSignedCertExpirationDays const	2021-11-19 10:28:49 -08:00
Nick Gerace	f147aa4166	Fix defaultNewSignedCertExpirationDays const This a quick fix for `2644a6ed16`	2021-11-19 12:31:47 -05:00
Kinara Shah	63157c59ce	Merge pull request #46 from nickgerace/days Allow for default expiration days to be loaded from env	2021-11-19 08:59:57 -08:00
Nick Gerace	2644a6ed16	Allow for default expiration days to be loaded from env	2021-11-18 12:38:35 -05:00
Brian Downs	27f4642299	Add ability to force cert regeneration (#43 ) * add ability to force cert regeneration	2021-11-15 13:50:26 -07:00
Caleb Bron	cd5d71f2fe	Merge pull request #44 from cmurphy/fix-type Fix net.Conn type assertion	2021-11-04 13:09:48 -07:00
Colleen Murphy	fb66484384	Fix net.Conn type assertion Don't assert that all connections are wrapped, as they won't be if the CloseConnOnCertChange setting is false. Only run the assertion within a conditional for wrapped connections, where it is safe. This prevents a panic from happening when CloseConnOnCertChange is not used.	2021-10-29 11:03:02 -07:00
Darren Shepherd	6b37dc1212	Merge pull request #42 from cmurphy/fix-close-conn Skip closing an initializing connection	2021-10-27 08:35:21 -07:00
Colleen Murphy	c7dd355394	Skip closing an initializing connection Without this change, if a cert is updated (e.g. to add CNs) while the listener is in the middle of Accept()ing a new connection, the connection gets dropped, we'll see a message like this in the server logs: http: TLS handshake error from 127.0.0.1:51232: write tcp 127.0.7.1:8443->127.0.0.1:51232: use of closed network connection and the client (like a browser) won't necessarily reconnect. This change modifies the GetCertificate routine in the listener's tls.Config to keep track of the state of the incoming connections and only close connections that have completed GetCertificate and therefore are finished with their TLS handshake, so that only old established connections are closed.	2021-10-25 13:17:24 -07:00
Darren Shepherd	94e22490cf	Merge pull request #41 from weihanglo/nil-defer-storage-tls Merge TLS only if TLS factory is set	2021-08-03 10:23:59 -07:00
Weihang Lo	b45d8a455e	Merge TLS only if TLS factory is set Since `storage.tls` is optional, we should check it existence before calling its methods.	2021-07-12 18:25:01 +08:00
Darren Shepherd	9865ae859c	Don't reset connections on the first load of the certs	2021-06-16 01:00:09 -07:00
Darren Shepherd	db883ae66a	Don't reset connections on the first load of the certs	2021-06-16 00:23:14 -07:00
Darren Shepherd	9dfd7df057	Pass context to http server as BaseContext	2021-06-15 22:42:42 -07:00
Darren Shepherd	ff22834bde	Avoid panic when secret is nil	2021-06-15 22:42:42 -07:00
Sjoerd Simons	dc7452dbb8	Accept IPv6 address as CN names Expand the cnRegexp to also accept ipv6 addresses such as: * ::1 * 2a00:1450:400e:80e:: * 2a00:1450:400e:80e::200e Fixes: #37 Signed-off-by: Sjoerd Simons <sjoerd@collabora.com>	2021-06-14 11:07:13 -07:00
Dan Ramich	86af265dcd	Merge pull request #35 from dramich/panic Update IsStatic to check for nil annotations	2021-04-26 09:21:45 -06:00
Dan Ramich	f373fc1c7c	Update IsStatic to check for nil annotations	2021-04-23 14:56:14 -06:00