Fix listenAndServe certificate expiration by preloading certs

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>
(cherry picked from commit 284cc004e8)

listener.go

@@ -7,6 +7,7 @@ import (
 	"crypto/x509"
 	"net"
 	"net/http"
+	"runtime"
 	"strings"
 	"sync"
 	"time"
@@ -162,7 +163,10 @@ type listener struct {
 func (l *listener) WrapExpiration(days int) net.Listener {
 	ctx, cancel := context.WithCancel(context.Background())
 	go func() {
-		time.Sleep(30 * time.Second)
+		// busy-wait for certificate preload to complete
+		for l.cert == nil {
+			runtime.Gosched()
+		}
 
 		for {
 			wait := 6 * time.Hour
@@ -258,7 +262,13 @@ func (l *listener) checkExpiration(days int) error {
 func (l *listener) Accept() (net.Conn, error) {
 	l.init.Do(func() {
 		if len(l.sans) > 0 {
-			l.updateCert(l.sans...)
+			if err := l.updateCert(l.sans...); err != nil {
+				logrus.Errorf("failed to update cert with configured SANs: %v", err)
+				return
+			}
+			if _, err := l.loadCert(nil); err != nil {
+				logrus.Errorf("failed to preload certificate: %v", err)
+			}
 		}
 	})
 
@@ -280,7 +290,7 @@ func (l *listener) Accept() (net.Conn, error) {
 
 	if !strings.Contains(host, ":") {
 		if err := l.updateCert(host); err != nil {
-			logrus.Infof("failed to create TLS cert for: %s, %v", host, err)
+			logrus.Errorf("failed to update cert with listener address: %v", err)
 		}
 	}
 
@@ -308,8 +318,9 @@ func (l *listener) wrap(conn net.Conn) net.Conn {
 
 type closeWrapper struct {
 	net.Conn
-	id int
-	l  *listener
+	id    int
+	l     *listener
+	ready bool
 }
 
 func (c *closeWrapper) close() error {
@@ -324,13 +335,15 @@ func (c *closeWrapper) Close() error {
 }
 
 func (l *listener) getCertificate(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
+	newConn := hello.Conn
 	if hello.ServerName != "" {
 		if err := l.updateCert(hello.ServerName); err != nil {
+			logrus.Errorf("failed to update cert with TLS ServerName: %v", err)
 			return nil, err
 		}
 	}
 
-	return l.loadCert()
+	return l.loadCert(newConn)
 }
 
 func (l *listener) updateCert(cn ...string) error {
@@ -347,7 +360,7 @@ func (l *listener) updateCert(cn ...string) error {
 		return err
 	}
 
-	if !factory.IsStatic(secret) && !factory.NeedsUpdate(l.maxSANs, secret, cn...) {
+	if factory.IsStatic(secret) || !factory.NeedsUpdate(l.maxSANs, secret, cn...) {
 		return nil
 	}
 
@@ -365,14 +378,16 @@ func (l *listener) updateCert(cn ...string) error {
 		if err := l.storage.Update(secret); err != nil {
 			return err
 		}
-		// clear version to force cert reload
+		// Clear version to force cert reload next time loadCert is called by TLSConfig's
+		// GetCertificate hook to provide a certificate for a new connection. Note that this
+		// means the old certificate stays in l.cert until a new connection is made.
 		l.version = ""
 	}
 
 	return nil
 }
 
-func (l *listener) loadCert() (*tls.Certificate, error) {
+func (l *listener) loadCert(currentConn net.Conn) (*tls.Certificate, error) {
 	l.RLock()
 	defer l.RUnlock()
 
@@ -402,12 +417,17 @@ func (l *listener) loadCert() (*tls.Certificate, error) {
 		return nil, err
 	}
 
-	// cert has changed, close closeWrapper wrapped connections
-	if l.conns != nil {
+	// cert has changed, close closeWrapper wrapped connections if this isn't the first load
+	if currentConn != nil && l.conns != nil && l.cert != nil {
 		l.connLock.Lock()
 		for _, conn := range l.conns {
+			// Don't close a connection that's in the middle of completing a TLS handshake
+			if !conn.ready {
+				continue
+			}
 			_ = conn.close()
 		}
+		l.conns[currentConn.(*closeWrapper).id].ready = true
 		l.connLock.Unlock()
 	}
 
@@ -431,7 +451,9 @@ func (l *listener) cacheHandler() http.Handler {
 				}
 			}
 
-			l.updateCert(h)
+			if err := l.updateCert(h); err != nil {
+				logrus.Errorf("failed to update cert with HTTP request Host header: %v", err)
+			}
 		}
 	})
 }