backport issue 48484 (#203)

When using a chained store of Kubernetes -> Memory -> File, a file-backed cert with a valid ResourceVersion could not be updated when the Kubernetes store was offline, as the Memory store was skipping the update if the ResourceVersion was not changed.
The Kubernetes store passes through the secret update without a modified ResourceVersion if the Secret controller is not yet available to round-trip the secret through the apiserver, as the apiserver is what handles updating the ResourceVersion when the Secret changes.
In RKE2, this caused a deadlock on startup when the certificate is expired, as the apiserver cannot be started until the cert is updated, but the cert cannot be updated until the apiserver is up.

Fix this by also considering the certificate hash annotation when deciding if the update can be skipped.

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>
(cherry picked from commit 242c2af2db)
Signed-off-by: Brad Davidson <brad.davidson@rancher.com>

.github/workflows/ci.yaml

@@ -17,4 +17,4 @@ jobs:
       uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
       with:
         go-version-file: 'go.mod'
-    - run: go test -race -cover ./...
+    - run: go test -v -race -cover ./...

factory/gen.go

@@ -25,7 +25,7 @@ import (
 const (
 	cnPrefix    = "listener.cattle.io/cn-"
 	Static      = "listener.cattle.io/static"
-	fingerprint = "listener.cattle.io/fingerprint"
+	Fingerprint = "listener.cattle.io/fingerprint"
 )
 
 var (
@@ -189,7 +189,7 @@ func (t *TLS) generateCert(secret *v1.Secret, cn ...string) (*v1.Secret, bool, e
 	secret.Type = v1.SecretTypeTLS
 	secret.Data[v1.TLSCertKey] = certBytes
 	secret.Data[v1.TLSPrivateKeyKey] = keyBytes
-	secret.Annotations[fingerprint] = fmt.Sprintf("SHA1=%X", sha1.Sum(newCert.Raw))
+	secret.Annotations[Fingerprint] = fmt.Sprintf("SHA1=%X", sha1.Sum(newCert.Raw))
 
 	return secret, true, nil
 }

server/server.go

@@ -1,11 +1,13 @@
 package server
 
 import (
+	"bytes"
 	"context"
 	"crypto"
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
+	"io"
 	"log"
 	"net"
 	"net/http"
@@ -40,7 +42,24 @@ type ListenOpts struct {
 	// Override legacy behavior where server logs written to the application's logrus object
 	// were dropped unless logrus was set to debug-level (such as by launching steve with '--debug').
 	// Setting this to true results in server logs appearing at an ERROR level.
-	DisplayServerLogs bool
+	DisplayServerLogs       bool
+	IgnoreTLSHandshakeError bool
+}
+
+var TLSHandshakeError = []byte("http: TLS handshake error")
+
+var _ io.Writer = &TLSErrorDebugger{}
+
+type TLSErrorDebugger struct{}
+
+func (t *TLSErrorDebugger) Write(p []byte) (n int, err error) {
+	p = bytes.TrimSpace(p)
+	if bytes.HasPrefix(p, TLSHandshakeError) {
+		logrus.Debug(string(p))
+	} else {
+		logrus.Error(string(p))
+	}
+	return len(p), err
 }
 
 func ListenAndServe(ctx context.Context, httpsPort, httpPort int, handler http.Handler, opts *ListenOpts) error {
@@ -52,9 +71,15 @@ func ListenAndServe(ctx context.Context, httpsPort, httpPort int, handler http.H
 	if opts.DisplayServerLogs {
 		writer = logger.WriterLevel(logrus.ErrorLevel)
 	}
-	// Otherwise preserve legacy behaviour of displaying server logs only in debug mode.
 
-	errorLog := log.New(writer, "", log.LstdFlags)
+	var errorLog *log.Logger
+	if opts.IgnoreTLSHandshakeError {
+		debugWriter := &TLSErrorDebugger{}
+		errorLog = log.New(debugWriter, "", 0)
+	} else {
+		// Otherwise preserve legacy behaviour of displaying server logs only in debug mode.
+		errorLog = log.New(writer, "", 0)
+	}
 
 	if opts.TLSListenerConfig.TLSConfig == nil {
 		opts.TLSListenerConfig.TLSConfig = &tls.Config{}

server/server_test.go

@@ -39,6 +39,47 @@ func (s *safeWriter) Write(p []byte) (n int, err error) {
 	return s.writer.Write(p)
 }
 
+func TestTLSHandshakeErrorWriter(t *testing.T) {
+	tests := []struct {
+		name                    string
+		ignoreTLSHandshakeError bool
+		message                 []byte
+		expectedLevel           logrus.Level
+	}{
+		{
+			name:          "TLS handshake error is logged as debug",
+			message:       []byte("http: TLS handshake error: EOF"),
+			expectedLevel: logrus.DebugLevel,
+		},
+		{
+			name:          "other errors are logged as error",
+			message:       []byte("some other server error"),
+			expectedLevel: logrus.ErrorLevel,
+		},
+	}
+	var baseLogLevel = logrus.GetLevel()
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert := assertPkg.New(t)
+
+			var buf bytes.Buffer
+			logrus.SetOutput(&buf)
+			logrus.SetLevel(logrus.DebugLevel)
+
+			debugger := &TLSErrorDebugger{}
+			n, err := debugger.Write(tt.message)
+
+			assert.Nil(err)
+			assert.Equal(len(tt.message), n)
+
+			logOutput := buf.String()
+			assert.Contains(logOutput, "level="+tt.expectedLevel.String())
+			assert.Contains(logOutput, string(tt.message))
+		})
+	}
+	logrus.SetLevel(baseLogLevel)
+}
+
 func TestHttpServerLogWithLogrus(t *testing.T) {
 	assert := assertPkg.New(t)
 	message := "debug-level writer"
@@ -84,7 +125,7 @@ func doRequest(safeWriter *safeWriter, message string, logLevel logrus.Level) er
 	msg := fmt.Sprintf("panicking context: %s", message)
 	handler := alwaysPanicHandler{msg: msg}
 	listenOpts := &ListenOpts{
-		BindHost: host,
+		BindHost:          host,
 		DisplayServerLogs: logLevel == logrus.ErrorLevel,
 	}
 

storage/memory/memory.go

@@ -2,6 +2,7 @@ package memory
 
 import (
 	"github.com/rancher/dynamiclistener"
+	"github.com/rancher/dynamiclistener/factory"
 	"github.com/sirupsen/logrus"
 	v1 "k8s.io/api/core/v1"
 )
@@ -32,7 +33,7 @@ func (m *memory) Get() (*v1.Secret, error) {
 }
 
 func (m *memory) Update(secret *v1.Secret) error {
-	if m.secret == nil || m.secret.ResourceVersion == "" || m.secret.ResourceVersion != secret.ResourceVersion {
+	if isChanged(m.secret, secret) {
 		if m.storage != nil {
 			if err := m.storage.Update(secret); err != nil {
 				return err
@@ -44,3 +45,19 @@ func (m *memory) Update(secret *v1.Secret) error {
 	}
 	return nil
 }
+
+func isChanged(old, new *v1.Secret) bool {
+	if old == nil {
+		return true
+	}
+	if old.ResourceVersion == "" {
+		return true
+	}
+	if old.ResourceVersion != new.ResourceVersion {
+		return true
+	}
+	if old.Annotations[factory.Fingerprint] != new.Annotations[factory.Fingerprint] {
+		return true
+	}
+	return false
+}