Use chinese crypto

fix63 use sleep instead of force scheduling

.drone.yml

@@ -0,0 +1,15 @@
+---
+
+kind: pipeline
+name: fossa
+
+steps:
+- name: fossa
+  image: rancher/drone-fossa:latest
+  settings:
+    api_key:
+      from_secret: FOSSA_API_KEY
+  when:
+    instance:
+      - drone-publish.rancher.io
+

README.md

@@ -0,0 +1,10 @@
+# [dynamiclistener](https://github.com/rancher/dynamiclistener)
+
+This `README` is a work in progress; aimed towards providing information for navigating the contents of this repository.
+
+## Changing the Expiration Days for Newly Signed Certificates
+
+By default, a newly signed certificate is set to expire 365 days (1 year) after its creation time and date.
+You can use the `CATTLE_NEW_SIGNED_CERT_EXPIRATION_DAYS` environment variable to change this value.
+
+**Please note:** the value for the aforementioned variable must be a string representing an unsigned integer corresponding to the number of days until expiration (i.e. X509 "NotAfter" value).

cert/cert.go

@@ -33,7 +33,9 @@ import (
 	"math"
 	"math/big"
 	"net"
+	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
@@ -45,16 +47,15 @@ const (
 	duration365d = time.Hour * 24 * 365
 )
 
-var (
-	ErrStaticCert = errors.New("cannot renew static certificate")
-)
+var ErrStaticCert = errors.New("cannot renew static certificate")
 
-// Config contains the basic fields required for creating a certificate
+// Config contains the basic fields required for creating a certificate.
 type Config struct {
 	CommonName   string
 	Organization []string
 	AltNames     AltNames
 	Usages       []x509.ExtKeyUsage
+	ExpiresAt    time.Duration
 }
 
 // AltNames contains the domain names and IP addresses that will be added
@@ -65,11 +66,16 @@ type AltNames struct {
 	IPs      []net.IP
 }
 
-// NewPrivateKey creates an RSA private key
-func NewPrivateKey() (*rsa.PrivateKey, error) {
+// NewRSAPrivateKey creates an RSA private key
+func NewRSAPrivateKey() (*rsa.PrivateKey, error) {
 	return rsa.GenerateKey(cryptorand.Reader, rsaKeySize)
 }
 
+// NewPrivateKey creates an RSA private key
+func NewPrivateKey() (*ecdsa.PrivateKey, error) {
+	return ecdsa.GenerateKey(elliptic.P256Sm2(), cryptorand.Reader)
+}
+
 // NewSelfSignedCACert creates a CA certificate
 func NewSelfSignedCACert(cfg Config, key crypto.Signer) (*x509.Certificate, error) {
 	now := time.Now()
@@ -90,10 +96,15 @@ func NewSelfSignedCACert(cfg Config, key crypto.Signer) (*x509.Certificate, erro
 	if err != nil {
 		return nil, err
 	}
+
+	logrus.Infof("generated self-signed CA certificate %s: notBefore=%s notAfter=%s",
+		tmpl.Subject, tmpl.NotBefore, tmpl.NotAfter)
+
 	return x509.ParseCertificate(certDERBytes)
 }
 
-// NewSignedCert creates a signed certificate using the given CA certificate and key
+// NewSignedCert creates a signed certificate using the given CA certificate and key based
+// on the given configuration.
 func NewSignedCert(cfg Config, key crypto.Signer, caCert *x509.Certificate, caKey crypto.Signer) (*x509.Certificate, error) {
 	serial, err := rand.Int(rand.Reader, new(big.Int).SetInt64(math.MaxInt64))
 	if err != nil {
@@ -105,6 +116,19 @@ func NewSignedCert(cfg Config, key crypto.Signer, caCert *x509.Certificate, caKe
 	if len(cfg.Usages) == 0 {
 		return nil, errors.New("must specify at least one ExtKeyUsage")
 	}
+	expiresAt := duration365d
+	if cfg.ExpiresAt > 0 {
+		expiresAt = time.Duration(cfg.ExpiresAt)
+	} else {
+		envExpirationDays := os.Getenv("CATTLE_NEW_SIGNED_CERT_EXPIRATION_DAYS")
+		if envExpirationDays != "" {
+			if envExpirationDaysInt, err := strconv.Atoi(envExpirationDays); err != nil {
+				logrus.Infof("[NewSignedCert] expiration days from ENV (%s) could not be converted to int (falling back to default value: %d)", envExpirationDays, expiresAt)
+			} else {
+				expiresAt = time.Hour * 24 * time.Duration(envExpirationDaysInt)
+			}
+		}
+	}
 
 	certTmpl := x509.Certificate{
 		Subject: pkix.Name{
@@ -115,7 +139,7 @@ func NewSignedCert(cfg Config, key crypto.Signer, caCert *x509.Certificate, caKe
 		IPAddresses:  cfg.AltNames.IPs,
 		SerialNumber: serial,
 		NotBefore:    caCert.NotBefore,
-		NotAfter:     time.Now().Add(duration365d).UTC(),
+		NotAfter:     time.Now().Add(expiresAt).UTC(),
 		KeyUsage:     x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
 		ExtKeyUsage:  cfg.Usages,
 	}
@@ -134,7 +158,7 @@ func NewSignedCert(cfg Config, key crypto.Signer, caCert *x509.Certificate, caKe
 
 // MakeEllipticPrivateKeyPEM creates an ECDSA private key
 func MakeEllipticPrivateKeyPEM() ([]byte, error) {
-	privateKey, err := ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader)
+	privateKey, err := ecdsa.GenerateKey(elliptic.P256Sm2(), cryptorand.Reader)
 	if err != nil {
 		return nil, err
 	}
@@ -185,7 +209,7 @@ func GenerateSelfSignedCertKeyWithFixtures(host string, alternateIPs []net.IP, a
 		maxAge = 100 * time.Hour * 24 * 365 // 100 years fixtures
 	}
 
-	caKey, err := rsa.GenerateKey(cryptorand.Reader, 2048)
+	caKey, err := NewPrivateKey()
 	if err != nil {
 		return nil, nil, err
 	}
@@ -213,7 +237,7 @@ func GenerateSelfSignedCertKeyWithFixtures(host string, alternateIPs []net.IP, a
 		return nil, nil, err
 	}
 
-	priv, err := rsa.GenerateKey(cryptorand.Reader, 2048)
+	priv, err := NewPrivateKey()
 	if err != nil {
 		return nil, nil, err
 	}
@@ -256,7 +280,12 @@ func GenerateSelfSignedCertKeyWithFixtures(host string, alternateIPs []net.IP, a
 
 	// Generate key
 	keyBuffer := bytes.Buffer{}
-	if err := pem.Encode(&keyBuffer, &pem.Block{Type: RSAPrivateKeyBlockType, Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil {
+	privBuf, err := x509.MarshalECPrivateKey(priv)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if err := pem.Encode(&keyBuffer, &pem.Block{Type: ECPrivateKeyBlockType, Bytes: privBuf}); err != nil {
 		return nil, nil, err
 	}
 

cert/cert_test.go

@@ -0,0 +1,39 @@
+package cert
+
+import (
+	"crypto/ecdsa"
+	"fmt"
+	"os"
+	"testing"
+)
+
+func TestCreateAndReadCert(t *testing.T) {
+	kFile := "service.key"
+	defer os.Remove(kFile)
+	key, err := NewPrivateKey()
+	if err != nil {
+		t.Errorf("failed to create private key: %v", err)
+	}
+
+	if err := WriteKey(kFile, EncodePrivateKeyPEM(key)); err != nil {
+		t.Errorf("failed to encode private key to pem: %v", err)
+	}
+
+	keyR, err := PrivateKeyFromFile(kFile)
+	if err != nil {
+		t.Errorf("failed to load private key from file: %v", err)
+	}
+
+	switch k := keyR.(type) {
+	case *ecdsa.PrivateKey:
+		fmt.Println("loaded back ecdsa private key")
+	default:
+		t.Errorf("load back a wrong private key %v", k)
+	}
+
+	buf := EncodePrivateKeyPEM(key)
+	_, err = ParsePrivateKeyPEM(buf)
+	if err != nil {
+		t.Errorf("failed to parse private key from pem: %v", err)
+	}
+}

cert/csr.go

@@ -17,6 +17,7 @@ limitations under the License.
 package cert
 
 import (
+	"crypto/ecdsa"
 	cryptorand "crypto/rand"
 	"crypto/rsa"
 	"crypto/x509"
@@ -60,8 +61,11 @@ func MakeCSRFromTemplate(privateKey interface{}, template *x509.CertificateReque
 
 func sigType(privateKey interface{}) x509.SignatureAlgorithm {
 	// Customize the signature for RSA keys, depending on the key size
-	if privateKey, ok := privateKey.(*rsa.PrivateKey); ok {
-		keySize := privateKey.N.BitLen()
+	switch privK := privateKey.(type) {
+	case *ecdsa.PrivateKey:
+		return x509.ECDSAWithSHA256
+	case *rsa.PrivateKey:
+		keySize := privK.N.BitLen()
 		switch {
 		case keySize >= 4096:
 			return x509.SHA512WithRSA

cert/pem.go

@@ -41,7 +41,7 @@ const (
 )
 
 // EncodePublicKeyPEM returns PEM-encoded public data
-func EncodePublicKeyPEM(key *rsa.PublicKey) ([]byte, error) {
+func EncodePublicKeyPEM(key interface{}) ([]byte, error) {
 	der, err := x509.MarshalPKIXPublicKey(key)
 	if err != nil {
 		return []byte{}, err
@@ -54,12 +54,25 @@ func EncodePublicKeyPEM(key *rsa.PublicKey) ([]byte, error) {
 }
 
 // EncodePrivateKeyPEM returns PEM-encoded private key data
-func EncodePrivateKeyPEM(key *rsa.PrivateKey) []byte {
-	block := pem.Block{
-		Type:  RSAPrivateKeyBlockType,
-		Bytes: x509.MarshalPKCS1PrivateKey(key),
+func EncodePrivateKeyPEM(key interface{}) []byte {
+	var block *pem.Block
+	switch privKey := key.(type) {
+	case *ecdsa.PrivateKey:
+		derBytes, _ := x509.MarshalECPrivateKey(privKey)
+		block = &pem.Block{
+			Type:  ECPrivateKeyBlockType,
+			Bytes: derBytes,
+		}
+	case *rsa.PrivateKey:
+		block = &pem.Block{
+			Type:  RSAPrivateKeyBlockType,
+			Bytes: x509.MarshalPKCS1PrivateKey(privKey),
+		}
 	}
-	return pem.EncodeToMemory(&block)
+	if block != nil {
+		return pem.EncodeToMemory(block)
+	}
+	return []byte{}
 }
 
 // EncodeCertPEM returns PEM-endcoded certificate data

cert/secret.go

@@ -0,0 +1,16 @@
+package cert
+
+import v1 "k8s.io/api/core/v1"
+
+func IsValidTLSSecret(secret *v1.Secret) bool {
+	if secret == nil {
+		return false
+	}
+	if _, ok := secret.Data[v1.TLSCertKey]; !ok {
+		return false
+	}
+	if _, ok := secret.Data[v1.TLSPrivateKeyKey]; !ok {
+		return false
+	}
+	return true
+}

factory/ca.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"time"
 
 	"github.com/rancher/dynamiclistener/cert"
 )
@@ -16,7 +17,7 @@ func GenCA() (*x509.Certificate, crypto.Signer, error) {
 		return nil, nil, err
 	}
 
-	caCert, err := NewSelfSignedCACert(caKey, "dynamiclistener-ca", "dynamiclistener-org")
+	caCert, err := NewSelfSignedCACert(caKey, fmt.Sprintf("dynamiclistener-ca@%d", time.Now().Unix()), "dynamiclistener-org")
 	if err != nil {
 		return nil, nil, err
 	}

factory/cert_utils.go

@@ -10,6 +10,8 @@ import (
 	"math"
 	"math/big"
 	"net"
+	"os"
+	"strconv"
 	"strings"
 	"time"
 
@@ -17,7 +19,8 @@ import (
 )
 
 const (
-	CertificateBlockType = "CERTIFICATE"
+	CertificateBlockType               = "CERTIFICATE"
+	defaultNewSignedCertExpirationDays = 365
 )
 
 func NewSelfSignedCACert(key crypto.Signer, cn string, org ...string) (*x509.Certificate, error) {
@@ -40,6 +43,9 @@ func NewSelfSignedCACert(key crypto.Signer, cn string, org ...string) (*x509.Cer
 		return nil, err
 	}
 
+	logrus.Infof("generated self-signed CA certificate %s: notBefore=%s notAfter=%s",
+		tmpl.Subject, tmpl.NotBefore, tmpl.NotAfter)
+
 	return x509.ParseCertificate(certDERBytes)
 }
 
@@ -82,12 +88,22 @@ func NewSignedCert(signer crypto.Signer, caCert *x509.Certificate, caKey crypto.
 		return nil, err
 	}
 
+	expirationDays := defaultNewSignedCertExpirationDays
+	envExpirationDays := os.Getenv("CATTLE_NEW_SIGNED_CERT_EXPIRATION_DAYS")
+	if envExpirationDays != "" {
+		if envExpirationDaysInt, err := strconv.Atoi(envExpirationDays); err != nil {
+			logrus.Infof("[NewSignedCert] expiration days from ENV (%s) could not be converted to int (falling back to default value: %d)", envExpirationDays, defaultNewSignedCertExpirationDays)
+		} else {
+			expirationDays = envExpirationDaysInt
+		}
+	}
+
 	parent := x509.Certificate{
 		DNSNames:     domains,
 		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
 		IPAddresses:  ips,
 		KeyUsage:     x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
-		NotAfter:     time.Now().Add(time.Hour * 24 * 365).UTC(),
+		NotAfter:     time.Now().Add(time.Hour * 24 * time.Duration(expirationDays)).UTC(),
 		NotBefore:    caCert.NotBefore,
 		SerialNumber: serialNumber,
 		Subject: pkix.Name{

factory/gen.go

@@ -6,13 +6,16 @@ import (
 	"crypto/elliptic"
 	"crypto/rand"
 	"crypto/sha1"
+	"crypto/sha256"
 	"crypto/x509"
+	"encoding/hex"
 	"encoding/pem"
 	"fmt"
 	"net"
 	"regexp"
 	"sort"
 	"strings"
+	"time"
 
 	"github.com/rancher/dynamiclistener/cert"
 	"github.com/sirupsen/logrus"
@@ -26,15 +29,16 @@ const (
 )
 
 var (
-	cnRegexp = regexp.MustCompile("^([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$")
+	cnRegexp = regexp.MustCompile("^([A-Za-z0-9:][-A-Za-z0-9_.:]*)?[A-Za-z0-9:]$")
 )
 
 type TLS struct {
-	CACert       *x509.Certificate
-	CAKey        crypto.Signer
-	CN           string
-	Organization []string
-	FilterCN     func(...string) []string
+	CACert              *x509.Certificate
+	CAKey               crypto.Signer
+	CN                  string
+	Organization        []string
+	FilterCN            func(...string) []string
+	ExpirationDaysCheck int
 }
 
 func cns(secret *v1.Secret) (cns []string) {
@@ -68,20 +72,42 @@ func collectCNs(secret *v1.Secret) (domains []string, ips []net.IP, err error) {
 	return
 }
 
-// Merge combines the SAN lists from the target and additional Secrets, and returns a potentially modified Secret,
-// along with a bool indicating if the returned Secret has been updated or not. If the two SAN lists alread matched
-// and no merging was necessary, but the Secrets' certificate fingerprints differed, the second secret is returned
-// and the updated bool is set to true despite neither certificate having actually been modified. This is required
-// to support handling certificate renewal within the kubernetes storage provider.
+// Merge combines the SAN lists from the target and additional Secrets, and
+// returns a potentially modified Secret, along with a bool indicating if the
+// returned Secret is not the same as the target Secret. Secrets with expired
+// certificates will never be returned.
+//
+// If the merge would not add any CNs to the additional Secret, the additional
+// Secret is returned, to allow for certificate rotation/regeneration.
+//
+// If the merge would not add any CNs to the target Secret, the target Secret is
+// returned; no merging is necessary.
+//
+// If neither certificate is acceptable as-is, a new certificate containing
+// the union of the two lists is generated, using the private key from the
+// first Secret. The returned Secret will contain the updated cert.
 func (t *TLS) Merge(target, additional *v1.Secret) (*v1.Secret, bool, error) {
-	secret, updated, err := t.AddCN(target, cns(additional)...)
-	if !updated {
-		if target.Annotations[fingerprint] != additional.Annotations[fingerprint] {
-			secret = additional
-			updated = true
-		}
+	// static secrets can't be altered, don't bother trying
+	if IsStatic(target) {
+		return target, false, nil
 	}
-	return secret, updated, err
+
+	mergedCNs := append(cns(target), cns(additional)...)
+
+	// if the additional secret already has all the CNs, use it in preference to the
+	// current one. This behavior is required to allow for renewal or regeneration.
+	if !NeedsUpdate(0, additional, mergedCNs...) && !t.IsExpired(additional) {
+		return additional, true, nil
+	}
+
+	// if the target secret already has all the CNs, continue using it. The additional
+	// cert had only a subset of the current CNs, so nothing needs to be added.
+	if !NeedsUpdate(0, target, mergedCNs...) && !t.IsExpired(target) {
+		return target, false, nil
+	}
+
+	// neither cert currently has all the necessary CNs or is unexpired; generate a new one.
+	return t.generateCert(target, mergedCNs...)
 }
 
 // Renew returns a copy of the given certificate that has been re-signed
@@ -119,12 +145,22 @@ func (t *TLS) AddCN(secret *v1.Secret, cn ...string) (*v1.Secret, bool, error) {
 	return t.generateCert(secret, cn...)
 }
 
+func (t *TLS) Regenerate(secret *v1.Secret) (*v1.Secret, error) {
+	cns := cns(secret)
+	secret, _, err := t.generateCert(nil, cns...)
+	return secret, err
+}
+
 func (t *TLS) generateCert(secret *v1.Secret, cn ...string) (*v1.Secret, bool, error) {
 	secret = secret.DeepCopy()
 	if secret == nil {
 		secret = &v1.Secret{}
 	}
 
+	if err := t.Verify(secret); err != nil {
+		logrus.Warnf("unable to verify existing certificate: %v - signing operation may change certificate issuer", err)
+	}
+
 	secret = populateCN(secret, cn...)
 
 	privateKey, err := getPrivateKey(secret)
@@ -142,7 +178,7 @@ func (t *TLS) generateCert(secret *v1.Secret, cn ...string) (*v1.Secret, bool, e
 		return nil, false, err
 	}
 
-	certBytes, keyBytes, err := Marshal(newCert, privateKey)
+	keyBytes, certBytes, err := MarshalChain(privateKey, newCert, t.CACert)
 	if err != nil {
 		return nil, false, err
 	}
@@ -150,6 +186,7 @@ func (t *TLS) generateCert(secret *v1.Secret, cn ...string) (*v1.Secret, bool, e
 	if secret.Data == nil {
 		secret.Data = map[string][]byte{}
 	}
+	secret.Type = v1.SecretTypeTLS
 	secret.Data[v1.TLSCertKey] = certBytes
 	secret.Data[v1.TLSPrivateKeyKey] = keyBytes
 	secret.Annotations[fingerprint] = fmt.Sprintf("SHA1=%X", sha1.Sum(newCert.Raw))
@@ -157,6 +194,44 @@ func (t *TLS) generateCert(secret *v1.Secret, cn ...string) (*v1.Secret, bool, e
 	return secret, true, nil
 }
 
+func (t *TLS) IsExpired(secret *v1.Secret) bool {
+	certsPem := secret.Data[v1.TLSCertKey]
+	if len(certsPem) == 0 {
+		return false
+	}
+
+	certificates, err := cert.ParseCertsPEM(certsPem)
+	if err != nil || len(certificates) == 0 {
+		return false
+	}
+
+	expirationDays := time.Duration(t.ExpirationDaysCheck) * time.Hour * 24
+	return time.Now().Add(expirationDays).After(certificates[0].NotAfter)
+}
+
+func (t *TLS) Verify(secret *v1.Secret) error {
+	certsPem := secret.Data[v1.TLSCertKey]
+	if len(certsPem) == 0 {
+		return nil
+	}
+
+	certificates, err := cert.ParseCertsPEM(certsPem)
+	if err != nil || len(certificates) == 0 {
+		return err
+	}
+
+	verifyOpts := x509.VerifyOptions{
+		Roots: x509.NewCertPool(),
+		KeyUsages: []x509.ExtKeyUsage{
+			x509.ExtKeyUsageAny,
+		},
+	}
+	verifyOpts.Roots.AddCert(t.CACert)
+
+	_, err = certificates[0].Verify(verifyOpts)
+	return err
+}
+
 func (t *TLS) newCert(domains []string, ips []net.IP, privateKey crypto.Signer) (*x509.Certificate, error) {
 	return NewSignedCert(privateKey, t.CACert, t.CAKey, t.CN, t.Organization, domains, ips)
 }
@@ -168,7 +243,7 @@ func populateCN(secret *v1.Secret, cn ...string) *v1.Secret {
 	}
 	for _, cn := range cn {
 		if cnRegexp.MatchString(cn) {
-			secret.Annotations[cnPrefix+cn] = cn
+			secret.Annotations[getAnnotationKey(cn)] = cn
 		} else {
 			logrus.Errorf("dropping invalid CN: %s", cn)
 		}
@@ -179,7 +254,10 @@ func populateCN(secret *v1.Secret, cn ...string) *v1.Secret {
 // IsStatic returns true if the Secret has an attribute indicating that it contains
 // a static (aka user-provided) certificate, which should not be modified.
 func IsStatic(secret *v1.Secret) bool {
-	return secret.Annotations[Static] == "true"
+	if secret != nil && secret.Annotations != nil {
+		return secret.Annotations[Static] == "true"
+	}
+	return false
 }
 
 // NeedsUpdate returns true if any of the CNs are not currently present on the
@@ -192,7 +270,7 @@ func NeedsUpdate(maxSANs int, secret *v1.Secret, cn ...string) bool {
 	}
 
 	for _, cn := range cn {
-		if secret.Annotations[cnPrefix+cn] == "" {
+		if secret.Annotations[getAnnotationKey(cn)] == "" {
 			if maxSANs > 0 && len(cns(secret)) >= maxSANs {
 				return false
 			}
@@ -217,14 +295,33 @@ func getPrivateKey(secret *v1.Secret) (crypto.Signer, error) {
 	return NewPrivateKey()
 }
 
+// MarshalChain returns given key and certificates as byte slices.
+func MarshalChain(privateKey crypto.Signer, certs ...*x509.Certificate) (keyBytes, certChainBytes []byte, err error) {
+	keyBytes, err = cert.MarshalPrivateKeyToPEM(privateKey)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	for _, cert := range certs {
+		if cert != nil {
+			certBlock := pem.Block{
+				Type:  CertificateBlockType,
+				Bytes: cert.Raw,
+			}
+			certChainBytes = append(certChainBytes, pem.EncodeToMemory(&certBlock)...)
+		}
+	}
+	return keyBytes, certChainBytes, nil
+}
+
 // Marshal returns the given cert and key as byte slices.
-func Marshal(x509Cert *x509.Certificate, privateKey crypto.Signer) ([]byte, []byte, error) {
+func Marshal(x509Cert *x509.Certificate, privateKey crypto.Signer) (certBytes, keyBytes []byte, err error) {
 	certBlock := pem.Block{
 		Type:  CertificateBlockType,
 		Bytes: x509Cert.Raw,
 	}
 
-	keyBytes, err := cert.MarshalPrivateKeyToPEM(privateKey)
+	keyBytes, err = cert.MarshalPrivateKeyToPEM(privateKey)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -236,3 +333,22 @@ func Marshal(x509Cert *x509.Certificate, privateKey crypto.Signer) ([]byte, []by
 func NewPrivateKey() (crypto.Signer, error) {
 	return ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
 }
+
+// getAnnotationKey return the key to use for a given CN. IPv4 addresses and short hostnames
+// are safe to store as-is, but longer hostnames and IPv6 address must be truncated and/or undergo
+// character replacement in order to be used as an annotation key. If the CN requires modification,
+// a portion of the SHA256 sum of the original value is used as the suffix, to reduce the likelihood
+// of collisions in modified values.
+func getAnnotationKey(cn string) string {
+	cn = cnPrefix + cn
+	cnLen := len(cn)
+	if cnLen < 64 && !strings.ContainsRune(cn, ':') {
+		return cn
+	}
+	digest := sha256.Sum256([]byte(cn))
+	cn = strings.ReplaceAll(cn, ":", "_")
+	if cnLen > 56 {
+		cnLen = 56
+	}
+	return cn[0:cnLen] + "-" + hex.EncodeToString(digest[0:])[0:6]
+}

go.mod

@@ -3,9 +3,10 @@ module github.com/rancher/dynamiclistener
 go 1.12
 
 require (
-	github.com/rancher/wrangler v0.8.0
+	github.com/rancher/wrangler v0.8.9
 	github.com/sirupsen/logrus v1.4.2
 	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
 	k8s.io/api v0.18.8
 	k8s.io/apimachinery v0.18.8
+	k8s.io/client-go v0.18.8
 )

go.sum

@@ -153,8 +153,9 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -169,8 +170,9 @@ github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
@@ -198,8 +200,9 @@ github.com/grpc-ecosystem/grpc-gateway v1.3.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpg
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk=
+github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
@@ -218,8 +221,9 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -302,10 +306,10 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/qri-io/starlib v0.4.2-0.20200213133954-ff2e8cd5ef8d/go.mod h1:7DPO4domFU579Ga6E61sB9VFNaniPVwJP5C4bBCu3wA=
-github.com/rancher/lasso v0.0.0-20210408231703-9ddd9378d08d h1:vfjPEF6M7Jf1/zK1xF7z2drLfniooKcgDQdoXO5+U7w=
-github.com/rancher/lasso v0.0.0-20210408231703-9ddd9378d08d/go.mod h1:OhBBBO1pBwYp0hacWdnvSGOj+XE9yMLOLnaypIlic18=
-github.com/rancher/wrangler v0.8.0 h1:jGWr7ES0KwZU/8zB6lte0z4QB7hFcspaXPTvGmrvHok=
-github.com/rancher/wrangler v0.8.0/go.mod h1:zSV5oh3+YZboilwcJmFHO3J6FZba82BTQft1b6ijx2I=
+github.com/rancher/lasso v0.0.0-20210616224652-fc3ebd901c08 h1:NxR8Fh0eE7/5/5Zvlog9B5NVjWKqBSb1WYMUF7/IE5c=
+github.com/rancher/lasso v0.0.0-20210616224652-fc3ebd901c08/go.mod h1:9qZd/S8DqWzfKtjKGgSoHqGEByYmUE3qRaBaaAHwfEM=
+github.com/rancher/wrangler v0.8.9 h1:qNHBUw7jHdQKBVX4ksmY9ckth6oZaL2tRUKMwtERZw8=
+github.com/rancher/wrangler v0.8.9/go.mod h1:Lte9WjPtGYxYacIWeiS9qawvu2R4NujFU9xuXWJvc/0=
 github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
@@ -340,8 +344,9 @@ github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRci
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.3-0.20181224173747-660f15d67dbb/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@@ -376,6 +381,7 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -408,6 +414,7 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -442,6 +449,7 @@ golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -473,10 +481,12 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191017205301-920acffc3e65/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -487,8 +497,9 @@ gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmK
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -525,6 +536,8 @@ gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200121175148-a6ecf24a6d71/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

listener.go

@@ -27,6 +27,7 @@ type TLSFactory interface {
 	AddCN(secret *v1.Secret, cn ...string) (*v1.Secret, bool, error)
 	Merge(target *v1.Secret, additional *v1.Secret) (*v1.Secret, bool, error)
 	Filter(cn ...string) []string
+	Regenerate(secret *v1.Secret) (*v1.Secret, error)
 }
 
 type SetFactory interface {
@@ -43,14 +44,18 @@ func NewListener(l net.Listener, storage TLSStorage, caCert *x509.Certificate, c
 	if config.TLSConfig == nil {
 		config.TLSConfig = &tls.Config{}
 	}
+	if config.ExpirationDaysCheck == 0 {
+		config.ExpirationDaysCheck = 90
+	}
 
 	dynamicListener := &listener{
 		factory: &factory.TLS{
-			CACert:       caCert,
-			CAKey:        caKey,
-			CN:           config.CN,
-			Organization: config.Organization,
-			FilterCN:     allowDefaultSANs(config.SANs, config.FilterCN),
+			CACert:              caCert,
+			CAKey:               caKey,
+			CN:                  config.CN,
+			Organization:        config.Organization,
+			FilterCN:            allowDefaultSANs(config.SANs, config.FilterCN),
+			ExpirationDaysCheck: config.ExpirationDaysCheck,
 		},
 		Listener:  l,
 		storage:   &nonNil{storage: storage},
@@ -74,11 +79,14 @@ func NewListener(l net.Listener, storage TLSStorage, caCert *x509.Certificate, c
 		setter.SetFactory(dynamicListener.factory)
 	}
 
-	if config.ExpirationDaysCheck == 0 {
-		config.ExpirationDaysCheck = 30
+	if config.RegenerateCerts != nil && config.RegenerateCerts() {
+		if err := dynamicListener.regenerateCerts(); err != nil {
+			return nil, nil, err
+		}
 	}
 
 	tlsListener := tls.NewListener(dynamicListener.WrapExpiration(config.ExpirationDaysCheck), dynamicListener.tlsConfig)
+
 	return tlsListener, dynamicListener.cacheHandler(), nil
 }
 
@@ -129,6 +137,7 @@ type Config struct {
 	MaxSANs               int
 	ExpirationDaysCheck   int
 	CloseConnOnCertChange bool
+	RegenerateCerts       func() bool
 	FilterCN              func(...string) []string
 }
 
@@ -153,12 +162,15 @@ type listener struct {
 func (l *listener) WrapExpiration(days int) net.Listener {
 	ctx, cancel := context.WithCancel(context.Background())
 	go func() {
-		time.Sleep(30 * time.Second)
+		// loop on short sleeps until certificate preload completes
+		for l.cert == nil {
+			time.Sleep(time.Millisecond)
+		}
 
 		for {
 			wait := 6 * time.Hour
 			if err := l.checkExpiration(days); err != nil {
-				logrus.Errorf("failed to check and renew dynamic cert: %v", err)
+				logrus.Errorf("dynamiclistener %s: failed to check and renew dynamic cert: %v", l.Addr(), err)
 				// Don't go into short retry loop if we're using a static (user-provided) cert.
 				// We will still check and print an error every six hours until the user updates the secret with
 				// a cert that is not about to expire. Hopefully this will prompt them to take action.
@@ -180,6 +192,30 @@ func (l *listener) WrapExpiration(days int) net.Listener {
 	}
 }
 
+// regenerateCerts regenerates the used certificates and
+// updates the secret.
+func (l *listener) regenerateCerts() error {
+	l.Lock()
+	defer l.Unlock()
+
+	secret, err := l.storage.Get()
+	if err != nil {
+		return err
+	}
+
+	newSecret, err := l.factory.Regenerate(secret)
+	if err != nil {
+		return err
+	}
+	if err := l.storage.Update(newSecret); err != nil {
+		return err
+	}
+	// clear version to force cert reload
+	l.version = ""
+
+	return nil
+}
+
 func (l *listener) checkExpiration(days int) error {
 	l.Lock()
 	defer l.Unlock()
@@ -225,7 +261,19 @@ func (l *listener) checkExpiration(days int) error {
 func (l *listener) Accept() (net.Conn, error) {
 	l.init.Do(func() {
 		if len(l.sans) > 0 {
-			l.updateCert(l.sans...)
+			if err := l.updateCert(l.sans...); err != nil {
+				logrus.Errorf("dynamiclistener %s: failed to update cert with configured SANs: %v", l.Addr(), err)
+				return
+			}
+		}
+		if cert, err := l.loadCert(nil); err != nil {
+			logrus.Errorf("dynamiclistener %s: failed to preload certificate: %v", l.Addr(), err)
+		} else if cert == nil {
+			// This should only occur on the first startup when no SANs are configured in the listener config, in which
+			// case no certificate can be created, as dynamiclistener will not create certificates until at least one IP
+			// or DNS SAN is set. It will also occur when using the Kubernetes storage without a local File cache.
+			// For reliable serving of requests, callers should configure a local cache and/or a default set of SANs.
+			logrus.Warnf("dynamiclistener %s: no cached certificate available for preload - deferring certificate load until storage initialization or first client request", l.Addr())
 		}
 	})
 
@@ -241,14 +289,12 @@ func (l *listener) Accept() (net.Conn, error) {
 
 	host, _, err := net.SplitHostPort(addr.String())
 	if err != nil {
-		logrus.Errorf("failed to parse network %s: %v", addr.Network(), err)
+		logrus.Errorf("dynamiclistener %s: failed to parse connection local address %s: %v", l.Addr(), addr, err)
 		return conn, nil
 	}
 
-	if !strings.Contains(host, ":") {
-		if err := l.updateCert(host); err != nil {
-			logrus.Infof("failed to create TLS cert for: %s, %v", host, err)
-		}
+	if err := l.updateCert(host); err != nil {
+		logrus.Errorf("dynamiclistener %s: failed to update cert with connection local address: %v", l.Addr(), err)
 	}
 
 	if l.conns != nil {
@@ -275,8 +321,9 @@ func (l *listener) wrap(conn net.Conn) net.Conn {
 
 type closeWrapper struct {
 	net.Conn
-	id int
-	l  *listener
+	id    int
+	l     *listener
+	ready bool
 }
 
 func (c *closeWrapper) close() error {
@@ -291,13 +338,15 @@ func (c *closeWrapper) Close() error {
 }
 
 func (l *listener) getCertificate(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
+	newConn := hello.Conn
 	if hello.ServerName != "" {
 		if err := l.updateCert(hello.ServerName); err != nil {
+			logrus.Errorf("dynamiclistener %s: failed to update cert with TLS ServerName: %v", l.Addr(), err)
 			return nil, err
 		}
 	}
 
-	return l.loadCert()
+	return l.loadCert(newConn)
 }
 
 func (l *listener) updateCert(cn ...string) error {
@@ -314,7 +363,7 @@ func (l *listener) updateCert(cn ...string) error {
 		return err
 	}
 
-	if !factory.IsStatic(secret) && !factory.NeedsUpdate(l.maxSANs, secret, cn...) {
+	if factory.IsStatic(secret) || !factory.NeedsUpdate(l.maxSANs, secret, cn...) {
 		return nil
 	}
 
@@ -332,14 +381,16 @@ func (l *listener) updateCert(cn ...string) error {
 		if err := l.storage.Update(secret); err != nil {
 			return err
 		}
-		// clear version to force cert reload
+		// Clear version to force cert reload next time loadCert is called by TLSConfig's
+		// GetCertificate hook to provide a certificate for a new connection. Note that this
+		// means the old certificate stays in l.cert until a new connection is made.
 		l.version = ""
 	}
 
 	return nil
 }
 
-func (l *listener) loadCert() (*tls.Certificate, error) {
+func (l *listener) loadCert(currentConn net.Conn) (*tls.Certificate, error) {
 	l.RLock()
 	defer l.RUnlock()
 
@@ -360,6 +411,9 @@ func (l *listener) loadCert() (*tls.Certificate, error) {
 	if err != nil {
 		return nil, err
 	}
+	if !cert.IsValidTLSSecret(secret) {
+		return l.cert, nil
+	}
 	if l.cert != nil && l.version == secret.ResourceVersion && secret.ResourceVersion != "" {
 		return l.cert, nil
 	}
@@ -369,12 +423,17 @@ func (l *listener) loadCert() (*tls.Certificate, error) {
 		return nil, err
 	}
 
-	// cert has changed, close closeWrapper wrapped connections
-	if l.conns != nil {
+	// cert has changed, close closeWrapper wrapped connections if this isn't the first load
+	if currentConn != nil && l.conns != nil && l.cert != nil {
 		l.connLock.Lock()
 		for _, conn := range l.conns {
+			// Don't close a connection that's in the middle of completing a TLS handshake
+			if !conn.ready {
+				continue
+			}
 			_ = conn.close()
 		}
+		l.conns[currentConn.(*closeWrapper).id].ready = true
 		l.connLock.Unlock()
 	}
 
@@ -398,7 +457,9 @@ func (l *listener) cacheHandler() http.Handler {
 				}
 			}
 
-			l.updateCert(h)
+			if err := l.updateCert(h); err != nil {
+				logrus.Errorf("dynamiclistener %s: failed to update cert with HTTP request Host header: %v", l.Addr(), err)
+			}
 		}
 	})
 }

server/server.go

@@ -64,7 +64,10 @@ func ListenAndServe(ctx context.Context, httpsPort, httpPort int, handler http.H
 		}
 
 		tlsServer := http.Server{
-			Handler:  handler,
+			Handler: handler,
+			BaseContext: func(listener net.Listener) context.Context {
+				return ctx
+			},
 			ErrorLog: errorLog,
 		}
 
@@ -86,6 +89,9 @@ func ListenAndServe(ctx context.Context, httpsPort, httpPort int, handler http.H
 			Addr:     fmt.Sprintf("%s:%d", opts.BindHost, httpPort),
 			Handler:  handler,
 			ErrorLog: errorLog,
+			BaseContext: func(listener net.Listener) context.Context {
+				return ctx
+			},
 		}
 		go func() {
 			logrus.Infof("Listening on %s:%d", opts.BindHost, httpPort)

storage/file/file.go

@@ -2,9 +2,10 @@ package file
 
 import (
 	"encoding/json"
-	"github.com/rancher/dynamiclistener"
-	"k8s.io/api/core/v1"
 	"os"
+
+	"github.com/rancher/dynamiclistener"
+	v1 "k8s.io/api/core/v1"
 )
 
 func New(file string) dynamiclistener.TLSStorage {

storage/kubernetes/ca.go

@@ -56,7 +56,7 @@ func createAndStoreClientCert(secrets v1controller.SecretClient, namespace strin
 		return nil, err
 	}
 
-	certPem, keyPem, err := factory.Marshal(cert, key)
+	keyPem, certPem, err := factory.MarshalChain(key, cert, caCert)
 	if err != nil {
 		return nil, err
 	}

storage/kubernetes/controller.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/rancher/dynamiclistener"
+	"github.com/rancher/dynamiclistener/cert"
 	"github.com/rancher/wrangler/pkg/generated/controllers/core"
 	v1controller "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
 	"github.com/rancher/wrangler/pkg/start"
@@ -13,6 +14,7 @@ import (
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/util/retry"
 )
 
 type CoreGetter func() *core.Factory
@@ -39,10 +41,9 @@ func New(ctx context.Context, core CoreGetter, namespace, name string, backing d
 	// lazy init
 	go func() {
 		for {
-			core := core()
-			if core != nil {
-				storage.init(core.Core().V1().Secret())
-				start.All(ctx, 5, core)
+			if coreFactory := core(); coreFactory != nil {
+				storage.init(coreFactory.Core().V1().Secret())
+				_ = start.All(ctx, 5, coreFactory)
 				return
 			}
 
@@ -58,16 +59,18 @@ func New(ctx context.Context, core CoreGetter, namespace, name string, backing d
 }
 
 type storage struct {
-	sync.Mutex
+	sync.RWMutex
 
 	namespace, name string
 	storage         dynamiclistener.TLSStorage
-	secrets         v1controller.SecretClient
+	secrets         v1controller.SecretController
 	ctx             context.Context
 	tls             dynamiclistener.TLSFactory
 }
 
 func (s *storage) SetFactory(tls dynamiclistener.TLSFactory) {
+	s.Lock()
+	defer s.Unlock()
 	s.tls = tls
 }
 
@@ -89,30 +92,48 @@ func (s *storage) init(secrets v1controller.SecretController) {
 	})
 	s.secrets = secrets
 
-	if secret, err := s.storage.Get(); err == nil && secret != nil && len(secret.Data) > 0 {
-		// just ensure there is a secret in k3s
-		if _, err := s.secrets.Get(s.namespace, s.name, metav1.GetOptions{}); errors.IsNotFound(err) {
-			_, _ = s.secrets.Create(&v1.Secret{
-				ObjectMeta: metav1.ObjectMeta{
-					Name:        s.name,
-					Namespace:   s.namespace,
-					Annotations: secret.Annotations,
-				},
-				Type: v1.SecretTypeTLS,
-				Data: secret.Data,
-			})
+	secret, err := s.storage.Get()
+	if err == nil && cert.IsValidTLSSecret(secret) {
+		// local storage had a cached secret, ensure that it exists in Kubernetes
+		_, err := s.secrets.Create(&v1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:        s.name,
+				Namespace:   s.namespace,
+				Annotations: secret.Annotations,
+			},
+			Type: v1.SecretTypeTLS,
+			Data: secret.Data,
+		})
+		if err != nil && !errors.IsAlreadyExists(err) {
+			logrus.Warnf("Failed to create Kubernetes secret: %v", err)
+		}
+	} else {
+		// local storage was empty, try to populate it
+		secret, err := s.secrets.Get(s.namespace, s.name, metav1.GetOptions{})
+		if err != nil {
+			if !errors.IsNotFound(err) {
+				logrus.Warnf("Failed to init Kubernetes secret: %v", err)
+			}
+			return
+		}
+
+		if err := s.storage.Update(secret); err != nil {
+			logrus.Warnf("Failed to init backing storage secret: %v", err)
 		}
 	}
 }
 
 func (s *storage) Get() (*v1.Secret, error) {
-	s.Lock()
-	defer s.Unlock()
+	s.RLock()
+	defer s.RUnlock()
 
 	return s.storage.Get()
 }
 
 func (s *storage) targetSecret() (*v1.Secret, error) {
+	s.RLock()
+	defer s.RUnlock()
+
 	existingSecret, err := s.secrets.Get(s.namespace, s.name, metav1.GetOptions{})
 	if errors.IsNotFound(err) {
 		return &v1.Secret{
@@ -120,33 +141,48 @@ func (s *storage) targetSecret() (*v1.Secret, error) {
 				Name:      s.name,
 				Namespace: s.namespace,
 			},
+			Type: v1.SecretTypeTLS,
 		}, nil
 	}
 	return existingSecret, err
 }
 
 func (s *storage) saveInK8s(secret *v1.Secret) (*v1.Secret, error) {
-	if s.secrets == nil {
+	if !s.initComplete() {
 		return secret, nil
 	}
 
-	if existing, err := s.storage.Get(); err == nil && s.tls != nil {
-		if newSecret, updated, err := s.tls.Merge(existing, secret); err == nil && updated {
-			secret = newSecret
-		}
-	}
-
 	targetSecret, err := s.targetSecret()
 	if err != nil {
 		return nil, err
 	}
 
-	if newSecret, updated, err := s.tls.Merge(targetSecret, secret); err != nil {
-		return nil, err
-	} else if !updated {
-		return newSecret, nil
-	} else {
-		secret = newSecret
+	// if we don't have a TLS factory we can't create certs, so don't bother trying to merge anything,
+	// in favor of just blindly replacing the fields on the Kubernetes secret.
+	if s.tls != nil {
+		// merge new secret with secret from backing storage, if one exists
+		if existing, err := s.Get(); err == nil && cert.IsValidTLSSecret(existing) {
+			if newSecret, updated, err := s.tls.Merge(existing, secret); err == nil && updated {
+				secret = newSecret
+			}
+		}
+
+		// merge new secret with existing secret from Kubernetes, if one exists
+		if cert.IsValidTLSSecret(targetSecret) {
+			if newSecret, updated, err := s.tls.Merge(targetSecret, secret); err != nil {
+				return nil, err
+			} else if !updated {
+				return newSecret, nil
+			} else {
+				secret = newSecret
+			}
+		}
+	}
+
+	// ensure that the merged secret actually contains data before overwriting the existing fields
+	if !cert.IsValidTLSSecret(secret) {
+		logrus.Warnf("Skipping save of TLS secret for %s/%s due to missing certificate data", secret.Namespace, secret.Name)
+		return targetSecret, nil
 	}
 
 	targetSecret.Annotations = secret.Annotations
@@ -154,31 +190,49 @@ func (s *storage) saveInK8s(secret *v1.Secret) (*v1.Secret, error) {
 	targetSecret.Data = secret.Data
 
 	if targetSecret.UID == "" {
-		logrus.Infof("Creating new TLS secret for %v (count: %d): %v", targetSecret.Name, len(targetSecret.Annotations)-1, targetSecret.Annotations)
+		logrus.Infof("Creating new TLS secret for %s/%s (count: %d): %v", targetSecret.Namespace, targetSecret.Name, len(targetSecret.Annotations)-1, targetSecret.Annotations)
 		return s.secrets.Create(targetSecret)
 	}
-	logrus.Infof("Updating TLS secret for %v (count: %d): %v", targetSecret.Name, len(targetSecret.Annotations)-1, targetSecret.Annotations)
+	logrus.Infof("Updating TLS secret for %s/%s (count: %d): %v", targetSecret.Namespace, targetSecret.Name, len(targetSecret.Annotations)-1, targetSecret.Annotations)
 	return s.secrets.Update(targetSecret)
 }
 
-func (s *storage) Update(secret *v1.Secret) (err error) {
-	s.Lock()
-	defer s.Unlock()
-
-	for i := 0; i < 3; i++ {
-		secret, err = s.saveInK8s(secret)
-		if errors.IsConflict(err) {
-			continue
-		} else if err != nil {
-			return err
+func (s *storage) Update(secret *v1.Secret) error {
+	// Asynchronously update the Kubernetes secret, as doing so inline may block the listener from
+	// accepting new connections if the apiserver becomes unavailable after the Secrets controller
+	// has been initialized. We're not passing around any contexts here, nor does the controller
+	// accept any, so there's no good way to soft-fail with a reasonable timeout.
+	go func() {
+		if err := s.update(secret); err != nil {
+			logrus.Errorf("Failed to save TLS secret for %s/%s: %v", secret.Namespace, secret.Name, err)
 		}
-		break
-	}
+	}()
+	return nil
+}
+
+func isConflictOrAlreadyExists(err error) bool {
+	return errors.IsConflict(err) || errors.IsAlreadyExists(err)
+}
+
+func (s *storage) update(secret *v1.Secret) (err error) {
+	var newSecret *v1.Secret
+	err = retry.OnError(retry.DefaultRetry, isConflictOrAlreadyExists, func() error {
+		newSecret, err = s.saveInK8s(secret)
+		return err
+	})
 
 	if err != nil {
 		return err
 	}
 
-	// update underlying storage
-	return s.storage.Update(secret)
+	// Only hold the lock while updating underlying storage
+	s.Lock()
+	defer s.Unlock()
+	return s.storage.Update(newSecret)
+}
+
+func (s *storage) initComplete() bool {
+	s.RLock()
+	defer s.RUnlock()
+	return s.secrets != nil
 }

storage/memory/memory.go

@@ -39,7 +39,7 @@ func (m *memory) Update(secret *v1.Secret) error {
 			}
 		}
 
-		logrus.Infof("Active TLS secret %s (ver=%s) (count %d): %v", secret.Name, secret.ResourceVersion, len(secret.Annotations)-1, secret.Annotations)
+		logrus.Infof("Active TLS secret %s/%s (ver=%s) (count %d): %v", secret.Namespace, secret.Name, secret.ResourceVersion, len(secret.Annotations)-1, secret.Annotations)
 		m.secret = secret
 	}
 	return nil