From a75e84bc817138c13d98898f42eb0e451c379a20 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Thu, 30 Jan 2020 22:18:44 -0700 Subject: [PATCH] Add more helpers --- factory/ca.go | 24 +++--- factory/gen.go | 11 +++ listener.go | 98 +++++++++++++++++++++-- server/server.go | 162 ++++++++++++++++++++++++++++++++------- storage/kubernetes/ca.go | 59 ++++++++++++++ 5 files changed, 308 insertions(+), 46 deletions(-) create mode 100644 storage/kubernetes/ca.go diff --git a/factory/ca.go b/factory/ca.go index cc3380f..5dfa863 100644 --- a/factory/ca.go +++ b/factory/ca.go @@ -59,16 +59,7 @@ func loadCA() (*x509.Certificate, crypto.Signer, error) { return LoadCerts("./certs/ca.pem", "./certs/ca.key") } -func LoadCerts(certFile, keyFile string) (*x509.Certificate, crypto.Signer, error) { - caPem, err := ioutil.ReadFile(certFile) - if err != nil { - return nil, nil, err - } - caKey, err := ioutil.ReadFile(keyFile) - if err != nil { - return nil, nil, err - } - +func LoadCA(caPem, caKey []byte) (*x509.Certificate, crypto.Signer, error) { key, err := cert.ParsePrivateKeyPEM(caKey) if err != nil { return nil, nil, err @@ -85,3 +76,16 @@ func LoadCerts(certFile, keyFile string) (*x509.Certificate, crypto.Signer, erro return cert, signer, nil } + +func LoadCerts(certFile, keyFile string) (*x509.Certificate, crypto.Signer, error) { + caPem, err := ioutil.ReadFile(certFile) + if err != nil { + return nil, nil, err + } + caKey, err := ioutil.ReadFile(keyFile) + if err != nil { + return nil, nil, err + } + + return LoadCA(caPem, caKey) +} diff --git a/factory/gen.go b/factory/gen.go index 0def051..50314f4 100644 --- a/factory/gen.go +++ b/factory/gen.go @@ -31,6 +31,9 @@ type TLS struct { } func cns(secret *v1.Secret) (cns []string) { + if secret == nil { + return nil + } for k, v := range secret.Annotations { if strings.HasPrefix(k, cnPrefix) { cns = append(cns, v) @@ -65,6 +68,14 @@ func (t *TLS) Merge(secret, other *v1.Secret) (*v1.Secret, bool, error) { return t.AddCN(secret, cns(other)...) } +func (t *TLS) Refresh(secret *v1.Secret) (*v1.Secret, error) { + cns := cns(secret) + secret = secret.DeepCopy() + secret.Annotations = map[string]string{} + secret, _, err := t.AddCN(secret, cns...) + return secret, err +} + func (t *TLS) AddCN(secret *v1.Secret, cn ...string) (*v1.Secret, bool, error) { var ( err error diff --git a/listener.go b/listener.go index 86b4e58..17fcc81 100644 --- a/listener.go +++ b/listener.go @@ -1,6 +1,7 @@ package dynamiclistener import ( + "context" "crypto" "crypto/tls" "crypto/x509" @@ -8,6 +9,7 @@ import ( "net/http" "strings" "sync" + "time" "github.com/rancher/dynamiclistener/factory" "github.com/sirupsen/logrus" @@ -20,6 +22,7 @@ type TLSStorage interface { } type TLSFactory interface { + Refresh(secret *v1.Secret) (*v1.Secret, error) AddCN(secret *v1.Secret, cn ...string) (*v1.Secret, bool, error) Merge(secret *v1.Secret, existing *v1.Secret) (*v1.Secret, bool, error) } @@ -28,13 +31,6 @@ type SetFactory interface { SetFactory(tls TLSFactory) } -type Config struct { - CN string - Organization []string - TLSConfig tls.Config - SANs []string -} - func NewListener(l net.Listener, storage TLSStorage, caCert *x509.Certificate, caKey crypto.Signer, config Config) (net.Listener, http.Handler, error) { if config.CN == "" { config.CN = "dynamic" @@ -61,7 +57,30 @@ func NewListener(l net.Listener, storage TLSStorage, caCert *x509.Certificate, c setter.SetFactory(dynamicListener.factory) } - return tls.NewListener(dynamicListener, &dynamicListener.tlsConfig), dynamicListener.cacheHandler(), nil + if config.ExpirationDaysCheck == 0 { + config.ExpirationDaysCheck = 30 + } + + tlsListener := tls.NewListener(dynamicListener.WrapExpiration(config.ExpirationDaysCheck), &dynamicListener.tlsConfig) + return tlsListener, dynamicListener.cacheHandler(), nil +} + +type cancelClose struct { + cancel func() + net.Listener +} + +func (c *cancelClose) Close() error { + c.cancel() + return c.Listener.Close() +} + +type Config struct { + CN string + Organization []string + TLSConfig tls.Config + SANs []string + ExpirationDaysCheck int } type listener struct { @@ -77,6 +96,69 @@ type listener struct { init sync.Once } +func (l *listener) WrapExpiration(days int) net.Listener { + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(5 * time.Minute) + + for { + wait := 6 * time.Hour + if err := l.checkExpiration(days); err != nil { + logrus.Errorf("failed to check and refresh dynamic cert: %v", err) + wait = 5 + time.Minute + } + select { + case <-ctx.Done(): + return + case <-time.After(wait): + } + } + }() + + return &cancelClose{ + cancel: cancel, + Listener: l, + } +} + +func (l *listener) checkExpiration(days int) error { + l.Lock() + defer l.Unlock() + + if days == 0 { + return nil + } + + if l.cert == nil { + return nil + } + + secret, err := l.storage.Get() + if err != nil { + return err + } + + cert, err := tls.X509KeyPair(secret.Data[v1.TLSCertKey], secret.Data[v1.TLSPrivateKeyKey]) + if err != nil { + return err + } + + certParsed, err := x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + return err + } + + if time.Now().UTC().Add(time.Hour * 24 * time.Duration(days)).After(certParsed.NotAfter) { + secret, err := l.factory.Refresh(secret) + if err != nil { + return err + } + return l.storage.Update(secret) + } + + return nil +} + func (l *listener) Accept() (net.Conn, error) { l.init.Do(func() { if len(l.sans) > 0 { diff --git a/server/server.go b/server/server.go index 2b793cc..251ad98 100644 --- a/server/server.go +++ b/server/server.go @@ -3,20 +3,35 @@ package server import ( "context" "crypto" + "crypto/tls" "crypto/x509" "fmt" + "log" + "net" "net/http" "github.com/rancher/dynamiclistener" "github.com/rancher/dynamiclistener/factory" + "github.com/rancher/dynamiclistener/storage/file" + "github.com/rancher/dynamiclistener/storage/kubernetes" "github.com/rancher/dynamiclistener/storage/memory" + v1 "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1" "github.com/sirupsen/logrus" + "golang.org/x/crypto/acme/autocert" ) type ListenOpts struct { - CA *x509.Certificate - CAKey crypto.Signer - Storage dynamiclistener.TLSStorage + CA *x509.Certificate + CAKey crypto.Signer + Storage dynamiclistener.TLSStorage + Secrets v1.SecretController + CertNamespace string + CertName string + CANamespace string + CAName string + CertBackup string + AcmeDomains []string + TLSListenerConfig dynamiclistener.Config } func ListenAndServe(ctx context.Context, httpsPort, httpPort int, handler http.Handler, opts *ListenOpts) error { @@ -29,40 +44,26 @@ func ListenAndServe(ctx context.Context, httpsPort, httpPort int, handler http.H opts = &ListenOpts{} } + logger := logrus.StandardLogger() + errorLog := log.New(logger.WriterLevel(logrus.DebugLevel), "", log.LstdFlags) + if httpsPort > 0 { - var ( - caCert *x509.Certificate - caKey crypto.Signer - err error - ) - - if opts.CA != nil && opts.CAKey != nil { - caCert, caKey = opts.CA, opts.CAKey - } else { - caCert, caKey, err = factory.LoadOrGenCA() - if err != nil { - return err - } - } - tlsTCPListener, err := dynamiclistener.NewTCPListener("0.0.0.0", httpsPort) if err != nil { return err } - storage := opts.Storage - if storage == nil { - storage = memory.New() - } - - dynListener, dynHandler, err := dynamiclistener.NewListener(tlsTCPListener, storage, caCert, caKey, dynamiclistener.Config{}) + dynListener, dynHandler, err := getTLSListener(ctx, tlsTCPListener, *opts) if err != nil { return err } - targetHandler = wrapHandler(dynHandler, handler) + if dynHandler != nil { + targetHandler = wrapHandler(dynHandler, handler) + } tlsServer := http.Server{ - Handler: targetHandler, + Handler: targetHandler, + ErrorLog: errorLog, } targetHandler = dynamiclistener.HTTPRedirect(targetHandler) @@ -81,8 +82,9 @@ func ListenAndServe(ctx context.Context, httpsPort, httpPort int, handler http.H if httpPort > 0 { httpServer := http.Server{ - Addr: fmt.Sprintf("0.0.0.0:%d", httpPort), - Handler: targetHandler, + Addr: fmt.Sprintf("0.0.0.0:%d", httpPort), + Handler: targetHandler, + ErrorLog: errorLog, } go func() { logrus.Infof("Listening on 0.0.0.0:%d", httpPort) @@ -100,9 +102,113 @@ func ListenAndServe(ctx context.Context, httpsPort, httpPort int, handler http.H return nil } +func getTLSListener(ctx context.Context, tcp net.Listener, opts ListenOpts) (net.Listener, http.Handler, error) { + if len(opts.TLSListenerConfig.TLSConfig.NextProtos) == 0 { + opts.TLSListenerConfig.TLSConfig.NextProtos = []string{"h2", "http/1.1"} + } + + if len(opts.TLSListenerConfig.TLSConfig.Certificates) > 0 { + return tls.NewListener(tcp, &opts.TLSListenerConfig.TLSConfig), nil, nil + } + + if len(opts.AcmeDomains) > 0 { + return acmeListener(tcp, opts), nil, nil + } + + storage := opts.Storage + if storage == nil { + storage = newStorage(ctx, opts) + } + + caCert, caKey, err := getCA(opts) + if err != nil { + return nil, nil, err + } + + return dynamiclistener.NewListener(tcp, storage, caCert, caKey, opts.TLSListenerConfig) +} + +func getCA(opts ListenOpts) (*x509.Certificate, crypto.Signer, error) { + if opts.CA != nil && opts.CAKey != nil { + return opts.CA, opts.CAKey, nil + } + + if opts.Secrets == nil { + return factory.LoadOrGenCA() + } + + if opts.CAName == "" { + opts.CAName = "serving-ca" + } + + if opts.CANamespace == "" { + opts.CANamespace = opts.CertNamespace + } + + if opts.CANamespace == "" { + opts.CANamespace = "kube-system" + } + + return kubernetes.LoadOrGenCA(opts.Secrets, opts.CANamespace, opts.CAName) +} + +func newStorage(ctx context.Context, opts ListenOpts) dynamiclistener.TLSStorage { + var result dynamiclistener.TLSStorage + if opts.CertBackup == "" { + result = memory.New() + } else { + result = memory.NewBacked(file.New(opts.CertBackup)) + } + + if opts.Secrets == nil { + return result + } + + if opts.CertName == "" { + opts.CertName = "serving-cert" + } + + if opts.CertNamespace == "" { + opts.CertNamespace = "kube-system" + } + + return kubernetes.Load(ctx, opts.Secrets, opts.CertNamespace, opts.CertName, result) +} + func wrapHandler(handler http.Handler, next http.Handler) http.Handler { return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { handler.ServeHTTP(rw, req) next.ServeHTTP(rw, req) }) } + +func acmeListener(tcp net.Listener, opts ListenOpts) net.Listener { + hosts := map[string]bool{} + for _, domain := range opts.AcmeDomains { + hosts[domain] = true + } + + manager := autocert.Manager{ + Cache: autocert.DirCache("certs-cache"), + Prompt: func(tosURL string) bool { + return true + }, + HostPolicy: func(ctx context.Context, host string) error { + if !hosts[host] { + return fmt.Errorf("host %s is not configured", host) + } + return nil + }, + } + + opts.TLSListenerConfig.TLSConfig.GetCertificate = func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { + if hello.ServerName == "localhost" || hello.ServerName == "" { + newHello := *hello + newHello.ServerName = opts.AcmeDomains[0] + return manager.GetCertificate(&newHello) + } + return manager.GetCertificate(hello) + } + + return tls.NewListener(tcp, &opts.TLSListenerConfig.TLSConfig) +} diff --git a/storage/kubernetes/ca.go b/storage/kubernetes/ca.go new file mode 100644 index 0000000..3889364 --- /dev/null +++ b/storage/kubernetes/ca.go @@ -0,0 +1,59 @@ +package kubernetes + +import ( + "crypto" + "crypto/x509" + + "github.com/rancher/dynamiclistener/factory" + v1controller "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func LoadOrGenCA(secrets v1controller.SecretClient, namespace, name string) (*x509.Certificate, crypto.Signer, error) { + secret, err := getSecret(secrets, namespace, name) + if err != nil { + return nil, nil, err + } + return factory.LoadCA(secret.Data[v1.TLSCertKey], secret.Data[v1.TLSPrivateKeyKey]) +} + +func getSecret(secrets v1controller.SecretClient, namespace, name string) (*v1.Secret, error) { + s, err := secrets.Get(namespace, name, metav1.GetOptions{}) + if !errors.IsNotFound(err) { + return s, err + } + + if err := createAndStore(secrets, namespace, name); err != nil { + return nil, err + } + return secrets.Get(namespace, name, metav1.GetOptions{}) +} + +func createAndStore(secrets v1controller.SecretClient, namespace string, name string) error { + ca, cert, err := factory.GenCA() + if err != nil { + return err + } + + certPem, keyPem, err := factory.Marshal(ca, cert) + if err != nil { + return err + } + + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Data: map[string][]byte{ + v1.TLSCertKey: certPem, + v1.TLSPrivateKeyKey: keyPem, + }, + Type: v1.SecretTypeTLS, + } + + secrets.Create(secret) + return nil +}