From 1bb4ecc0ae783160a9b00ef990a2f4f6706531dd Mon Sep 17 00:00:00 2001 From: Jacob Simpson Date: Thu, 9 Mar 2017 16:34:55 -0800 Subject: [PATCH] Move certificate manager to client. Kubernetes-commit: 415c4d2c3af1fadb839bf4b73c6ea16db6b8c59f --- util/certificate/BUILD | 59 ++ util/certificate/OWNERS | 8 + util/certificate/certificate_manager.go | 427 +++++++++++ util/certificate/certificate_manager_test.go | 768 +++++++++++++++++++ util/certificate/certificate_store.go | 317 ++++++++ util/certificate/certificate_store_test.go | 505 ++++++++++++ 6 files changed, 2084 insertions(+) create mode 100644 util/certificate/BUILD create mode 100644 util/certificate/OWNERS create mode 100644 util/certificate/certificate_manager.go create mode 100644 util/certificate/certificate_manager_test.go create mode 100644 util/certificate/certificate_store.go create mode 100644 util/certificate/certificate_store_test.go diff --git a/util/certificate/BUILD b/util/certificate/BUILD new file mode 100644 index 00000000..6743ff55 --- /dev/null +++ b/util/certificate/BUILD @@ -0,0 +1,59 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", + "go_test", +) + +go_test( + name = "go_default_test", + srcs = [ + "certificate_manager_test.go", + "certificate_store_test.go", + ], + library = ":go_default_library", + tags = ["automanaged"], + deps = [ + "//vendor/k8s.io/api/certificates/v1beta1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library", + "//vendor/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library", + "//vendor/k8s.io/client-go/util/cert:go_default_library", + ], +) + +go_library( + name = "go_default_library", + srcs = [ + "certificate_manager.go", + "certificate_store.go", + ], + tags = ["automanaged"], + deps = [ + "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/api/certificates/v1beta1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library", + "//vendor/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library", + "//vendor/k8s.io/client-go/util/cert:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/util/certificate/OWNERS b/util/certificate/OWNERS new file mode 100644 index 00000000..2dce803b --- /dev/null +++ b/util/certificate/OWNERS @@ -0,0 +1,8 @@ +reviewers: +- mikedanese +- liggit +- smarterclayton +approvers: +- mikedanese +- liggit +- smarterclayton diff --git a/util/certificate/certificate_manager.go b/util/certificate/certificate_manager.go new file mode 100644 index 00000000..08363491 --- /dev/null +++ b/util/certificate/certificate_manager.go @@ -0,0 +1,427 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certificate + +import ( + "crypto/ecdsa" + "crypto/elliptic" + cryptorand "crypto/rand" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" + "sync" + "time" + + "github.com/golang/glog" + + certificates "k8s.io/api/certificates/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apimachinery/pkg/watch" + certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" + "k8s.io/client-go/util/cert" +) + +// Manager maintains and updates the certificates in use by this certificate +// manager. In the background it communicates with the API server to get new +// certificates for certificates about to expire. +type Manager interface { + // CertificateSigningRequestClient sets the client interface that is used for + // signing new certificates generated as part of rotation. + SetCertificateSigningRequestClient(certificatesclient.CertificateSigningRequestInterface) error + // Start the API server status sync loop. + Start() + // Current returns the currently selected certificate from the + // certificate manager, as well as the associated certificate and key data + // in PEM format. + Current() *tls.Certificate +} + +// Config is the set of configuration parameters available for a new Manager. +type Config struct { + // CertificateSigningRequestClient will be used for signing new certificate + // requests generated when a key rotation occurs. It must be set either at + // initialization or by using CertificateSigningRequestClient before + // Manager.Start() is called. + CertificateSigningRequestClient certificatesclient.CertificateSigningRequestInterface + // Template is the CertificateRequest that will be used as a template for + // generating certificate signing requests for all new keys generated as + // part of rotation. It follows the same rules as the template parameter of + // crypto.x509.CreateCertificateRequest in the Go standard libraries. + Template *x509.CertificateRequest + // Usages is the types of usages that certificates generated by the manager + // can be used for. + Usages []certificates.KeyUsage + // CertificateStore is a persistent store where the current cert/key is + // kept and future cert/key pairs will be persisted after they are + // generated. + CertificateStore Store + // BootstrapCertificatePEM is the certificate data that will be returned + // from the Manager if the CertificateStore doesn't have any cert/key pairs + // currently available and has not yet had a chance to get a new cert/key + // pair from the API. If the CertificateStore does have a cert/key pair, + // this will be ignored. If there is no cert/key pair available in the + // CertificateStore, as soon as Start is called, it will request a new + // cert/key pair from the CertificateSigningRequestClient. This is intended + // to allow the first boot of a component to be initialized using a + // generic, multi-use cert/key pair which will be quickly replaced with a + // unique cert/key pair. + BootstrapCertificatePEM []byte + // BootstrapKeyPEM is the key data that will be returned from the Manager + // if the CertificateStore doesn't have any cert/key pairs currently + // available. If the CertificateStore does have a cert/key pair, this will + // be ignored. If the bootstrap cert/key pair are used, they will be + // rotated at the first opportunity, possibly well in advance of expiring. + // This is intended to allow the first boot of a component to be + // initialized using a generic, multi-use cert/key pair which will be + // quickly replaced with a unique cert/key pair. + BootstrapKeyPEM []byte + // CertificateExpiration will record a metric that shows the remaining + // lifetime of the certificate. + CertificateExpiration Gauge +} + +// Store is responsible for getting and updating the current certificate. +// Depending on the concrete implementation, the backing store for this +// behavior may vary. +type Store interface { + // Current returns the currently selected certificate, as well as the + // associated certificate and key data in PEM format. If the Store doesn't + // have a cert/key pair currently, it should return a NoCertKeyError so + // that the Manager can recover by using bootstrap certificates to request + // a new cert/key pair. + Current() (*tls.Certificate, error) + // Update accepts the PEM data for the cert/key pair and makes the new + // cert/key pair the 'current' pair, that will be returned by future calls + // to Current(). + Update(cert, key []byte) (*tls.Certificate, error) +} + +// Gauge will record the remaining lifetime of the certificate each time it is +// updated. +type Gauge interface { + Set(float64) +} + +// NoCertKeyError indicates there is no cert/key currently available. +type NoCertKeyError string + +func (e *NoCertKeyError) Error() string { return string(*e) } + +type manager struct { + certSigningRequestClient certificatesclient.CertificateSigningRequestInterface + template *x509.CertificateRequest + usages []certificates.KeyUsage + certStore Store + certAccessLock sync.RWMutex + cert *tls.Certificate + rotationDeadline time.Time + forceRotation bool + certificateExpiration Gauge +} + +// NewManager returns a new certificate manager. A certificate manager is +// responsible for being the authoritative source of certificates in the +// Kubelet and handling updates due to rotation. +func NewManager(config *Config) (Manager, error) { + cert, forceRotation, err := getCurrentCertificateOrBootstrap( + config.CertificateStore, + config.BootstrapCertificatePEM, + config.BootstrapKeyPEM) + if err != nil { + return nil, err + } + + m := manager{ + certSigningRequestClient: config.CertificateSigningRequestClient, + template: config.Template, + usages: config.Usages, + certStore: config.CertificateStore, + cert: cert, + forceRotation: forceRotation, + certificateExpiration: config.CertificateExpiration, + } + + return &m, nil +} + +// Current returns the currently selected certificate from the certificate +// manager. This can be nil if the manager was initialized without a +// certificate and has not yet received one from the +// CertificateSigningRequestClient. +func (m *manager) Current() *tls.Certificate { + m.certAccessLock.RLock() + defer m.certAccessLock.RUnlock() + return m.cert +} + +// SetCertificateSigningRequestClient sets the client interface that is used +// for signing new certificates generated as part of rotation. It must be +// called before Start() and can not be used to change the +// CertificateSigningRequestClient that has already been set. This method is to +// support the one specific scenario where the CertificateSigningRequestClient +// uses the CertificateManager. +func (m *manager) SetCertificateSigningRequestClient(certSigningRequestClient certificatesclient.CertificateSigningRequestInterface) error { + if m.certSigningRequestClient == nil { + m.certSigningRequestClient = certSigningRequestClient + return nil + } + return fmt.Errorf("property CertificateSigningRequestClient is already set") +} + +// Start will start the background work of rotating the certificates. +func (m *manager) Start() { + // Certificate rotation depends on access to the API server certificate + // signing API, so don't start the certificate manager if we don't have a + // client. This will happen on the cluster master, where the kubelet is + // responsible for bootstrapping the pods of the master components. + if m.certSigningRequestClient == nil { + glog.V(2).Infof("Certificate rotation is not enabled, no connection to the apiserver.") + return + } + + glog.V(2).Infof("Certificate rotation is enabled.") + + m.setRotationDeadline() + + // Synchronously request a certificate before entering the background + // loop to allow bootstrap scenarios, where the certificate manager + // doesn't have a certificate at all yet. + if m.shouldRotate() { + glog.V(1).Infof("shouldRotate() is true, forcing immediate rotation") + _, err := m.rotateCerts() + if err != nil { + glog.Errorf("Could not rotate certificates: %v", err) + } + } + backoff := wait.Backoff{ + Duration: 2 * time.Second, + Factor: 2, + Jitter: 0.1, + Steps: 7, + } + go wait.Forever(func() { + sleepInterval := m.rotationDeadline.Sub(time.Now()) + glog.V(2).Infof("Waiting %v for next certificate rotation", sleepInterval) + time.Sleep(sleepInterval) + if err := wait.ExponentialBackoff(backoff, m.rotateCerts); err != nil { + glog.Errorf("Reached backoff limit, still unable to rotate certs: %v", err) + wait.PollInfinite(128*time.Second, m.rotateCerts) + } + }, 0) +} + +func getCurrentCertificateOrBootstrap( + store Store, + bootstrapCertificatePEM []byte, + bootstrapKeyPEM []byte) (cert *tls.Certificate, shouldRotate bool, errResult error) { + + currentCert, err := store.Current() + if err == nil { + return currentCert, false, nil + } + + if _, ok := err.(*NoCertKeyError); !ok { + return nil, false, err + } + + if bootstrapCertificatePEM == nil || bootstrapKeyPEM == nil { + return nil, true, nil + } + + bootstrapCert, err := tls.X509KeyPair(bootstrapCertificatePEM, bootstrapKeyPEM) + if err != nil { + return nil, false, err + } + if len(bootstrapCert.Certificate) < 1 { + return nil, false, fmt.Errorf("no cert/key data found") + } + + certs, err := x509.ParseCertificates(bootstrapCert.Certificate[0]) + if err != nil { + return nil, false, fmt.Errorf("unable to parse certificate data: %v", err) + } + bootstrapCert.Leaf = certs[0] + return &bootstrapCert, true, nil +} + +// shouldRotate looks at how close the current certificate is to expiring and +// decides if it is time to rotate or not. +func (m *manager) shouldRotate() bool { + m.certAccessLock.RLock() + defer m.certAccessLock.RUnlock() + if m.cert == nil { + return true + } + if m.forceRotation { + return true + } + return time.Now().After(m.rotationDeadline) +} + +func (m *manager) rotateCerts() (bool, error) { + glog.V(2).Infof("Rotating certificates") + + csrPEM, keyPEM, err := m.generateCSR() + if err != nil { + glog.Errorf("Unable to generate a certificate signing request: %v", err) + return false, nil + } + + // Call the Certificate Signing Request API to get a certificate for the + // new private key. + crtPEM, err := requestCertificate(m.certSigningRequestClient, csrPEM, m.usages) + if err != nil { + glog.Errorf("Failed while requesting a signed certificate from the master: %v", err) + return false, nil + } + + cert, err := m.certStore.Update(crtPEM, keyPEM) + if err != nil { + glog.Errorf("Unable to store the new cert/key pair: %v", err) + return false, nil + } + + m.updateCached(cert) + m.setRotationDeadline() + m.forceRotation = false + return true, nil +} + +// setRotationDeadline sets a cached value for the threshold at which the +// current certificate should be rotated, 80%+/-10% of the expiration of the +// certificate. +func (m *manager) setRotationDeadline() { + m.certAccessLock.RLock() + defer m.certAccessLock.RUnlock() + if m.cert == nil { + m.rotationDeadline = time.Now() + return + } + + notAfter := m.cert.Leaf.NotAfter + totalDuration := float64(notAfter.Sub(m.cert.Leaf.NotBefore)) + + m.rotationDeadline = m.cert.Leaf.NotBefore.Add(jitteryDuration(totalDuration)) + glog.V(2).Infof("Certificate expiration is %v, rotation deadline is %v", notAfter, m.rotationDeadline) + if m.certificateExpiration != nil { + m.certificateExpiration.Set(float64(notAfter.Unix())) + } +} + +// jitteryDuration uses some jitter to set the rotation threshold so each node +// will rotate at approximately 70-90% of the total lifetime of the +// certificate. With jitter, if a number of nodes are added to a cluster at +// approximately the same time (such as cluster creation time), they won't all +// try to rotate certificates at the same time for the rest of the life of the +// cluster. +// +// This function is represented as a variable to allow replacement during testing. +var jitteryDuration = func(totalDuration float64) time.Duration { + return wait.Jitter(time.Duration(totalDuration), 0.2) - time.Duration(totalDuration*0.3) +} + +func (m *manager) updateCached(cert *tls.Certificate) { + m.certAccessLock.Lock() + defer m.certAccessLock.Unlock() + m.cert = cert +} + +func (m *manager) generateCSR() (csrPEM []byte, keyPEM []byte, err error) { + // Generate a new private key. + privateKey, err := ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader) + if err != nil { + return nil, nil, fmt.Errorf("unable to generate a new private key: %v", err) + } + der, err := x509.MarshalECPrivateKey(privateKey) + if err != nil { + return nil, nil, fmt.Errorf("unable to marshal the new key to DER: %v", err) + } + + keyPEM = pem.EncodeToMemory(&pem.Block{Type: cert.ECPrivateKeyBlockType, Bytes: der}) + + csrPEM, err = cert.MakeCSRFromTemplate(privateKey, m.template) + if err != nil { + return nil, nil, fmt.Errorf("unable to create a csr from the private key: %v", err) + } + return csrPEM, keyPEM, nil +} + +// requestCertificate will create a certificate signing request using the PEM +// encoded CSR and send it to API server, then it will watch the object's +// status, once approved by API server, it will return the API server's issued +// certificate (pem-encoded). If there is any errors, or the watch timeouts, it +// will return an error. +// +// NOTE This is a copy of a function with the same name in +// k8s.io/kubernetes/pkg/kubelet/util/csr/csr.go, changing only the package that +// CertificateSigningRequestInterface and KeyUsage are imported from. +func requestCertificate(client certificatesclient.CertificateSigningRequestInterface, csrData []byte, usages []certificates.KeyUsage) (certData []byte, err error) { + glog.Infof("Requesting new certificate.") + req, err := client.Create(&certificates.CertificateSigningRequest{ + // Username, UID, Groups will be injected by API server. + TypeMeta: metav1.TypeMeta{Kind: "CertificateSigningRequest"}, + ObjectMeta: metav1.ObjectMeta{GenerateName: "csr-"}, + + Spec: certificates.CertificateSigningRequestSpec{ + Request: csrData, + Usages: usages, + }, + }) + if err != nil { + return nil, fmt.Errorf("cannot create certificate signing request: %v", err) + } + + // Make a default timeout = 3600s. + var defaultTimeoutSeconds int64 = 3600 + certWatch, err := client.Watch(metav1.ListOptions{ + Watch: true, + TimeoutSeconds: &defaultTimeoutSeconds, + FieldSelector: fields.OneTermEqualSelector("metadata.name", req.Name).String(), + }) + if err != nil { + return nil, fmt.Errorf("cannot watch on the certificate signing request: %v", err) + } + defer certWatch.Stop() + ch := certWatch.ResultChan() + + for { + event, ok := <-ch + if !ok { + break + } + + if event.Type == watch.Modified || event.Type == watch.Added { + if event.Object.(*certificates.CertificateSigningRequest).UID != req.UID { + continue + } + status := event.Object.(*certificates.CertificateSigningRequest).Status + for _, c := range status.Conditions { + if c.Type == certificates.CertificateDenied { + return nil, fmt.Errorf("certificate signing request is not approved, reason: %v, message: %v", c.Reason, c.Message) + } + if c.Type == certificates.CertificateApproved && status.Certificate != nil { + return status.Certificate, nil + } + } + } + } + + return nil, fmt.Errorf("watch channel closed") +} diff --git a/util/certificate/certificate_manager_test.go b/util/certificate/certificate_manager_test.go new file mode 100644 index 00000000..6a77f99b --- /dev/null +++ b/util/certificate/certificate_manager_test.go @@ -0,0 +1,768 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certificate + +import ( + "bytes" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "fmt" + "strings" + "testing" + "time" + + certificates "k8s.io/api/certificates/v1beta1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + watch "k8s.io/apimachinery/pkg/watch" + certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" +) + +var storeCertData = newCertificateData(`-----BEGIN CERTIFICATE----- +MIICRzCCAfGgAwIBAgIJALMb7ecMIk3MMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNV +BAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNVBAcMBkxvbmRvbjEYMBYGA1UE +CgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1JVCBEZXBhcnRtZW50MRswGQYD +VQQDDBJ0ZXN0LWNlcnRpZmljYXRlLTAwIBcNMTcwNDI2MjMyNjUyWhgPMjExNzA0 +MDIyMzI2NTJaMH4xCzAJBgNVBAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNV +BAcMBkxvbmRvbjEYMBYGA1UECgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1J +VCBEZXBhcnRtZW50MRswGQYDVQQDDBJ0ZXN0LWNlcnRpZmljYXRlLTAwXDANBgkq +hkiG9w0BAQEFAANLADBIAkEAtBMa7NWpv3BVlKTCPGO/LEsguKqWHBtKzweMY2CV +tAL1rQm913huhxF9w+ai76KQ3MHK5IVnLJjYYA5MzP2H5QIDAQABo1AwTjAdBgNV +HQ4EFgQU22iy8aWkNSxv0nBxFxerfsvnZVMwHwYDVR0jBBgwFoAU22iy8aWkNSxv +0nBxFxerfsvnZVMwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAANBAEOefGbV +NcHxklaW06w6OBYJPwpIhCVozC1qdxGX1dg8VkEKzjOzjgqVD30m59OFmSlBmHsl +nkVA6wyOSDYBf3o= +-----END CERTIFICATE-----`, `-----BEGIN RSA PRIVATE KEY----- +MIIBUwIBADANBgkqhkiG9w0BAQEFAASCAT0wggE5AgEAAkEAtBMa7NWpv3BVlKTC +PGO/LEsguKqWHBtKzweMY2CVtAL1rQm913huhxF9w+ai76KQ3MHK5IVnLJjYYA5M +zP2H5QIDAQABAkAS9BfXab3OKpK3bIgNNyp+DQJKrZnTJ4Q+OjsqkpXvNltPJosf +G8GsiKu/vAt4HGqI3eU77NvRI+mL4MnHRmXBAiEA3qM4FAtKSRBbcJzPxxLEUSwg +XSCcosCktbkXvpYrS30CIQDPDxgqlwDEJQ0uKuHkZI38/SPWWqfUmkecwlbpXABK +iQIgZX08DA8VfvcA5/Xj1Zjdey9FVY6POLXen6RPiabE97UCICp6eUW7ht+2jjar +e35EltCRCjoejRHTuN9TC0uCoVipAiAXaJIx/Q47vGwiw6Y8KXsNU6y54gTbOSxX +54LzHNk/+Q== +-----END RSA PRIVATE KEY-----`) +var bootstrapCertData = newCertificateData( + `-----BEGIN CERTIFICATE----- +MIICRzCCAfGgAwIBAgIJANXr+UzRFq4TMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNV +BAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNVBAcMBkxvbmRvbjEYMBYGA1UE +CgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1JVCBEZXBhcnRtZW50MRswGQYD +VQQDDBJ0ZXN0LWNlcnRpZmljYXRlLTEwIBcNMTcwNDI2MjMyNzMyWhgPMjExNzA0 +MDIyMzI3MzJaMH4xCzAJBgNVBAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNV +BAcMBkxvbmRvbjEYMBYGA1UECgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1J +VCBEZXBhcnRtZW50MRswGQYDVQQDDBJ0ZXN0LWNlcnRpZmljYXRlLTEwXDANBgkq +hkiG9w0BAQEFAANLADBIAkEAqvbkN4RShH1rL37JFp4fZPnn0JUhVWWsrP8NOomJ +pXdBDUMGWuEQIsZ1Gf9JrCQLu6ooRyHSKRFpAVbMQ3ABJwIDAQABo1AwTjAdBgNV +HQ4EFgQUEGBc6YYheEZ/5MhwqSUYYPYRj2MwHwYDVR0jBBgwFoAUEGBc6YYheEZ/ +5MhwqSUYYPYRj2MwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAANBAIyNmznk +5dgJY52FppEEcfQRdS5k4XFPc22SHPcz77AHf5oWZ1WG9VezOZZPp8NCiFDDlDL8 +yma33a5eMyTjLD8= +-----END CERTIFICATE-----`, `-----BEGIN RSA PRIVATE KEY----- +MIIBVAIBADANBgkqhkiG9w0BAQEFAASCAT4wggE6AgEAAkEAqvbkN4RShH1rL37J +Fp4fZPnn0JUhVWWsrP8NOomJpXdBDUMGWuEQIsZ1Gf9JrCQLu6ooRyHSKRFpAVbM +Q3ABJwIDAQABAkBC2OBpGLMPHN8BJijIUDFkURakBvuOoX+/8MYiYk7QxEmfLCk6 +L6r+GLNFMfXwXcBmXtMKfZKAIKutKf098JaBAiEA10azfqt3G/5owrNA00plSyT6 +ZmHPzY9Uq1p/QTR/uOcCIQDLTkfBkLHm0UKeobbO/fSm6ZflhyBRDINy4FvwmZMt +wQIgYV/tmQJeIh91q3wBepFQOClFykG8CTMoDUol/YyNqUkCIHfp6Rr7fGL3JIMq +QQgf9DCK8SPZqq8DYXjdan0kKBJBAiEAyDb+07o2gpggo8BYUKSaiRCiyXfaq87f +eVqgpBq/QN4= +-----END RSA PRIVATE KEY-----`) +var apiServerCertData = newCertificateData( + `-----BEGIN CERTIFICATE----- +MIICRzCCAfGgAwIBAgIJAIydTIADd+yqMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNV +BAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNVBAcMBkxvbmRvbjEYMBYGA1UE +CgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1JVCBEZXBhcnRtZW50MRswGQYD +VQQDDBJ0ZXN0LWNlcnRpZmljYXRlLTIwIBcNMTcwNDI2MjMyNDU4WhgPMjExNzA0 +MDIyMzI0NThaMH4xCzAJBgNVBAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNV +BAcMBkxvbmRvbjEYMBYGA1UECgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1J +VCBEZXBhcnRtZW50MRswGQYDVQQDDBJ0ZXN0LWNlcnRpZmljYXRlLTIwXDANBgkq +hkiG9w0BAQEFAANLADBIAkEAuiRet28DV68Dk4A8eqCaqgXmymamUEjW/DxvIQqH +3lbhtm8BwSnS9wUAajSLSWiq3fci2RbRgaSPjUrnbOHCLQIDAQABo1AwTjAdBgNV +HQ4EFgQU0vhI4OPGEOqT+VAWwxdhVvcmgdIwHwYDVR0jBBgwFoAU0vhI4OPGEOqT ++VAWwxdhVvcmgdIwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAANBALNeJGDe +nV5cXbp9W1bC12Tc8nnNXn4ypLE2JTQAvyp51zoZ8hQoSnRVx/VCY55Yu+br8gQZ ++tW+O/PoE7B3tuY= +-----END CERTIFICATE-----`, `-----BEGIN RSA PRIVATE KEY----- +MIIBVgIBADANBgkqhkiG9w0BAQEFAASCAUAwggE8AgEAAkEAuiRet28DV68Dk4A8 +eqCaqgXmymamUEjW/DxvIQqH3lbhtm8BwSnS9wUAajSLSWiq3fci2RbRgaSPjUrn +bOHCLQIDAQABAkEArDR1g9IqD3aUImNikDgAngbzqpAokOGyMoxeavzpEaFOgCzi +gi7HF7yHRmZkUt8CzdEvnHSqRjFuaaB0gGA+AQIhAOc8Z1h8ElLRSqaZGgI3jCTp +Izx9HNY//U5NGrXD2+ttAiEAzhOqkqI4+nDab7FpiD7MXI6fO549mEXeVBPvPtsS +OcECIQCIfkpOm+ZBBpO3JXaJynoqK4gGI6ALA/ik6LSUiIlfPQIhAISjd9hlfZME +bDQT1r8Q3Gx+h9LRqQeHgPBQ3F5ylqqBAiBaJ0hkYvrIdWxNlcLqD3065bJpHQ4S +WQkuZUQN1M/Xvg== +-----END RSA PRIVATE KEY-----`) + +type certificateData struct { + keyPEM []byte + certificatePEM []byte + certificate *tls.Certificate +} + +func newCertificateData(certificatePEM string, keyPEM string) *certificateData { + certificate, err := tls.X509KeyPair([]byte(certificatePEM), []byte(keyPEM)) + if err != nil { + panic(fmt.Sprintf("Unable to initialize certificate: %v", err)) + } + certs, err := x509.ParseCertificates(certificate.Certificate[0]) + if err != nil { + panic(fmt.Sprintf("Unable to initialize certificate leaf: %v", err)) + } + certificate.Leaf = certs[0] + return &certificateData{ + keyPEM: []byte(keyPEM), + certificatePEM: []byte(certificatePEM), + certificate: &certificate, + } +} + +func TestNewManagerNoRotation(t *testing.T) { + store := &fakeStore{ + cert: storeCertData.certificate, + } + if _, err := NewManager(&Config{ + Template: &x509.CertificateRequest{}, + Usages: []certificates.KeyUsage{}, + CertificateStore: store, + }); err != nil { + t.Fatalf("Failed to initialize the certificate manager: %v", err) + } +} + +func TestShouldRotate(t *testing.T) { + now := time.Now() + tests := []struct { + name string + notBefore time.Time + notAfter time.Time + shouldRotate bool + }{ + {"just issued, still good", now.Add(-1 * time.Hour), now.Add(99 * time.Hour), false}, + {"half way expired, still good", now.Add(-24 * time.Hour), now.Add(24 * time.Hour), false}, + {"mostly expired, still good", now.Add(-69 * time.Hour), now.Add(31 * time.Hour), false}, + {"just about expired, should rotate", now.Add(-91 * time.Hour), now.Add(9 * time.Hour), true}, + {"nearly expired, should rotate", now.Add(-99 * time.Hour), now.Add(1 * time.Hour), true}, + {"already expired, should rotate", now.Add(-10 * time.Hour), now.Add(-1 * time.Hour), true}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + m := manager{ + cert: &tls.Certificate{ + Leaf: &x509.Certificate{ + NotBefore: test.notBefore, + NotAfter: test.notAfter, + }, + }, + template: &x509.CertificateRequest{}, + usages: []certificates.KeyUsage{}, + } + m.setRotationDeadline() + if m.shouldRotate() != test.shouldRotate { + t.Errorf("Time %v, a certificate issued for (%v, %v) should rotate should be %t.", + now, + m.cert.Leaf.NotBefore, + m.cert.Leaf.NotAfter, + test.shouldRotate) + } + }) + } +} + +type gaugeMock struct { + calls int + lastValue float64 +} + +func (g *gaugeMock) Set(v float64) { + g.calls++ + g.lastValue = v +} + +func TestSetRotationDeadline(t *testing.T) { + defer func(original func(float64) time.Duration) { jitteryDuration = original }(jitteryDuration) + + now := time.Now() + testCases := []struct { + name string + notBefore time.Time + notAfter time.Time + shouldRotate bool + }{ + {"just issued, still good", now.Add(-1 * time.Hour), now.Add(99 * time.Hour), false}, + {"half way expired, still good", now.Add(-24 * time.Hour), now.Add(24 * time.Hour), false}, + {"mostly expired, still good", now.Add(-69 * time.Hour), now.Add(31 * time.Hour), false}, + {"just about expired, should rotate", now.Add(-91 * time.Hour), now.Add(9 * time.Hour), true}, + {"nearly expired, should rotate", now.Add(-99 * time.Hour), now.Add(1 * time.Hour), true}, + {"already expired, should rotate", now.Add(-10 * time.Hour), now.Add(-1 * time.Hour), true}, + {"long duration", now.Add(-6 * 30 * 24 * time.Hour), now.Add(6 * 30 * 24 * time.Hour), true}, + {"short duration", now.Add(-30 * time.Second), now.Add(30 * time.Second), true}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + g := gaugeMock{} + m := manager{ + cert: &tls.Certificate{ + Leaf: &x509.Certificate{ + NotBefore: tc.notBefore, + NotAfter: tc.notAfter, + }, + }, + template: &x509.CertificateRequest{}, + usages: []certificates.KeyUsage{}, + certificateExpiration: &g, + } + jitteryDuration = func(float64) time.Duration { return time.Duration(float64(tc.notAfter.Sub(tc.notBefore)) * 0.7) } + lowerBound := tc.notBefore.Add(time.Duration(float64(tc.notAfter.Sub(tc.notBefore)) * 0.7)) + + m.setRotationDeadline() + + if !m.rotationDeadline.Equal(lowerBound) { + t.Errorf("For notBefore %v, notAfter %v, the rotationDeadline %v should be %v.", + tc.notBefore, + tc.notAfter, + m.rotationDeadline, + lowerBound) + } + if g.calls != 1 { + t.Errorf("%d metrics were recorded, wanted %d", g.calls, 1) + } + if g.lastValue != float64(tc.notAfter.Unix()) { + t.Errorf("%d value for metric was recorded, wanted %d", g.lastValue, tc.notAfter.Unix()) + } + }) + } +} + +func TestRotateCertCreateCSRError(t *testing.T) { + now := time.Now() + m := manager{ + cert: &tls.Certificate{ + Leaf: &x509.Certificate{ + NotBefore: now.Add(-2 * time.Hour), + NotAfter: now.Add(-1 * time.Hour), + }, + }, + template: &x509.CertificateRequest{}, + usages: []certificates.KeyUsage{}, + certSigningRequestClient: fakeClient{ + failureType: createError, + }, + } + + if success, err := m.rotateCerts(); success { + t.Errorf("Got success from 'rotateCerts', wanted failure") + } else if err != nil { + t.Errorf("Got error %v from 'rotateCerts', wanted no error.", err) + } +} + +func TestRotateCertWaitingForResultError(t *testing.T) { + now := time.Now() + m := manager{ + cert: &tls.Certificate{ + Leaf: &x509.Certificate{ + NotBefore: now.Add(-2 * time.Hour), + NotAfter: now.Add(-1 * time.Hour), + }, + }, + template: &x509.CertificateRequest{}, + usages: []certificates.KeyUsage{}, + certSigningRequestClient: fakeClient{ + failureType: watchError, + }, + } + + if success, err := m.rotateCerts(); success { + t.Errorf("Got success from 'rotateCerts', wanted failure.") + } else if err != nil { + t.Errorf("Got error %v from 'rotateCerts', wanted no error.", err) + } +} + +func TestNewManagerBootstrap(t *testing.T) { + store := &fakeStore{} + + var cm Manager + cm, err := NewManager(&Config{ + Template: &x509.CertificateRequest{}, + Usages: []certificates.KeyUsage{}, + CertificateStore: store, + BootstrapCertificatePEM: bootstrapCertData.certificatePEM, + BootstrapKeyPEM: bootstrapCertData.keyPEM, + }) + if err != nil { + t.Fatalf("Failed to initialize the certificate manager: %v", err) + } + + cert := cm.Current() + + if cert == nil { + t.Errorf("Certificate was nil, expected something.") + } + if m, ok := cm.(*manager); !ok { + t.Errorf("Expected a '*manager' from 'NewManager'") + } else if !m.shouldRotate() { + t.Errorf("Expected rotation should happen during bootstrap, but it won't.") + } +} + +func TestNewManagerNoBootstrap(t *testing.T) { + now := time.Now() + cert, err := tls.X509KeyPair(storeCertData.certificatePEM, storeCertData.keyPEM) + if err != nil { + t.Fatalf("Unable to initialize a certificate: %v", err) + } + cert.Leaf = &x509.Certificate{ + NotBefore: now.Add(-24 * time.Hour), + NotAfter: now.Add(24 * time.Hour), + } + store := &fakeStore{ + cert: &cert, + } + + cm, err := NewManager(&Config{ + Template: &x509.CertificateRequest{}, + Usages: []certificates.KeyUsage{}, + CertificateStore: store, + BootstrapCertificatePEM: bootstrapCertData.certificatePEM, + BootstrapKeyPEM: bootstrapCertData.keyPEM, + }) + + if err != nil { + t.Fatalf("Failed to initialize the certificate manager: %v", err) + } + + currentCert := cm.Current() + + if currentCert == nil { + t.Errorf("Certificate was nil, expected something.") + } + if m, ok := cm.(*manager); !ok { + t.Errorf("Expected a '*manager' from 'NewManager'") + } else { + m.setRotationDeadline() + if m.shouldRotate() { + t.Errorf("Expected rotation should happen during bootstrap, but it won't.") + } + } +} + +func TestGetCurrentCertificateOrBootstrap(t *testing.T) { + testCases := []struct { + description string + storeCert *tls.Certificate + bootstrapCertData []byte + bootstrapKeyData []byte + expectedCert *tls.Certificate + expectedShouldRotate bool + expectedErrMsg string + }{ + { + "return cert from store", + storeCertData.certificate, + nil, + nil, + storeCertData.certificate, + false, + "", + }, + { + "no cert in store and no bootstrap cert", + nil, + nil, + nil, + nil, + true, + "", + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + store := &fakeStore{ + cert: tc.storeCert, + } + + certResult, shouldRotate, err := getCurrentCertificateOrBootstrap( + store, + tc.bootstrapCertData, + tc.bootstrapKeyData) + if certResult == nil || certResult.Certificate == nil || tc.expectedCert == nil { + if certResult != nil && tc.expectedCert != nil { + t.Errorf("Got certificate %v, wanted %v", certResult, tc.expectedCert) + } + } else { + if !certificatesEqual(certResult, tc.expectedCert) { + t.Errorf("Got certificate %v, wanted %v", certResult, tc.expectedCert) + } + } + if shouldRotate != tc.expectedShouldRotate { + t.Errorf("Got shouldRotate %t, wanted %t", shouldRotate, tc.expectedShouldRotate) + } + if err == nil { + if tc.expectedErrMsg != "" { + t.Errorf("Got err %v, wanted %q", err, tc.expectedErrMsg) + } + } else { + if tc.expectedErrMsg == "" || !strings.Contains(err.Error(), tc.expectedErrMsg) { + t.Errorf("Got err %v, wanted %q", err, tc.expectedErrMsg) + } + } + }) + } +} + +func TestInitializeCertificateSigningRequestClient(t *testing.T) { + var nilCertificate = &certificateData{} + testCases := []struct { + description string + storeCert *certificateData + bootstrapCert *certificateData + apiCert *certificateData + expectedCertBeforeStart *certificateData + expectedCertAfterStart *certificateData + }{ + { + description: "No current certificate, no bootstrap certificate", + storeCert: nilCertificate, + bootstrapCert: nilCertificate, + apiCert: apiServerCertData, + expectedCertBeforeStart: nilCertificate, + expectedCertAfterStart: apiServerCertData, + }, + { + description: "No current certificate, bootstrap certificate", + storeCert: nilCertificate, + bootstrapCert: bootstrapCertData, + apiCert: apiServerCertData, + expectedCertBeforeStart: bootstrapCertData, + expectedCertAfterStart: apiServerCertData, + }, + { + description: "Current certificate, no bootstrap certificate", + storeCert: storeCertData, + bootstrapCert: nilCertificate, + apiCert: apiServerCertData, + expectedCertBeforeStart: storeCertData, + expectedCertAfterStart: storeCertData, + }, + { + description: "Current certificate, bootstrap certificate", + storeCert: storeCertData, + bootstrapCert: bootstrapCertData, + apiCert: apiServerCertData, + expectedCertBeforeStart: storeCertData, + expectedCertAfterStart: storeCertData, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + certificateStore := &fakeStore{ + cert: tc.storeCert.certificate, + } + + certificateManager, err := NewManager(&Config{ + Template: &x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"system:nodes"}, + CommonName: "system:node:fake-node-name", + }, + }, + Usages: []certificates.KeyUsage{ + certificates.UsageDigitalSignature, + certificates.UsageKeyEncipherment, + certificates.UsageClientAuth, + }, + CertificateStore: certificateStore, + BootstrapCertificatePEM: tc.bootstrapCert.certificatePEM, + BootstrapKeyPEM: tc.bootstrapCert.keyPEM, + }) + if err != nil { + t.Errorf("Got %v, wanted no error.", err) + } + + certificate := certificateManager.Current() + if !certificatesEqual(certificate, tc.expectedCertBeforeStart.certificate) { + t.Errorf("Got %v, wanted %v", certificateString(certificate), certificateString(tc.expectedCertBeforeStart.certificate)) + } + if err := certificateManager.SetCertificateSigningRequestClient(&fakeClient{ + certificatePEM: tc.apiCert.certificatePEM, + }); err != nil { + t.Errorf("Got error %v, expected none.", err) + } + + if m, ok := certificateManager.(*manager); !ok { + t.Errorf("Expected a '*manager' from 'NewManager'") + } else { + m.setRotationDeadline() + if m.shouldRotate() { + if success, err := m.rotateCerts(); !success { + t.Errorf("Got failure from 'rotateCerts', wanted success.") + } else if err != nil { + t.Errorf("Got error %v, expected none.", err) + } + } + } + + certificate = certificateManager.Current() + if !certificatesEqual(certificate, tc.expectedCertAfterStart.certificate) { + t.Errorf("Got %v, wanted %v", certificateString(certificate), certificateString(tc.expectedCertAfterStart.certificate)) + } + }) + } +} + +func TestInitializeOtherRESTClients(t *testing.T) { + var nilCertificate = &certificateData{} + testCases := []struct { + description string + storeCert *certificateData + bootstrapCert *certificateData + apiCert *certificateData + expectedCertBeforeStart *certificateData + expectedCertAfterStart *certificateData + }{ + { + description: "No current certificate, no bootstrap certificate", + storeCert: nilCertificate, + bootstrapCert: nilCertificate, + apiCert: apiServerCertData, + expectedCertBeforeStart: nilCertificate, + expectedCertAfterStart: apiServerCertData, + }, + { + description: "No current certificate, bootstrap certificate", + storeCert: nilCertificate, + bootstrapCert: bootstrapCertData, + apiCert: apiServerCertData, + expectedCertBeforeStart: bootstrapCertData, + expectedCertAfterStart: apiServerCertData, + }, + { + description: "Current certificate, no bootstrap certificate", + storeCert: storeCertData, + bootstrapCert: nilCertificate, + apiCert: apiServerCertData, + expectedCertBeforeStart: storeCertData, + expectedCertAfterStart: storeCertData, + }, + { + description: "Current certificate, bootstrap certificate", + storeCert: storeCertData, + bootstrapCert: bootstrapCertData, + apiCert: apiServerCertData, + expectedCertBeforeStart: storeCertData, + expectedCertAfterStart: storeCertData, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + certificateStore := &fakeStore{ + cert: tc.storeCert.certificate, + } + + certificateManager, err := NewManager(&Config{ + Template: &x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"system:nodes"}, + CommonName: "system:node:fake-node-name", + }, + }, + Usages: []certificates.KeyUsage{ + certificates.UsageDigitalSignature, + certificates.UsageKeyEncipherment, + certificates.UsageClientAuth, + }, + CertificateStore: certificateStore, + BootstrapCertificatePEM: tc.bootstrapCert.certificatePEM, + BootstrapKeyPEM: tc.bootstrapCert.keyPEM, + CertificateSigningRequestClient: &fakeClient{ + certificatePEM: tc.apiCert.certificatePEM, + }, + }) + if err != nil { + t.Errorf("Got %v, wanted no error.", err) + } + + certificate := certificateManager.Current() + if !certificatesEqual(certificate, tc.expectedCertBeforeStart.certificate) { + t.Errorf("Got %v, wanted %v", certificateString(certificate), certificateString(tc.expectedCertBeforeStart.certificate)) + } + + if m, ok := certificateManager.(*manager); !ok { + t.Errorf("Expected a '*manager' from 'NewManager'") + } else { + m.setRotationDeadline() + if m.shouldRotate() { + if success, err := certificateManager.(*manager).rotateCerts(); !success { + t.Errorf("Got failure from 'rotateCerts', expected success") + } else if err != nil { + t.Errorf("Got error %v, expected none.", err) + } + } + } + + certificate = certificateManager.Current() + if !certificatesEqual(certificate, tc.expectedCertAfterStart.certificate) { + t.Errorf("Got %v, wanted %v", certificateString(certificate), certificateString(tc.expectedCertAfterStart.certificate)) + } + }) + } +} + +type fakeClientFailureType int + +const ( + none fakeClientFailureType = iota + createError + watchError + certificateSigningRequestDenied +) + +type fakeClient struct { + certificatesclient.CertificateSigningRequestInterface + failureType fakeClientFailureType + certificatePEM []byte +} + +func (c fakeClient) Create(*certificates.CertificateSigningRequest) (*certificates.CertificateSigningRequest, error) { + if c.failureType == createError { + return nil, fmt.Errorf("Create error") + } + csrReply := certificates.CertificateSigningRequest{} + csrReply.UID = "fake-uid" + return &csrReply, nil +} + +func (c fakeClient) Watch(opts v1.ListOptions) (watch.Interface, error) { + if c.failureType == watchError { + return nil, fmt.Errorf("Watch error") + } + return &fakeWatch{ + failureType: c.failureType, + certificatePEM: c.certificatePEM, + }, nil +} + +type fakeWatch struct { + failureType fakeClientFailureType + certificatePEM []byte +} + +func (w *fakeWatch) Stop() { +} + +func (w *fakeWatch) ResultChan() <-chan watch.Event { + var condition certificates.CertificateSigningRequestCondition + if w.failureType == certificateSigningRequestDenied { + condition = certificates.CertificateSigningRequestCondition{ + Type: certificates.CertificateDenied, + } + } else { + condition = certificates.CertificateSigningRequestCondition{ + Type: certificates.CertificateApproved, + } + } + + csr := certificates.CertificateSigningRequest{ + Status: certificates.CertificateSigningRequestStatus{ + Conditions: []certificates.CertificateSigningRequestCondition{ + condition, + }, + Certificate: []byte(w.certificatePEM), + }, + } + csr.UID = "fake-uid" + + c := make(chan watch.Event, 1) + c <- watch.Event{ + Type: watch.Added, + Object: &csr, + } + return c +} + +type fakeStore struct { + cert *tls.Certificate +} + +func (s *fakeStore) Current() (*tls.Certificate, error) { + if s.cert == nil { + noKeyErr := NoCertKeyError("") + return nil, &noKeyErr + } + return s.cert, nil +} + +// Accepts the PEM data for the cert/key pair and makes the new cert/key +// pair the 'current' pair, that will be returned by future calls to +// Current(). +func (s *fakeStore) Update(certPEM, keyPEM []byte) (*tls.Certificate, error) { + // In order to make the mocking work, whenever a cert/key pair is passed in + // to be updated in the mock store, assume that the certificate manager + // generated the key, and then asked the mock CertificateSigningRequest API + // to sign it, then the faked API returned a canned response. The canned + // signing response will not match the generated key. In order to make + // things work out, search here for the correct matching key and use that + // instead of the passed in key. That way this file of test code doesn't + // have to implement an actual certificate signing process. + for _, tc := range []*certificateData{storeCertData, bootstrapCertData, apiServerCertData} { + if bytes.Equal(tc.certificatePEM, certPEM) { + keyPEM = tc.keyPEM + } + } + cert, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + return nil, err + } + now := time.Now() + s.cert = &cert + s.cert.Leaf = &x509.Certificate{ + NotBefore: now.Add(-24 * time.Hour), + NotAfter: now.Add(24 * time.Hour), + } + return s.cert, nil +} + +func certificatesEqual(c1 *tls.Certificate, c2 *tls.Certificate) bool { + if c1 == nil || c2 == nil { + return c1 == c2 + } + if len(c1.Certificate) != len(c2.Certificate) { + return false + } + for i := 0; i < len(c1.Certificate); i++ { + if !bytes.Equal(c1.Certificate[i], c2.Certificate[i]) { + return false + } + } + return true +} + +func certificateString(c *tls.Certificate) string { + if c == nil { + return "certificate == nil" + } + if c.Leaf == nil { + return "certificate.Leaf == nil" + } + return c.Leaf.Subject.CommonName +} diff --git a/util/certificate/certificate_store.go b/util/certificate/certificate_store.go new file mode 100644 index 00000000..029f9916 --- /dev/null +++ b/util/certificate/certificate_store.go @@ -0,0 +1,317 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certificate + +import ( + "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "time" + + "github.com/golang/glog" +) + +const ( + keyExtension = ".key" + certExtension = ".crt" + pemExtension = ".pem" + currentPair = "current" + updatedPair = "updated" +) + +type fileStore struct { + pairNamePrefix string + certDirectory string + keyDirectory string + certFile string + keyFile string +} + +// NewFileStore returns a concrete implementation of a Store that is based on +// storing the cert/key pairs in a single file per pair on disk in the +// designated directory. When starting up it will look for the currently +// selected cert/key pair in: +// +// 1. ${certDirectory}/${pairNamePrefix}-current.pem - both cert and key are in the same file. +// 2. ${certFile}, ${keyFile} +// 3. ${certDirectory}/${pairNamePrefix}.crt, ${keyDirectory}/${pairNamePrefix}.key +// +// The first one found will be used. If rotation is enabled, future cert/key +// updates will be written to the ${certDirectory} directory and +// ${certDirectory}/${pairNamePrefix}-current.pem will be created as a soft +// link to the currently selected cert/key pair. +func NewFileStore( + pairNamePrefix string, + certDirectory string, + keyDirectory string, + certFile string, + keyFile string) (Store, error) { + + s := fileStore{ + pairNamePrefix: pairNamePrefix, + certDirectory: certDirectory, + keyDirectory: keyDirectory, + certFile: certFile, + keyFile: keyFile, + } + if err := s.recover(); err != nil { + return nil, err + } + return &s, nil +} + +// recover checks if there is a certificate rotation that was interrupted while +// progress, and if so, attempts to recover to a good state. +func (s *fileStore) recover() error { + // If the 'current' file doesn't exist, continue on with the recovery process. + currentPath := filepath.Join(s.certDirectory, s.filename(currentPair)) + if exists, err := fileExists(currentPath); err != nil { + return err + } else if exists { + return nil + } + + // If the 'updated' file exists, and it is a symbolic link, continue on + // with the recovery process. + updatedPath := filepath.Join(s.certDirectory, s.filename(updatedPair)) + if fi, err := os.Lstat(updatedPath); err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } else if fi.Mode()&os.ModeSymlink != os.ModeSymlink { + return fmt.Errorf("expected %q to be a symlink but it is a file", updatedPath) + } + + // Move the 'updated' symlink to 'current'. + if err := os.Rename(updatedPath, currentPath); err != nil { + return fmt.Errorf("unable to rename %q to %q: %v", updatedPath, currentPath, err) + } + return nil +} + +func (s *fileStore) Current() (*tls.Certificate, error) { + pairFile := filepath.Join(s.certDirectory, s.filename(currentPair)) + if pairFileExists, err := fileExists(pairFile); err != nil { + return nil, err + } else if pairFileExists { + glog.Infof("Loading cert/key pair from %q.", pairFile) + return loadFile(pairFile) + } + + certFileExists, err := fileExists(s.certFile) + if err != nil { + return nil, err + } + keyFileExists, err := fileExists(s.keyFile) + if err != nil { + return nil, err + } + if certFileExists && keyFileExists { + glog.Infof("Loading cert/key pair from (%q, %q).", s.certFile, s.keyFile) + return loadX509KeyPair(s.certFile, s.keyFile) + } + + c := filepath.Join(s.certDirectory, s.pairNamePrefix+certExtension) + k := filepath.Join(s.keyDirectory, s.pairNamePrefix+keyExtension) + certFileExists, err = fileExists(c) + if err != nil { + return nil, err + } + keyFileExists, err = fileExists(k) + if err != nil { + return nil, err + } + if certFileExists && keyFileExists { + glog.Infof("Loading cert/key pair from (%q, %q).", c, k) + return loadX509KeyPair(c, k) + } + + noKeyErr := NoCertKeyError( + fmt.Sprintf("no cert/key files read at %q, (%q, %q) or (%q, %q)", + pairFile, + s.certFile, + s.keyFile, + s.certDirectory, + s.keyDirectory)) + return nil, &noKeyErr +} + +func loadFile(pairFile string) (*tls.Certificate, error) { + certBlock, keyBlock, err := loadCertKeyBlocks(pairFile) + if err != nil { + return nil, err + } + cert, err := tls.X509KeyPair(pem.EncodeToMemory(certBlock), pem.EncodeToMemory(keyBlock)) + if err != nil { + return nil, fmt.Errorf("could not convert data from %q into cert/key pair: %v", pairFile, err) + } + certs, err := x509.ParseCertificates(cert.Certificate[0]) + if err != nil { + return nil, fmt.Errorf("unable to parse certificate data: %v", err) + } + cert.Leaf = certs[0] + return &cert, nil +} + +func loadCertKeyBlocks(pairFile string) (cert *pem.Block, key *pem.Block, err error) { + data, err := ioutil.ReadFile(pairFile) + if err != nil { + return nil, nil, fmt.Errorf("could not load cert/key pair from %q: %v", pairFile, err) + } + certBlock, rest := pem.Decode(data) + if certBlock == nil { + return nil, nil, fmt.Errorf("could not decode the first block from %q from expected PEM format", pairFile) + } + keyBlock, _ := pem.Decode(rest) + if keyBlock == nil { + return nil, nil, fmt.Errorf("could not decode the second block from %q from expected PEM format", pairFile) + } + return certBlock, keyBlock, nil +} + +func (s *fileStore) Update(certData, keyData []byte) (*tls.Certificate, error) { + ts := time.Now().Format("2006-01-02-15-04-05") + pemFilename := s.filename(ts) + + if err := os.MkdirAll(s.certDirectory, 0755); err != nil { + return nil, fmt.Errorf("could not create directory %q to store certificates: %v", s.certDirectory, err) + } + certPath := filepath.Join(s.certDirectory, pemFilename) + + f, err := os.OpenFile(certPath, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0600) + if err != nil { + return nil, fmt.Errorf("could not open %q: %v", certPath, err) + } + defer f.Close() + certBlock, _ := pem.Decode(certData) + if certBlock == nil { + return nil, fmt.Errorf("invalid certificate data") + } + pem.Encode(f, certBlock) + keyBlock, _ := pem.Decode(keyData) + if keyBlock == nil { + return nil, fmt.Errorf("invalid key data") + } + pem.Encode(f, keyBlock) + + cert, err := loadFile(certPath) + if err != nil { + return nil, err + } + + if err := s.updateSymlink(certPath); err != nil { + return nil, err + } + return cert, nil +} + +// updateSymLink updates the current symlink to point to the file that is +// passed it. It will fail if there is a non-symlink file exists where the +// symlink is expected to be. +func (s *fileStore) updateSymlink(filename string) error { + // If the 'current' file either doesn't exist, or is already a symlink, + // proceed. Otherwise, this is an unrecoverable error. + currentPath := filepath.Join(s.certDirectory, s.filename(currentPair)) + currentPathExists := false + if fi, err := os.Lstat(currentPath); err != nil { + if !os.IsNotExist(err) { + return err + } + } else if fi.Mode()&os.ModeSymlink != os.ModeSymlink { + return fmt.Errorf("expected %q to be a symlink but it is a file", currentPath) + } else { + currentPathExists = true + } + + // If the 'updated' file doesn't exist, proceed. If it exists but it is a + // symlink, delete it. Otherwise, this is an unrecoverable error. + updatedPath := filepath.Join(s.certDirectory, s.filename(updatedPair)) + if fi, err := os.Lstat(updatedPath); err != nil { + if !os.IsNotExist(err) { + return err + } + } else if fi.Mode()&os.ModeSymlink != os.ModeSymlink { + return fmt.Errorf("expected %q to be a symlink but it is a file", updatedPath) + } else { + if err := os.Remove(updatedPath); err != nil { + return fmt.Errorf("unable to remove %q: %v", updatedPath, err) + } + } + + // Check that the new cert/key pair file exists to avoid rotating to an + // invalid cert/key. + if filenameExists, err := fileExists(filename); err != nil { + return err + } else if !filenameExists { + return fmt.Errorf("file %q does not exist so it can not be used as the currently selected cert/key", filename) + } + + // Create the 'updated' symlink pointing to the requested file name. + if err := os.Symlink(filename, updatedPath); err != nil { + return fmt.Errorf("unable to create a symlink from %q to %q: %v", updatedPath, filename, err) + } + + // Replace the 'current' symlink. + if currentPathExists { + if err := os.Remove(currentPath); err != nil { + return fmt.Errorf("unable to remove %q: %v", currentPath, err) + } + } + if err := os.Rename(updatedPath, currentPath); err != nil { + return fmt.Errorf("unable to rename %q to %q: %v", updatedPath, currentPath, err) + } + return nil +} + +func (s *fileStore) filename(qualifier string) string { + return s.pairNamePrefix + "-" + qualifier + pemExtension +} + +// withoutExt returns the given filename after removing the extension. The +// extension to remove will be the result of filepath.Ext(). +func withoutExt(filename string) string { + return strings.TrimSuffix(filename, filepath.Ext(filename)) +} + +func loadX509KeyPair(certFile, keyFile string) (*tls.Certificate, error) { + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, err + } + certs, err := x509.ParseCertificates(cert.Certificate[0]) + if err != nil { + return nil, fmt.Errorf("unable to parse certificate data: %v", err) + } + cert.Leaf = certs[0] + return &cert, nil +} + +// FileExists checks if specified file exists. +func fileExists(filename string) (bool, error) { + if _, err := os.Stat(filename); os.IsNotExist(err) { + return false, nil + } else if err != nil { + return false, err + } + return true, nil +} diff --git a/util/certificate/certificate_store_test.go b/util/certificate/certificate_store_test.go new file mode 100644 index 00000000..5d4d860d --- /dev/null +++ b/util/certificate/certificate_store_test.go @@ -0,0 +1,505 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certificate + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "k8s.io/client-go/util/cert" +) + +func TestUpdateSymlinkExistingFileError(t *testing.T) { + dir, err := ioutil.TempDir("", "k8s-test-update-symlink") + if err != nil { + t.Fatalf("Unable to create the test directory %q: %v", dir, err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("Unable to clean up test directory %q: %v", dir, err) + } + }() + pairFile := filepath.Join(dir, "kubelet-current.pem") + if err := ioutil.WriteFile(pairFile, nil, 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", pairFile, err) + } + + s := fileStore{ + certDirectory: dir, + pairNamePrefix: "kubelet", + } + if err := s.updateSymlink(pairFile); err == nil { + t.Errorf("Got no error, wanted to fail updating the symlink because there is a file there.") + } +} + +func TestUpdateSymlinkNewFileNotExist(t *testing.T) { + dir, err := ioutil.TempDir("", "k8s-test-update-symlink") + if err != nil { + t.Fatalf("Unable to create the test directory %q: %v", dir, err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("Unable to clean up test directory %q: %v", dir, err) + } + }() + oldPairFile := filepath.Join(dir, "kubelet-oldpair.pem") + if err := ioutil.WriteFile(oldPairFile, nil, 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", oldPairFile, err) + } + + s := fileStore{ + certDirectory: dir, + pairNamePrefix: "kubelet", + } + if err := s.updateSymlink(oldPairFile); err != nil { + t.Errorf("Got %v, wanted successful update of the symlink to point to %q", err, oldPairFile) + } + + if _, err := os.Stat(oldPairFile); err != nil { + t.Errorf("Got %v, wanted file %q to be there.", oldPairFile, err) + } + + currentPairFile := filepath.Join(dir, "kubelet-current.pem") + if fi, err := os.Lstat(currentPairFile); err != nil { + t.Errorf("Got %v, wanted file %q to be there", currentPairFile, err) + } else if fi.Mode()&os.ModeSymlink != os.ModeSymlink { + t.Errorf("Got %q not a symlink.", currentPairFile) + } + + newPairFile := filepath.Join(dir, "kubelet-newpair.pem") + if err := s.updateSymlink(newPairFile); err == nil { + t.Errorf("Got no error, wanted to fail updating the symlink the file %q does not exist.", newPairFile) + } +} + +func TestUpdateSymlinkNoSymlink(t *testing.T) { + dir, err := ioutil.TempDir("", "k8s-test-update-symlink") + if err != nil { + t.Fatalf("Unable to create the test directory %q: %v", dir, err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("Unable to clean up test directory %q: %v", dir, err) + } + }() + pairFile := filepath.Join(dir, "kubelet-newfile.pem") + if err := ioutil.WriteFile(pairFile, nil, 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", pairFile, err) + } + + s := fileStore{ + certDirectory: dir, + pairNamePrefix: "kubelet", + } + if err := s.updateSymlink(pairFile); err != nil { + t.Errorf("Got error %v, wanted a new symlink to be created", err) + } + + if _, err := os.Stat(pairFile); err != nil { + t.Errorf("Got error %v, wanted file %q to be there", pairFile, err) + } + currentPairFile := filepath.Join(dir, "kubelet-current.pem") + if fi, err := os.Lstat(currentPairFile); err != nil { + t.Errorf("Got %v, wanted %q to be there", currentPairFile, err) + } else if fi.Mode()&os.ModeSymlink != os.ModeSymlink { + t.Errorf("%q not a symlink, wanted a symlink.", currentPairFile) + } +} + +func TestUpdateSymlinkReplaceExistingSymlink(t *testing.T) { + prefix := "kubelet" + dir, err := ioutil.TempDir("", "k8s-test-update-symlink") + if err != nil { + t.Fatalf("Unable to create the test directory %q: %v", dir, err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("Unable to clean up test directory %q: %v", dir, err) + } + }() + oldPairFile := filepath.Join(dir, prefix+"-oldfile.pem") + if err := ioutil.WriteFile(oldPairFile, nil, 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", oldPairFile, err) + } + newPairFile := filepath.Join(dir, prefix+"-newfile.pem") + if err := ioutil.WriteFile(newPairFile, nil, 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", newPairFile, err) + } + currentPairFile := filepath.Join(dir, prefix+"-current.pem") + if err := os.Symlink(oldPairFile, currentPairFile); err != nil { + t.Fatalf("unable to create a symlink from %q to %q: %v", currentPairFile, oldPairFile, err) + } + if resolved, err := os.Readlink(currentPairFile); err != nil { + t.Fatalf("Got %v when attempting to resolve symlink %q", err, currentPairFile) + } else if resolved != oldPairFile { + t.Fatalf("Got %q as resolution of symlink %q, wanted %q", resolved, currentPairFile, oldPairFile) + } + + s := fileStore{ + certDirectory: dir, + pairNamePrefix: prefix, + } + if err := s.updateSymlink(newPairFile); err != nil { + t.Errorf("Got error %v, wanted a new symlink to be created", err) + } + + if _, err := os.Stat(oldPairFile); err != nil { + t.Errorf("Got error %v, wanted file %q to be there", oldPairFile, err) + } + if _, err := os.Stat(newPairFile); err != nil { + t.Errorf("Got error %v, wanted file %q to be there", newPairFile, err) + } + if fi, err := os.Lstat(currentPairFile); err != nil { + t.Errorf("Got %v, wanted %q to be there", currentPairFile, err) + } else if fi.Mode()&os.ModeSymlink != os.ModeSymlink { + t.Errorf("%q not a symlink, wanted a symlink.", currentPairFile) + } + if resolved, err := os.Readlink(currentPairFile); err != nil { + t.Fatalf("Got %v when attempting to resolve symlink %q", err, currentPairFile) + } else if resolved != newPairFile { + t.Fatalf("Got %q as resolution of symlink %q, wanted %q", resolved, currentPairFile, newPairFile) + } +} + +func TestLoadCertKeyBlocksNoFile(t *testing.T) { + dir, err := ioutil.TempDir("", "k8s-test-load-cert-key-blocks") + if err != nil { + t.Fatalf("Unable to create the test directory %q: %v", dir, err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("Unable to clean up test directory %q: %v", dir, err) + } + }() + + pairFile := filepath.Join(dir, "kubelet-pair.pem") + + if _, _, err := loadCertKeyBlocks(pairFile); err == nil { + t.Errorf("Got no error, but expected %q not found.", pairFile) + } +} + +func TestLoadCertKeyBlocksEmptyFile(t *testing.T) { + dir, err := ioutil.TempDir("", "k8s-test-load-cert-key-blocks") + if err != nil { + t.Fatalf("Unable to create the test directory %q: %v", dir, err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("Unable to clean up test directory %q: %v", dir, err) + } + }() + + pairFile := filepath.Join(dir, "kubelet-pair.pem") + if err := ioutil.WriteFile(pairFile, nil, 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", pairFile, err) + } + + if _, _, err := loadCertKeyBlocks(pairFile); err == nil { + t.Errorf("Got no error, but expected %q not found.", pairFile) + } +} + +func TestLoadCertKeyBlocksPartialFile(t *testing.T) { + dir, err := ioutil.TempDir("", "k8s-test-load-cert-key-blocks") + if err != nil { + t.Fatalf("Unable to create the test directory %q: %v", dir, err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("Unable to clean up test directory %q: %v", dir, err) + } + }() + + pairFile := filepath.Join(dir, "kubelet-pair.pem") + if err := ioutil.WriteFile(pairFile, storeCertData.certificatePEM, 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", pairFile, err) + } + + if _, _, err := loadCertKeyBlocks(pairFile); err == nil { + t.Errorf("Got no error, but expected %q invalid.", pairFile) + } +} + +func TestLoadCertKeyBlocks(t *testing.T) { + dir, err := ioutil.TempDir("", "k8s-test-load-cert-key-blocks") + if err != nil { + t.Fatalf("Unable to create the test directory %q: %v", dir, err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("Unable to clean up test directory %q: %v", dir, err) + } + }() + + pairFile := filepath.Join(dir, "kubelet-pair.pem") + data := append(storeCertData.certificatePEM, []byte("\n")...) + data = append(data, storeCertData.keyPEM...) + if err := ioutil.WriteFile(pairFile, data, 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", pairFile, err) + } + + certBlock, keyBlock, err := loadCertKeyBlocks(pairFile) + if err != nil { + t.Errorf("Got %v, but expected no error.", pairFile) + } + if certBlock.Type != cert.CertificateBlockType { + t.Errorf("Got %q loaded from the pair file, expected a %q.", certBlock.Type, cert.CertificateBlockType) + } + if keyBlock.Type != cert.RSAPrivateKeyBlockType { + t.Errorf("Got %q loaded from the pair file, expected a %q.", keyBlock.Type, cert.RSAPrivateKeyBlockType) + } +} + +func TestLoadFile(t *testing.T) { + dir, err := ioutil.TempDir("", "k8s-test-load-cert-key-blocks") + if err != nil { + t.Fatalf("Unable to create the test directory %q: %v", dir, err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("Unable to clean up test directory %q: %v", dir, err) + } + }() + + pairFile := filepath.Join(dir, "kubelet-pair.pem") + data := append(storeCertData.certificatePEM, []byte("\n")...) + data = append(data, storeCertData.keyPEM...) + if err := ioutil.WriteFile(pairFile, data, 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", pairFile, err) + } + + cert, err := loadFile(pairFile) + if err != nil { + t.Fatalf("Could not load certificate from disk: %v", err) + } + if cert == nil { + t.Fatalf("There was no error, but no certificate data was returned.") + } + if cert.Leaf == nil { + t.Fatalf("Got an empty leaf, expected private data.") + } +} + +func TestUpdateNoRotation(t *testing.T) { + prefix := "kubelet-server" + dir, err := ioutil.TempDir("", "k8s-test-certstore-current") + if err != nil { + t.Fatalf("Unable to create the test directory %q: %v", dir, err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("Unable to clean up test directory %q: %v", dir, err) + } + }() + keyFile := filepath.Join(dir, "kubelet.key") + if err := ioutil.WriteFile(keyFile, storeCertData.keyPEM, 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", keyFile, err) + } + certFile := filepath.Join(dir, "kubelet.crt") + if err := ioutil.WriteFile(certFile, storeCertData.certificatePEM, 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", certFile, err) + } + + s, err := NewFileStore(prefix, dir, dir, certFile, keyFile) + if err != nil { + t.Fatalf("Got %v while creating a new store.", err) + } + + cert, err := s.Update(storeCertData.certificatePEM, storeCertData.keyPEM) + if err != nil { + t.Errorf("Got %v while updating certificate store.", err) + } + if cert == nil { + t.Errorf("Got nil certificate, expected something real.") + } +} + +func TestUpdateRotation(t *testing.T) { + prefix := "kubelet-server" + dir, err := ioutil.TempDir("", "k8s-test-certstore-current") + if err != nil { + t.Fatalf("Unable to create the test directory %q: %v", dir, err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("Unable to clean up test directory %q: %v", dir, err) + } + }() + keyFile := filepath.Join(dir, "kubelet.key") + if err := ioutil.WriteFile(keyFile, storeCertData.keyPEM, 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", keyFile, err) + } + certFile := filepath.Join(dir, "kubelet.crt") + if err := ioutil.WriteFile(certFile, storeCertData.certificatePEM, 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", certFile, err) + } + + s, err := NewFileStore(prefix, dir, dir, certFile, keyFile) + if err != nil { + t.Fatalf("Got %v while creating a new store.", err) + } + + cert, err := s.Update(storeCertData.certificatePEM, storeCertData.keyPEM) + if err != nil { + t.Fatalf("Got %v while updating certificate store.", err) + } + if cert == nil { + t.Fatalf("Got nil certificate, expected something real.") + } +} + +func TestUpdateWithBadCertKeyData(t *testing.T) { + prefix := "kubelet-server" + dir, err := ioutil.TempDir("", "k8s-test-certstore-current") + if err != nil { + t.Fatalf("Unable to create the test directory %q: %v", dir, err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("Unable to clean up test directory %q: %v", dir, err) + } + }() + keyFile := filepath.Join(dir, "kubelet.key") + if err := ioutil.WriteFile(keyFile, storeCertData.keyPEM, 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", keyFile, err) + } + certFile := filepath.Join(dir, "kubelet.crt") + if err := ioutil.WriteFile(certFile, storeCertData.certificatePEM, 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", certFile, err) + } + + s, err := NewFileStore(prefix, dir, dir, certFile, keyFile) + if err != nil { + t.Fatalf("Got %v while creating a new store.", err) + } + + cert, err := s.Update([]byte{0, 0}, storeCertData.keyPEM) + if err == nil { + t.Fatalf("Got no error while updating certificate store with invalid data.") + } + if cert != nil { + t.Fatalf("Got %v certificate returned from the update, expected nil.", cert) + } +} + +func TestCurrentPairFile(t *testing.T) { + prefix := "kubelet-server" + dir, err := ioutil.TempDir("", "k8s-test-certstore-current") + if err != nil { + t.Fatalf("Unable to create the test directory %q: %v", dir, err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("Unable to clean up test directory %q: %v", dir, err) + } + }() + pairFile := filepath.Join(dir, prefix+"-pair.pem") + data := append(storeCertData.certificatePEM, []byte("\n")...) + data = append(data, storeCertData.keyPEM...) + if err := ioutil.WriteFile(pairFile, data, 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", pairFile, err) + } + currentFile := filepath.Join(dir, prefix+"-current.pem") + if err := os.Symlink(pairFile, currentFile); err != nil { + t.Fatalf("unable to create a symlink from %q to %q: %v", currentFile, pairFile, err) + } + + store, err := NewFileStore("kubelet-server", dir, dir, "", "") + if err != nil { + t.Fatalf("Failed to initialize certificate store: %v", err) + } + + cert, err := store.Current() + if err != nil { + t.Fatalf("Could not load certificate from disk: %v", err) + } + if cert == nil { + t.Fatalf("There was no error, but no certificate data was returned.") + } + if cert.Leaf == nil { + t.Fatalf("Got an empty leaf, expected private data.") + } +} + +func TestCurrentCertKeyFiles(t *testing.T) { + prefix := "kubelet-server" + dir, err := ioutil.TempDir("", "k8s-test-certstore-current") + if err != nil { + t.Fatalf("Unable to create the test directory %q: %v", dir, err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("Unable to clean up test directory %q: %v", dir, err) + } + }() + certFile := filepath.Join(dir, "kubelet.crt") + if err := ioutil.WriteFile(certFile, storeCertData.certificatePEM, 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", certFile, err) + } + keyFile := filepath.Join(dir, "kubelet.key") + if err := ioutil.WriteFile(keyFile, storeCertData.keyPEM, 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", keyFile, err) + } + + store, err := NewFileStore(prefix, dir, dir, certFile, keyFile) + if err != nil { + t.Fatalf("Failed to initialize certificate store: %v", err) + } + + cert, err := store.Current() + if err != nil { + t.Fatalf("Could not load certificate from disk: %v", err) + } + if cert == nil { + t.Fatalf("There was no error, but no certificate data was returned.") + } + if cert.Leaf == nil { + t.Fatalf("Got an empty leaf, expected private data.") + } +} + +func TestCurrentNoFiles(t *testing.T) { + dir, err := ioutil.TempDir("", "k8s-test-certstore-current") + if err != nil { + t.Fatalf("Unable to create the test directory %q: %v", dir, err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("Unable to clean up test directory %q: %v", dir, err) + } + }() + + store, err := NewFileStore("kubelet-server", dir, dir, "", "") + if err != nil { + t.Fatalf("Failed to initialize certificate store: %v", err) + } + + cert, err := store.Current() + if err == nil { + t.Fatalf("Got no error, expected an error because the cert/key files don't exist.") + } + if _, ok := err.(*NoCertKeyError); !ok { + t.Fatalf("Got error %v, expected NoCertKeyError.", err) + } + if cert != nil { + t.Fatalf("Got certificate, expected no certificate because the cert/key files don't exist.") + } +}