From 855627e5cb1285ccc94e196cc586eb0033ab8b80 Mon Sep 17 00:00:00 2001 From: Jacob Simpson Date: Mon, 6 Feb 2017 09:26:32 -0800 Subject: [PATCH] Rotate the kubelet certificate when about to expire. Changes the kubelet so it doesn't use the cert/key files directly for starting the TLS server. Instead the TLS server reads the cert/key from the new CertificateManager component, which is responsible for requesting new certificates from the Certificate Signing Request API on the API Server. --- pkg/kubelet/BUILD | 1 + pkg/kubelet/certificate/BUILD | 59 +++ .../certificate/certificate_manager.go | 276 ++++++++++ .../certificate/certificate_manager_test.go | 259 ++++++++++ pkg/kubelet/certificate/certificate_store.go | 304 +++++++++++ .../certificate/certificate_store_test.go | 471 ++++++++++++++++++ .../src/k8s.io/client-go/util/cert/cert.go | 6 +- staging/src/k8s.io/client-go/util/cert/pem.go | 29 +- 8 files changed, 1395 insertions(+), 10 deletions(-) create mode 100644 pkg/kubelet/certificate/BUILD create mode 100644 pkg/kubelet/certificate/certificate_manager.go create mode 100644 pkg/kubelet/certificate/certificate_manager_test.go create mode 100644 pkg/kubelet/certificate/certificate_store.go create mode 100644 pkg/kubelet/certificate/certificate_store_test.go diff --git a/pkg/kubelet/BUILD b/pkg/kubelet/BUILD index 59b0f17878d..18f9967c978 100644 --- a/pkg/kubelet/BUILD +++ b/pkg/kubelet/BUILD @@ -234,6 +234,7 @@ filegroup( ":package-srcs", "//pkg/kubelet/api:all-srcs", "//pkg/kubelet/cadvisor:all-srcs", + "//pkg/kubelet/certificate:all-srcs", "//pkg/kubelet/client:all-srcs", "//pkg/kubelet/cm:all-srcs", "//pkg/kubelet/config:all-srcs", diff --git a/pkg/kubelet/certificate/BUILD b/pkg/kubelet/certificate/BUILD new file mode 100644 index 00000000000..68e84fb203c --- /dev/null +++ b/pkg/kubelet/certificate/BUILD @@ -0,0 +1,59 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", + "go_test", +) + +go_library( + name = "go_default_library", + srcs = [ + "certificate_manager.go", + "certificate_store.go", + ], + tags = ["automanaged"], + deps = [ + "//pkg/apis/certificates/v1beta1:go_default_library", + "//pkg/client/clientset_generated/clientset/typed/certificates/v1beta1:go_default_library", + "//pkg/util:go_default_library", + "//vendor:github.com/golang/glog", + "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1", + "//vendor:k8s.io/apimachinery/pkg/fields", + "//vendor:k8s.io/apimachinery/pkg/util/wait", + "//vendor:k8s.io/apimachinery/pkg/watch", + "//vendor:k8s.io/client-go/util/cert", + ], +) + +go_test( + name = "go_default_test", + srcs = [ + "certificate_manager_test.go", + "certificate_store_test.go", + ], + library = ":go_default_library", + tags = ["automanaged"], + deps = [ + "//pkg/apis/certificates/v1beta1:go_default_library", + "//pkg/client/clientset_generated/clientset/typed/certificates/v1beta1:go_default_library", + "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1", + "//vendor:k8s.io/apimachinery/pkg/watch", + "//vendor:k8s.io/client-go/util/cert", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/pkg/kubelet/certificate/certificate_manager.go b/pkg/kubelet/certificate/certificate_manager.go new file mode 100644 index 00000000000..7ff9653763a --- /dev/null +++ b/pkg/kubelet/certificate/certificate_manager.go @@ -0,0 +1,276 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certificate + +import ( + "crypto/ecdsa" + "crypto/elliptic" + cryptorand "crypto/rand" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" + "sync" + "time" + + "github.com/golang/glog" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/util/cert" + certificates "k8s.io/kubernetes/pkg/apis/certificates/v1beta1" + certificatesclient "k8s.io/kubernetes/pkg/client/clientset_generated/clientset/typed/certificates/v1beta1" +) + +const ( + syncPeriod = 1 * time.Hour +) + +// Manager maintains and updates the certificates in use by this certificate +// manager. In the background it communicates with the API server to get new +// certificates for certificates about to expire. +type Manager interface { + // Start the API server status sync loop. + Start() + // GetCertificate gets the current certificate from the certificate + // manager. This function matches the signature required by + // tls.Config.GetCertificate so it can be passed as TLS configuration. A + // TLS server will automatically call back here to get the correct + // certificate when establishing each new connection. + GetCertificate(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) +} + +// Store is responsible for getting and updating the current certificate. +// Depending on the concrete implementation, the backing store for this +// behavior may vary. +type Store interface { + // Current returns the currently selected certificate. + Current() (*tls.Certificate, error) + // Update accepts the PEM data for the cert/key pair and makes the new + // cert/key pair the 'current' pair, that will be returned by future calls + // to Current(). + Update(cert, key []byte) (*tls.Certificate, error) +} + +type manager struct { + certSigningRequestClient certificatesclient.CertificateSigningRequestInterface + template *x509.CertificateRequest + usages []certificates.KeyUsage + certStore Store + certAccessLock sync.RWMutex + cert *tls.Certificate + shouldRotatePercent uint +} + +// NewManager returns a new certificate manager. A certificate manager is +// responsible for being the authoritative source of certificates in the +// Kubelet and handling updates due to rotation. +func NewManager( + certSigningRequestClient certificatesclient.CertificateSigningRequestInterface, + template *x509.CertificateRequest, + usages []certificates.KeyUsage, + certificateStore Store, + certRotationPercent uint) (Manager, error) { + + cert, err := certificateStore.Current() + if err != nil { + return nil, err + } + + if certRotationPercent > 100 { + certRotationPercent = 100 + } + + m := manager{ + certSigningRequestClient: certSigningRequestClient, + template: template, + usages: usages, + certStore: certificateStore, + cert: cert, + shouldRotatePercent: certRotationPercent, + } + + return &m, nil +} + +// GetCertificate returns the certificate that should be used with TLS +// connections. The value returned by this function will change over time as +// the certificate is rotated. If a reference to this method is passed directly +// into the TLS options for a connection, certificate rotation will be handled +// correctly by the underlying go libraries. +// +// tlsOptions := &server.TLSOptions{ +// ... +// GetCertificate: certificateManager.GetCertificate +// ... +// } +// +func (m *manager) GetCertificate(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) { + m.certAccessLock.RLock() + defer m.certAccessLock.RUnlock() + return m.cert, nil +} + +// Start will start the background work of rotating the certificates. +func (m *manager) Start() { + if m.shouldRotatePercent < 1 { + glog.V(2).Infof("Certificate rotation is not enabled.") + return + } + + // Certificate rotation depends on access to the API server certificate + // signing API, so don't start the certificate manager if we don't have a + // client. This will happen on the master, where the kubelet is responsible + // for bootstrapping the pods of the master components. + if m.certSigningRequestClient == nil { + glog.V(2).Infof("Certificate rotation is not enabled, no connection to the apiserver.") + return + } + + glog.V(2).Infof("Certificate rotation is enabled.") + go wait.Forever(func() { + for range time.Tick(syncPeriod) { + err := m.rotateCerts() + if err != nil { + glog.Errorf("Could not rotate certificates: %v", err) + } + } + }, 0) +} + +// shouldRotate looks at how close the current certificate is to expiring and +// decides if it is time to rotate or not. +func (m *manager) shouldRotate() bool { + m.certAccessLock.RLock() + defer m.certAccessLock.RUnlock() + notAfter := m.cert.Leaf.NotAfter + total := notAfter.Sub(m.cert.Leaf.NotBefore) + remaining := notAfter.Sub(time.Now()) + return remaining < 0 || uint(remaining*100/total) < m.shouldRotatePercent +} + +func (m *manager) rotateCerts() error { + if !m.shouldRotate() { + return nil + } + + csrPEM, keyPEM, err := m.generateCSR() + if err != nil { + return err + } + + // Call the Certificate Signing Request API to get a certificate for the + // new private key. + crtPEM, err := requestCertificate(m.certSigningRequestClient, csrPEM, m.usages) + if err != nil { + return fmt.Errorf("unable to get a new key signed: %v", err) + } + + cert, err := m.certStore.Update(crtPEM, keyPEM) + if err != nil { + return fmt.Errorf("unable to store the new cert/key pair: %v", err) + } + + m.certAccessLock.Lock() + defer m.certAccessLock.Unlock() + m.cert = cert + return nil +} + +func (m *manager) generateCSR() (csrPEM []byte, keyPEM []byte, err error) { + // Generate a new private key. + privateKey, err := ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader) + if err != nil { + return nil, nil, fmt.Errorf("unable to generate a new private key: %v", err) + } + der, err := x509.MarshalECPrivateKey(privateKey) + if err != nil { + return nil, nil, fmt.Errorf("unable to marshal the new key to DER: %v", err) + } + + keyPEM = pem.EncodeToMemory(&pem.Block{Type: cert.ECPrivateKeyBlockType, Bytes: der}) + + csrPEM, err = cert.MakeCSRFromTemplate(privateKey, m.template) + if err != nil { + return nil, nil, fmt.Errorf("unable to create a csr from the private key: %v", err) + } + return csrPEM, keyPEM, nil +} + +// requestCertificate will create a certificate signing request using the PEM +// encoded CSR and send it to API server, then it will watch the object's +// status, once approved by API server, it will return the API server's issued +// certificate (pem-encoded). If there is any errors, or the watch timeouts, it +// will return an error. +// +// NOTE This is a copy of a function with the same name in +// k8s.io/kubernetes/pkg/kubelet/util/csr/csr.go, changing only the package that +// CertificateSigningRequestInterface and KeyUsage are imported from. +func requestCertificate(client certificatesclient.CertificateSigningRequestInterface, csrData []byte, usages []certificates.KeyUsage) (certData []byte, err error) { + req, err := client.Create(&certificates.CertificateSigningRequest{ + // Username, UID, Groups will be injected by API server. + TypeMeta: metav1.TypeMeta{Kind: "CertificateSigningRequest"}, + ObjectMeta: metav1.ObjectMeta{GenerateName: "csr-"}, + + Spec: certificates.CertificateSigningRequestSpec{ + Request: csrData, + Usages: usages, + }, + }) + if err != nil { + return nil, fmt.Errorf("cannot create certificate signing request: %v", err) + } + + // Make a default timeout = 3600s. + var defaultTimeoutSeconds int64 = 3600 + certWatch, err := client.Watch(metav1.ListOptions{ + Watch: true, + TimeoutSeconds: &defaultTimeoutSeconds, + FieldSelector: fields.OneTermEqualSelector("metadata.name", req.Name).String(), + }) + if err != nil { + return nil, fmt.Errorf("cannot watch on the certificate signing request: %v", err) + } + defer certWatch.Stop() + ch := certWatch.ResultChan() + + for { + event, ok := <-ch + if !ok { + break + } + + if event.Type == watch.Modified || event.Type == watch.Added { + if event.Object.(*certificates.CertificateSigningRequest).UID != req.UID { + continue + } + status := event.Object.(*certificates.CertificateSigningRequest).Status + for _, c := range status.Conditions { + if c.Type == certificates.CertificateDenied { + return nil, fmt.Errorf("certificate signing request is not approved, reason: %v, message: %v", c.Reason, c.Message) + } + if c.Type == certificates.CertificateApproved && status.Certificate != nil { + return status.Certificate, nil + } + } + } + } + + return nil, fmt.Errorf("watch channel closed") +} diff --git a/pkg/kubelet/certificate/certificate_manager_test.go b/pkg/kubelet/certificate/certificate_manager_test.go new file mode 100644 index 00000000000..5c200aed493 --- /dev/null +++ b/pkg/kubelet/certificate/certificate_manager_test.go @@ -0,0 +1,259 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certificate + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "testing" + "time" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + watch "k8s.io/apimachinery/pkg/watch" + certificates "k8s.io/kubernetes/pkg/apis/certificates/v1beta1" + certificatesclient "k8s.io/kubernetes/pkg/client/clientset_generated/clientset/typed/certificates/v1beta1" +) + +const ( + privateKeyData = `-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA03ppJ1S3xK2UaXIatBPMbstHm8U9fwIFAj3a2WDV6FHo6zi2 +YHVwCwSVnHL6D+Q5mmlbhnUpSD8SGTLk4EESAe2h203iBOBPBhymhTWA/gAEFk23 +aP1/KlubjYN1+eyksA0lOVcO3sCuRZ64yjYJ369IfV1w8APZ4BXoFtU3uuYpjxyF +XlydkbLqQZLrBa1B5E8hEkDn4ywNDptGjRN3gT2GMQwnaCkWiLjGK6AxTCleXnjG +/JyEwbczv0zAE43utcYPW7qk1m5QsKMUAu4/K8y8oGBFy2ygpY1qckcgr5haehOS +IbFEvVd2oqW8NBicKNmSlh0OcAvQQZtaXhLg/QIDAQABAoIBAFkBmUZLerjVkbQ7 +qQ+HkbBD8FSYVESjVfZWkEiTYBRSfSSbDu9UHh8VA97/6U1M8g2SMEpL/17/5J8k +c34LBQg4urmxcuI4gioBXviLx0mgOhglB3+xyZbLTZHm9X2F4t6R+cvDX2fTUsXM +gtvgmJFDlc/lxwXNqSKONct+W+FV/9D2H1Vzf8fQHfa+lltAy8e8MrbmGQTgev+5 +vz/UR/bZz/CHRxXVA6txgvf4AL8BYibxgx6ihW9zKHy6GykqtQ2p0T5XCkObt41S +6KwUmIHP8CHY23MJ9BPIxYH2+lOXFLizB1VFuxRE1W+je7wVWxzQgFS4IMOLVYDD +LtprVQUCgYEA4g9ODbyW5vvyp8mmAWAvgeunOR1aP79IIyHiwefEIup4FNo+K2wZ +QhRPf0LsVvnthJXFWeW9arAWZRWKCFWwISq/cIIB6KXCIIsjiTUe8SYE/8bxAkvL +0lJhWugTpOnFd8oVuRivrsIWL+SXTNiO5JOP3/qfo+HFk3dqjDhXg4MCgYEA73y1 +Cy+8vHweHKr8HTkPF13GAB1I43SvzTnGT2BT9q6Ia+zQDF1dHjnMrswD1v0+6Xmq +lKc5M69WBVuLIAfWfMQy0WANpsEMm5MYHShJ3YEYAqBiSTUWi23nLH/Poos4IUDV +nTAgFuoKFaG/9cLKA736zqJaiJCE/IR2/gqcYX8CgYA5PCjF/5axWt8ALmTyejjt +Cw4mvtDHzRVll8HC2HxnXrgSh4MwGUl32o6aKQaPqu3BIO57qVhA995jr4VoQNG8 +RAd+Y9w53CX/eVsA9UslQTwIyoTg0PIFCUiO7K10lp+hia/gUmjAtXFKpPTNxxK+ +usG1ss3Sf2o3wQdgAy/dIwKBgQCcHa1fZ3UfYcG3ancDDckasFR8ipqTO+PGYt01 +rVPOwSPJRwywosQrCf62C+SM53V1eYyLbx9I5AmtYGmnLbTSjIucFYOQqtPvLspP +Z44PSTI/tBGeK29Q4QoL5h2SljK26q7V0yN4DIUaaODb8mkCW3v967QcxikK+8ce +AAjFPQKBgHnfVRX+00xSeNE0zya1FtQH3db9+fm3IYGK10NI/jTNF6RhUwHJ6X3+ +TR6OhnTQ2j8eAo+6IlLqlDeC1X7GDvaxqstPvGi0lZjoQQGnQqw2m58AMJu3s9fW +2iddptVycNU0+187DIO39cM3o5s0822VUWDbmymD9cW4i8G6Yto9 +-----END RSA PRIVATE KEY-----` + certificateData = `-----BEGIN CERTIFICATE----- +MIIDEzCCAfugAwIBAgIBATANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhrLWEt +bm9kZS12YzFzQDE0ODYzMzM1NDgwHhcNMTcwMjA1MjIyNTQ4WhcNMTgwMjA1MjIy +NTQ4WjAjMSEwHwYDVQQDDBhrLWEtbm9kZS12YzFzQDE0ODYzMzM1NDgwggEiMA0G +CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDTemknVLfErZRpchq0E8xuy0ebxT1/ +AgUCPdrZYNXoUejrOLZgdXALBJWccvoP5DmaaVuGdSlIPxIZMuTgQRIB7aHbTeIE +4E8GHKaFNYD+AAQWTbdo/X8qW5uNg3X57KSwDSU5Vw7ewK5FnrjKNgnfr0h9XXDw +A9ngFegW1Te65imPHIVeXJ2RsupBkusFrUHkTyESQOfjLA0Om0aNE3eBPYYxDCdo +KRaIuMYroDFMKV5eeMb8nITBtzO/TMATje61xg9buqTWblCwoxQC7j8rzLygYEXL +bKCljWpyRyCvmFp6E5IhsUS9V3aipbw0GJwo2ZKWHQ5wC9BBm1peEuD9AgMBAAGj +UjBQMA4GA1UdDwEB/wQEAwICpDATBgNVHSUEDDAKBggrBgEFBQcDATAPBgNVHRMB +Af8EBTADAQH/MBgGA1UdEQQRMA+CDWstYS1ub2RlLXZjMXMwDQYJKoZIhvcNAQEL +BQADggEBAAHap+dwrAuejnIK8X/CA2kp2CNZgK8cQbTz6gHcAF7FESv5fL7BiYbJ +eljhZauh1MSU7hCeXNOK92I1ba7fa8gSdQoSblf9MOmeuNJ4tTwT0y5Cv0dE7anr +EEPWhp5BeHM10lvw/S2uPiN5CNo9pSniMamDcSC4JPXqfRbpqNQkeFOjByb/Y+ez +t+4mGQIouLdHDbx53xc0mmDXEfxwfE5K0gcF8T9EOE/azKlVA8Fk84vjMpVR2gka +O1eRCsCGPAnUCviFgNeH15ug+6N54DTTR6ZV/TTV64FDOcsox9nrhYcmH9sYuITi +0WC0XoXDL9tMOyzRR1ax/a26ks3Q3IY= +-----END CERTIFICATE-----` +) + +func TestNewManagerNoRotation(t *testing.T) { + cert, err := tls.X509KeyPair([]byte(certificateData), []byte(privateKeyData)) + if err != nil { + t.Fatalf("Unable to initialize a certificate: %v", err) + } + + store := &fakeStore{cert: &cert} + if _, err := NewManager(nil, &x509.CertificateRequest{}, []certificates.KeyUsage{}, store, 0); err != nil { + t.Fatalf("Failed to initialize the certificate manager: %v", err) + } +} + +func TestShouldRotate(t *testing.T) { + now := time.Now() + tests := []struct { + name string + notBefore time.Time + notAfter time.Time + shouldRotate bool + }{ + {"half way", now.Add(-24 * time.Hour), now.Add(24 * time.Hour), false}, + {"nearly there", now.Add(-100 * time.Hour), now.Add(1 * time.Hour), true}, + {"just started", now.Add(-1 * time.Hour), now.Add(100 * time.Hour), false}, + } + + for _, test := range tests { + m := manager{ + cert: &tls.Certificate{ + Leaf: &x509.Certificate{ + NotAfter: test.notAfter, + NotBefore: test.notBefore, + }, + }, + template: &x509.CertificateRequest{}, + usages: []certificates.KeyUsage{}, + shouldRotatePercent: 10, + } + + if m.shouldRotate() != test.shouldRotate { + t.Errorf("For test case %s, time %v, a certificate issued for (%v, %v) should rotate should be %t.", + test.name, + now, + m.cert.Leaf.NotBefore, + m.cert.Leaf.NotAfter, + test.shouldRotate) + } + } +} + +func TestRotateCertCreateCSRError(t *testing.T) { + now := time.Now() + m := manager{ + cert: &tls.Certificate{ + Leaf: &x509.Certificate{ + NotAfter: now.Add(-1 * time.Hour), + NotBefore: now.Add(-2 * time.Hour), + }, + }, + template: &x509.CertificateRequest{}, + usages: []certificates.KeyUsage{}, + certSigningRequestClient: fakeClient{ + failureType: createError, + }, + } + + if err := m.rotateCerts(); err == nil { + t.Errorf("Expected an error from 'rotateCerts'.") + } +} + +func TestRotateCertWaitingForResultError(t *testing.T) { + now := time.Now() + m := manager{ + cert: &tls.Certificate{ + Leaf: &x509.Certificate{ + NotAfter: now.Add(-1 * time.Hour), + NotBefore: now.Add(-2 * time.Hour), + }, + }, + template: &x509.CertificateRequest{}, + usages: []certificates.KeyUsage{}, + certSigningRequestClient: fakeClient{ + failureType: watchError, + }, + } + + if err := m.rotateCerts(); err == nil { + t.Errorf("Expected an error receiving results from the CSR request but nothing was received.") + } +} + +type fakeClientFailureType int + +const ( + none fakeClientFailureType = iota + createError + watchError + certificateSigningRequestDenied +) + +type fakeClient struct { + certificatesclient.CertificateSigningRequestInterface + failureType fakeClientFailureType +} + +func (c fakeClient) Create(*certificates.CertificateSigningRequest) (*certificates.CertificateSigningRequest, error) { + if c.failureType == createError { + return nil, fmt.Errorf("Create error") + } + csr := certificates.CertificateSigningRequest{} + csr.UID = "fake-uid" + return &csr, nil +} + +func (c fakeClient) Watch(opts v1.ListOptions) (watch.Interface, error) { + if c.failureType == watchError { + return nil, fmt.Errorf("Watch error") + } + return &fakeWatch{ + failureType: c.failureType, + }, nil +} + +type fakeWatch struct { + failureType fakeClientFailureType +} + +func (w *fakeWatch) Stop() { +} + +func (w *fakeWatch) ResultChan() <-chan watch.Event { + var condition certificates.CertificateSigningRequestCondition + if w.failureType == certificateSigningRequestDenied { + condition = certificates.CertificateSigningRequestCondition{ + Type: certificates.CertificateDenied, + } + } else { + condition = certificates.CertificateSigningRequestCondition{ + Type: certificates.CertificateApproved, + } + } + + csr := certificates.CertificateSigningRequest{ + Status: certificates.CertificateSigningRequestStatus{ + Conditions: []certificates.CertificateSigningRequestCondition{ + condition, + }, + Certificate: []byte(certificateData), + }, + } + csr.UID = "fake-uid" + + c := make(chan watch.Event, 1) + c <- watch.Event{ + Type: watch.Added, + Object: &csr, + } + return c +} + +type fakeStore struct { + cert *tls.Certificate +} + +func (s *fakeStore) Current() (*tls.Certificate, error) { + return s.cert, nil +} + +// Accepts the PEM data for the cert/key pair and makes the new cert/key +// pair the 'current' pair, that will be returned by future calls to +// Current(). +func (s *fakeStore) Update(certPEM, keyPEM []byte) (*tls.Certificate, error) { + cert, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + return nil, err + } + s.cert = &cert + return s.cert, nil +} diff --git a/pkg/kubelet/certificate/certificate_store.go b/pkg/kubelet/certificate/certificate_store.go new file mode 100644 index 00000000000..02f5faaf738 --- /dev/null +++ b/pkg/kubelet/certificate/certificate_store.go @@ -0,0 +1,304 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certificate + +import ( + "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "time" + + "github.com/golang/glog" + + "k8s.io/kubernetes/pkg/util" +) + +const ( + keyExtension = ".key" + certExtension = ".crt" + pemExtension = ".pem" + currentPair = "current" + updatedPair = "updated" +) + +type fileStore struct { + pairNamePrefix string + certDirectory string + keyDirectory string + certFile string + keyFile string +} + +// NewFileStore returns a concrete implementation of a Store that is based on +// storing the cert/key pairs in a single file per pair on disk in the +// designated directory. When starting up it will look for the currently +// selected cert/key pair in: +// +// 1. ${certDirectory}/${pairNamePrefix}-current.pem - both cert and key are in the same file. +// 2. ${certFile}, ${keyFile} +// 3. ${certDirectory}/${pairNamePrefix}.crt, ${keyDirectory}/${pairNamePrefix}.key +// +// The first one found will be used. If rotation is enabled, future cert/key +// updates will be written to the ${certDirectory} directory and +// ${certDirectory}/${pairNamePrefix}-current.pem will be created as a soft +// link to the currently selected cert/key pair. +func NewFileStore( + pairNamePrefix string, + certDirectory string, + keyDirectory string, + certFile string, + keyFile string) (Store, error) { + + s := fileStore{ + pairNamePrefix: pairNamePrefix, + certDirectory: certDirectory, + keyDirectory: keyDirectory, + certFile: certFile, + keyFile: keyFile, + } + if err := s.recover(); err != nil { + return nil, err + } + return &s, nil +} + +// recover checks if there is a certificate rotation that was interrupted while +// progress, and if so, attempts to recover to a good state. +func (s *fileStore) recover() error { + // If the 'current' file doesn't exist, continue on with the recovery process. + currentPath := filepath.Join(s.certDirectory, s.filename(currentPair)) + if exists, err := util.FileExists(currentPath); err != nil { + return err + } else if exists { + return nil + } + + // If the 'updated' file exists, and it is a symbolic link, continue on + // with the recovery process. + updatedPath := filepath.Join(s.certDirectory, s.filename(updatedPair)) + if fi, err := os.Lstat(updatedPath); err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } else if fi.Mode()&os.ModeSymlink != os.ModeSymlink { + return fmt.Errorf("expected %q to be a symlink but it is a file.", updatedPath) + } + + // Move the 'updated' symlink to 'current'. + if err := os.Rename(updatedPath, currentPath); err != nil { + return fmt.Errorf("unable to rename %q to %q: %v", updatedPath, currentPath, err) + } + return nil +} + +func (s *fileStore) Current() (*tls.Certificate, error) { + pairFile := filepath.Join(s.certDirectory, s.filename(currentPair)) + if pairFileExists, err := util.FileExists(pairFile); err != nil { + return nil, err + } else if pairFileExists { + glog.Infof("Loading cert/key pair from %q.", pairFile) + return loadFile(pairFile) + } + + certFileExists, err := util.FileExists(s.certFile) + if err != nil { + return nil, err + } + keyFileExists, err := util.FileExists(s.keyFile) + if err != nil { + return nil, err + } + if certFileExists && keyFileExists { + glog.Infof("Loading cert/key pair from (%q, %q).", s.certFile, s.keyFile) + return loadX509KeyPair(s.certFile, s.keyFile) + } + + c := filepath.Join(s.certDirectory, s.pairNamePrefix+certExtension) + k := filepath.Join(s.keyDirectory, s.pairNamePrefix+keyExtension) + certFileExists, err = util.FileExists(c) + if err != nil { + return nil, err + } + keyFileExists, err = util.FileExists(k) + if err != nil { + return nil, err + } + if certFileExists && keyFileExists { + glog.Infof("Loading cert/key pair from (%q, %q).", c, k) + return loadX509KeyPair(c, k) + } + + return nil, fmt.Errorf("no cert/key files read at %q, (%q, %q) or (%q, %q)", + pairFile, + s.certFile, + s.keyFile, + s.certDirectory, + s.keyDirectory) +} + +func loadFile(pairFile string) (*tls.Certificate, error) { + certBlock, keyBlock, err := loadCertKeyBlocks(pairFile) + if err != nil { + return nil, err + } + cert, err := tls.X509KeyPair(pem.EncodeToMemory(certBlock), pem.EncodeToMemory(keyBlock)) + if err != nil { + return nil, fmt.Errorf("could not convert data from %q into cert/key pair: %v", pairFile, err) + } + certs, err := x509.ParseCertificates(cert.Certificate[0]) + if err != nil { + return nil, fmt.Errorf("unable to parse certificate data: %v", err) + } + cert.Leaf = certs[0] + return &cert, nil +} + +func loadCertKeyBlocks(pairFile string) (cert *pem.Block, key *pem.Block, err error) { + data, err := ioutil.ReadFile(pairFile) + if err != nil { + return nil, nil, fmt.Errorf("could not load cert/key pair from %q: %v", pairFile, err) + } + certBlock, rest := pem.Decode(data) + if certBlock == nil { + return nil, nil, fmt.Errorf("could not decode the first block from %q from expected PEM format", pairFile) + } + keyBlock, rest := pem.Decode(rest) + if keyBlock == nil { + return nil, nil, fmt.Errorf("could not decode the second block from %q from expected PEM format", pairFile) + } + return certBlock, keyBlock, nil +} + +func (s *fileStore) Update(certData, keyData []byte) (*tls.Certificate, error) { + ts := time.Now().Format("2006-01-02-15-04-05") + pemFilename := s.filename(ts) + + certPath := filepath.Join(s.certDirectory, pemFilename) + + f, err := os.OpenFile(certPath, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0600) + if err != nil { + return nil, fmt.Errorf("could not open %q: %v", certPath, err) + } + defer f.Close() + certBlock, _ := pem.Decode(certData) + if certBlock == nil { + return nil, fmt.Errorf("invalid certificate data") + } + pem.Encode(f, certBlock) + keyBlock, _ := pem.Decode(keyData) + if keyBlock == nil { + return nil, fmt.Errorf("invalid key data") + } + pem.Encode(f, keyBlock) + + cert, err := loadFile(certPath) + if err != nil { + return nil, err + } + + if err := s.updateSymlink(certPath); err != nil { + return nil, err + } + return cert, nil +} + +// updateSymLink updates the current symlink to point to the file that is +// passed it. It will fail if there is a non-symlink file exists where the +// symlink is expected to be. +func (s *fileStore) updateSymlink(filename string) error { + // If the 'current' file either doesn't exist, or is already a symlink, + // proceed. Otherwise, this is an unrecoverable error. + currentPath := filepath.Join(s.certDirectory, s.filename(currentPair)) + currentPathExists := false + if fi, err := os.Lstat(currentPath); err != nil { + if !os.IsNotExist(err) { + return err + } + } else if fi.Mode()&os.ModeSymlink != os.ModeSymlink { + return fmt.Errorf("expected %q to be a symlink but it is a file.", currentPath) + } else { + currentPathExists = true + } + + // If the 'updated' file doesn't exist, proceed. If it exists but it is a + // symlink, delete it. Otherwise, this is an unrecoverable error. + updatedPath := filepath.Join(s.certDirectory, s.filename(updatedPair)) + if fi, err := os.Lstat(updatedPath); err != nil { + if !os.IsNotExist(err) { + return err + } + } else if fi.Mode()&os.ModeSymlink != os.ModeSymlink { + return fmt.Errorf("expected %q to be a symlink but it is a file.", updatedPath) + } else { + if err := os.Remove(updatedPath); err != nil { + return fmt.Errorf("unable to remove %q: %v", updatedPath, err) + } + } + + // Check that the new cert/key pair file exists to avoid rotating to an + // invalid cert/key. + if filenameExists, err := util.FileExists(filename); err != nil { + return err + } else if !filenameExists { + return fmt.Errorf("file %q does not exist so it can not be used as the currently selected cert/key", filename) + } + + // Create the 'updated' symlink pointing to the requested file name. + if err := os.Symlink(filename, updatedPath); err != nil { + return fmt.Errorf("unable to create a symlink from %q to %q: %v", updatedPath, filename, err) + } + + // Replace the 'current' symlink. + if currentPathExists { + if err := os.Remove(currentPath); err != nil { + return fmt.Errorf("unable to remove %q: %v", currentPath, err) + } + } + if err := os.Rename(updatedPath, currentPath); err != nil { + return fmt.Errorf("unable to rename %q to %q: %v", updatedPath, currentPath, err) + } + return nil +} + +func (s *fileStore) filename(qualifier string) string { + return s.pairNamePrefix + "-" + qualifier + pemExtension +} + +// withoutExt returns the given filename after removing the extension. The +// extension to remove will be the result of filepath.Ext(). +func withoutExt(filename string) string { + return strings.TrimSuffix(filename, filepath.Ext(filename)) +} + +func loadX509KeyPair(certFile, keyFile string) (*tls.Certificate, error) { + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, err + } + certs, err := x509.ParseCertificates(cert.Certificate[0]) + if err != nil { + return nil, fmt.Errorf("unable to parse certificate data: %v", err) + } + cert.Leaf = certs[0] + return &cert, nil +} diff --git a/pkg/kubelet/certificate/certificate_store_test.go b/pkg/kubelet/certificate/certificate_store_test.go new file mode 100644 index 00000000000..5769f463a92 --- /dev/null +++ b/pkg/kubelet/certificate/certificate_store_test.go @@ -0,0 +1,471 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certificate + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "k8s.io/client-go/util/cert" +) + +func TestUpdateSymlinkExistingFileError(t *testing.T) { + dir, err := ioutil.TempDir("", "k8s-test-update-symlink") + if err != nil { + t.Fatalf("Unable to create the test directory %q: %v", dir, err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("Unable to clean up test directory %q: %v", dir, err) + } + }() + pairFile := filepath.Join(dir, "kubelet-current.pem") + if err := ioutil.WriteFile(pairFile, nil, 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", pairFile, err) + } + + s := fileStore{ + certDirectory: dir, + pairNamePrefix: "kubelet", + } + if err := s.updateSymlink(pairFile); err == nil { + t.Errorf("Got no error, wanted to fail updating the symlink because there is a file there.") + } +} + +func TestUpdateSymlinkNewFileNotExist(t *testing.T) { + dir, err := ioutil.TempDir("", "k8s-test-update-symlink") + if err != nil { + t.Fatalf("Unable to create the test directory %q: %v", dir, err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("Unable to clean up test directory %q: %v", dir, err) + } + }() + oldPairFile := filepath.Join(dir, "kubelet-oldpair.pem") + if err := ioutil.WriteFile(oldPairFile, nil, 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", oldPairFile, err) + } + + s := fileStore{ + certDirectory: dir, + pairNamePrefix: "kubelet", + } + if err := s.updateSymlink(oldPairFile); err != nil { + t.Errorf("Got %v, wanted successful update of the symlink to point to %q", err, oldPairFile) + } + + if _, err := os.Stat(oldPairFile); err != nil { + t.Errorf("Got %v, wanted file %q to be there.", oldPairFile, err) + } + + currentPairFile := filepath.Join(dir, "kubelet-current.pem") + if fi, err := os.Lstat(currentPairFile); err != nil { + t.Errorf("Got %v, wanted file %q to be there", currentPairFile, err) + } else if fi.Mode()&os.ModeSymlink != os.ModeSymlink { + t.Errorf("Got %q not a symlink.", currentPairFile) + } + + newPairFile := filepath.Join(dir, "kubelet-newpair.pem") + if err := s.updateSymlink(newPairFile); err == nil { + t.Errorf("Got no error, wanted to fail updating the symlink the file %q does not exist.", newPairFile) + } +} + +func TestUpdateSymlinkNoSymlink(t *testing.T) { + dir, err := ioutil.TempDir("", "k8s-test-update-symlink") + if err != nil { + t.Fatalf("Unable to create the test directory %q: %v", dir, err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("Unable to clean up test directory %q: %v", dir, err) + } + }() + pairFile := filepath.Join(dir, "kubelet-newfile.pem") + if err := ioutil.WriteFile(pairFile, nil, 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", pairFile, err) + } + + s := fileStore{ + certDirectory: dir, + pairNamePrefix: "kubelet", + } + if err := s.updateSymlink(pairFile); err != nil { + t.Errorf("Got error %v, wanted a new symlink to be created", err) + } + + if _, err := os.Stat(pairFile); err != nil { + t.Errorf("Got error %v, wanted file %q to be there", pairFile, err) + } + currentPairFile := filepath.Join(dir, "kubelet-current.pem") + if fi, err := os.Lstat(currentPairFile); err != nil { + t.Errorf("Got %v, wanted %q to be there", currentPairFile, err) + } else if fi.Mode()&os.ModeSymlink != os.ModeSymlink { + t.Errorf("%q not a symlink, wanted a symlink.", currentPairFile) + } +} + +func TestUpdateSymlinkReplaceExistingSymlink(t *testing.T) { + prefix := "kubelet" + dir, err := ioutil.TempDir("", "k8s-test-update-symlink") + if err != nil { + t.Fatalf("Unable to create the test directory %q: %v", dir, err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("Unable to clean up test directory %q: %v", dir, err) + } + }() + oldPairFile := filepath.Join(dir, prefix+"-oldfile.pem") + if err := ioutil.WriteFile(oldPairFile, nil, 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", oldPairFile, err) + } + newPairFile := filepath.Join(dir, prefix+"-newfile.pem") + if err := ioutil.WriteFile(newPairFile, nil, 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", newPairFile, err) + } + currentPairFile := filepath.Join(dir, prefix+"-current.pem") + if err := os.Symlink(oldPairFile, currentPairFile); err != nil { + t.Fatalf("unable to create a symlink from %q to %q: %v", currentPairFile, oldPairFile, err) + } + if resolved, err := os.Readlink(currentPairFile); err != nil { + t.Fatalf("Got %v when attempting to resolve symlink %q", err, currentPairFile) + } else if resolved != oldPairFile { + t.Fatalf("Got %q as resolution of symlink %q, wanted %q", resolved, currentPairFile, oldPairFile) + } + + s := fileStore{ + certDirectory: dir, + pairNamePrefix: prefix, + } + if err := s.updateSymlink(newPairFile); err != nil { + t.Errorf("Got error %v, wanted a new symlink to be created", err) + } + + if _, err := os.Stat(oldPairFile); err != nil { + t.Errorf("Got error %v, wanted file %q to be there", oldPairFile, err) + } + if _, err := os.Stat(newPairFile); err != nil { + t.Errorf("Got error %v, wanted file %q to be there", newPairFile, err) + } + if fi, err := os.Lstat(currentPairFile); err != nil { + t.Errorf("Got %v, wanted %q to be there", currentPairFile, err) + } else if fi.Mode()&os.ModeSymlink != os.ModeSymlink { + t.Errorf("%q not a symlink, wanted a symlink.", currentPairFile) + } + if resolved, err := os.Readlink(currentPairFile); err != nil { + t.Fatalf("Got %v when attempting to resolve symlink %q", err, currentPairFile) + } else if resolved != newPairFile { + t.Fatalf("Got %q as resolution of symlink %q, wanted %q", resolved, currentPairFile, newPairFile) + } +} + +func TestLoadCertKeyBlocksNoFile(t *testing.T) { + dir, err := ioutil.TempDir("", "k8s-test-load-cert-key-blocks") + if err != nil { + t.Fatalf("Unable to created the test directory %q: %v", dir, err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("Unable to clean up test directory %q: %v", dir, err) + } + }() + + pairFile := filepath.Join(dir, "kubelet-pair.pem") + + if _, _, err := loadCertKeyBlocks(pairFile); err == nil { + t.Errorf("Got no error, but expected %q not found.", pairFile) + } +} + +func TestLoadCertKeyBlocksEmptyFile(t *testing.T) { + dir, err := ioutil.TempDir("", "k8s-test-load-cert-key-blocks") + if err != nil { + t.Fatalf("Unable to created the test directory %q: %v", dir, err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("Unable to clean up test directory %q: %v", dir, err) + } + }() + + pairFile := filepath.Join(dir, "kubelet-pair.pem") + if err := ioutil.WriteFile(pairFile, nil, 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", pairFile, err) + } + + if _, _, err := loadCertKeyBlocks(pairFile); err == nil { + t.Errorf("Got no error, but expected %q not found.", pairFile) + } +} + +func TestLoadCertKeyBlocksPartialFile(t *testing.T) { + dir, err := ioutil.TempDir("", "k8s-test-load-cert-key-blocks") + if err != nil { + t.Fatalf("Unable to created the test directory %q: %v", dir, err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("Unable to clean up test directory %q: %v", dir, err) + } + }() + + pairFile := filepath.Join(dir, "kubelet-pair.pem") + if err := ioutil.WriteFile(pairFile, []byte(certificateData), 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", pairFile, err) + } + + if _, _, err := loadCertKeyBlocks(pairFile); err == nil { + t.Errorf("Got no error, but expected %q invalid.", pairFile) + } +} + +func TestLoadCertKeyBlocks(t *testing.T) { + dir, err := ioutil.TempDir("", "k8s-test-load-cert-key-blocks") + if err != nil { + t.Fatalf("Unable to created the test directory %q: %v", dir, err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("Unable to clean up test directory %q: %v", dir, err) + } + }() + + pairFile := filepath.Join(dir, "kubelet-pair.pem") + if err := ioutil.WriteFile(pairFile, []byte(certificateData+"\n"+privateKeyData), 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", pairFile, err) + } + + certBlock, keyBlock, err := loadCertKeyBlocks(pairFile) + if err != nil { + t.Errorf("Got %v, but expected no error.", pairFile) + } + if certBlock.Type != cert.CertificateBlockType { + t.Errorf("Got %q loaded from the pair file, expected a %q.", certBlock.Type, cert.CertificateBlockType) + } + if keyBlock.Type != cert.RSAPrivateKeyBlockType { + t.Errorf("Got %q loaded from the pair file, expected a %q.", keyBlock.Type, cert.RSAPrivateKeyBlockType) + } +} + +func TestLoadFile(t *testing.T) { + dir, err := ioutil.TempDir("", "k8s-test-load-cert-key-blocks") + if err != nil { + t.Fatalf("Unable to created the test directory %q: %v", dir, err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("Unable to clean up test directory %q: %v", dir, err) + } + }() + + pairFile := filepath.Join(dir, "kubelet-pair.pem") + if err := ioutil.WriteFile(pairFile, []byte(certificateData+"\n"+privateKeyData), 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", pairFile, err) + } + + cert, err := loadFile(pairFile) + if err != nil { + t.Fatalf("Could not load certificate from disk: %v", err) + } + if cert == nil { + t.Fatalf("There was no error, but no certificate data was returned.") + } + if cert.Leaf == nil { + t.Fatalf("Got an empty leaf, expected private data.") + } +} + +func TestUpdateNoRotation(t *testing.T) { + prefix := "kubelet-server" + dir, err := ioutil.TempDir("", "k8s-test-certstore-current") + if err != nil { + t.Fatalf("Unable to created the test directory %q: %v", dir, err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("Unable to clean up test directory %q: %v", dir, err) + } + }() + keyFile := filepath.Join(dir, "kubelet.key") + if err := ioutil.WriteFile(keyFile, []byte(privateKeyData), 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", keyFile, err) + } + certFile := filepath.Join(dir, "kubelet.crt") + if err := ioutil.WriteFile(certFile, []byte(certificateData), 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", certFile, err) + } + + s, err := NewFileStore(prefix, dir, dir, certFile, keyFile) + if err != nil { + t.Fatalf("Got %v while creating a new store.", err) + } + + cert, err := s.Update([]byte(certificateData), []byte(privateKeyData)) + if err != nil { + t.Errorf("Got %v while updating certificate store.", err) + } + if cert == nil { + t.Errorf("Got nil certificate, expected something real.") + } +} + +func TestUpdateRotation(t *testing.T) { + prefix := "kubelet-server" + dir, err := ioutil.TempDir("", "k8s-test-certstore-current") + if err != nil { + t.Fatalf("Unable to created the test directory %q: %v", dir, err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("Unable to clean up test directory %q: %v", dir, err) + } + }() + keyFile := filepath.Join(dir, "kubelet.key") + if err := ioutil.WriteFile(keyFile, []byte(privateKeyData), 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", keyFile, err) + } + certFile := filepath.Join(dir, "kubelet.crt") + if err := ioutil.WriteFile(certFile, []byte(certificateData), 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", certFile, err) + } + + s, err := NewFileStore(prefix, dir, dir, certFile, keyFile) + if err != nil { + t.Fatalf("Got %v while creating a new store.", err) + } + + cert, err := s.Update([]byte(certificateData), []byte(privateKeyData)) + if err != nil { + t.Fatalf("Got %v while updating certificate store.", err) + } + if cert == nil { + t.Fatalf("Got nil certificate, expected something real.") + } +} + +func TestUpdateWithBadCertKeyData(t *testing.T) { + prefix := "kubelet-server" + dir, err := ioutil.TempDir("", "k8s-test-certstore-current") + if err != nil { + t.Fatalf("Unable to created the test directory %q: %v", dir, err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("Unable to clean up test directory %q: %v", dir, err) + } + }() + keyFile := filepath.Join(dir, "kubelet.key") + if err := ioutil.WriteFile(keyFile, []byte(privateKeyData), 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", keyFile, err) + } + certFile := filepath.Join(dir, "kubelet.crt") + if err := ioutil.WriteFile(certFile, []byte(certificateData), 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", certFile, err) + } + + s, err := NewFileStore(prefix, dir, dir, certFile, keyFile) + if err != nil { + t.Fatalf("Got %v while creating a new store.", err) + } + + cert, err := s.Update([]byte{0, 0}, []byte(privateKeyData)) + if err == nil { + t.Fatalf("Got no error while updating certificate store with invalid data.") + } + if cert != nil { + t.Fatalf("Got %v certificate returned from the update, expected nil.", cert) + } +} + +func TestCurrentPairFile(t *testing.T) { + prefix := "kubelet-server" + dir, err := ioutil.TempDir("", "k8s-test-certstore-current") + if err != nil { + t.Fatalf("Unable to created the test directory %q: %v", dir, err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("Unable to clean up test directory %q: %v", dir, err) + } + }() + pairFile := filepath.Join(dir, prefix+"-pair.pem") + if err := ioutil.WriteFile(pairFile, []byte(certificateData+"\n"+privateKeyData), 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", pairFile, err) + } + currentFile := filepath.Join(dir, prefix+"-current.pem") + if err := os.Symlink(pairFile, currentFile); err != nil { + t.Fatalf("unable to create a symlink from %q to %q: %v", currentFile, pairFile, err) + } + + store, err := NewFileStore("kubelet-server", dir, dir, "", "") + if err != nil { + t.Fatalf("Failed to initialize certificate store: %v", err) + } + + cert, err := store.Current() + if err != nil { + t.Fatalf("Could not load certificate from disk: %v", err) + } + if cert == nil { + t.Fatalf("There was no error, but no certificate data was returned.") + } + if cert.Leaf == nil { + t.Fatalf("Got an empty leaf, expected private data.") + } +} + +func TestCurrentCertKeyFiles(t *testing.T) { + prefix := "kubelet-server" + dir, err := ioutil.TempDir("", "k8s-test-certstore-current") + if err != nil { + t.Fatalf("Unable to created the test directory %q: %v", dir, err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("Unable to clean up test directory %q: %v", dir, err) + } + }() + certFile := filepath.Join(dir, "kubelet.crt") + if err := ioutil.WriteFile(certFile, []byte(certificateData), 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", certFile, err) + } + keyFile := filepath.Join(dir, "kubelet.key") + if err := ioutil.WriteFile(keyFile, []byte(privateKeyData), 0600); err != nil { + t.Fatalf("Unable to create the file %q: %v", keyFile, err) + } + + store, err := NewFileStore(prefix, dir, dir, certFile, keyFile) + if err != nil { + t.Fatalf("Failed to initialize certificate store: %v", err) + } + + cert, err := store.Current() + if err != nil { + t.Fatalf("Could not load certificate from disk: %v", err) + } + if cert == nil { + t.Fatalf("There was no error, but no certificate data was returned.") + } + if cert.Leaf == nil { + t.Fatalf("Got an empty leaf, expected private data.") + } +} diff --git a/staging/src/k8s.io/client-go/util/cert/cert.go b/staging/src/k8s.io/client-go/util/cert/cert.go index 941d6db67c6..6854d4152fe 100644 --- a/staging/src/k8s.io/client-go/util/cert/cert.go +++ b/staging/src/k8s.io/client-go/util/cert/cert.go @@ -128,7 +128,7 @@ func MakeEllipticPrivateKeyPEM() ([]byte, error) { } privateKeyPemBlock := &pem.Block{ - Type: "EC PRIVATE KEY", + Type: ECPrivateKeyBlockType, Bytes: derBytes, } return pem.EncodeToMemory(privateKeyPemBlock), nil @@ -173,13 +173,13 @@ func GenerateSelfSignedCertKey(host string, alternateIPs []net.IP, alternateDNS // Generate cert certBuffer := bytes.Buffer{} - if err := pem.Encode(&certBuffer, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil { + if err := pem.Encode(&certBuffer, &pem.Block{Type: CertificateBlockType, Bytes: derBytes}); err != nil { return nil, nil, err } // Generate key keyBuffer := bytes.Buffer{} - if err := pem.Encode(&keyBuffer, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil { + if err := pem.Encode(&keyBuffer, &pem.Block{Type: RSAPrivateKeyBlockType, Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil { return nil, nil, err } diff --git a/staging/src/k8s.io/client-go/util/cert/pem.go b/staging/src/k8s.io/client-go/util/cert/pem.go index e0b46d88a79..899845857c5 100644 --- a/staging/src/k8s.io/client-go/util/cert/pem.go +++ b/staging/src/k8s.io/client-go/util/cert/pem.go @@ -24,6 +24,21 @@ import ( "fmt" ) +const ( + // ECPrivateKeyBlockType is a possible value for pem.Block.Type. + ECPrivateKeyBlockType = "EC PRIVATE KEY" + // RSAPrivateKeyBlockType is a possible value for pem.Block.Type. + RSAPrivateKeyBlockType = "RSA PRIVATE KEY" + // CertificateBlockType is a possible value for pem.Block.Type. + CertificateBlockType = "CERTIFICATE" + // CertificateRequestBlockType is a possible value for pem.Block.Type. + CertificateRequestBlockType = "CERTIFICATE REQUEST" + // PrivateKeyBlockType is a possible value for pem.Block.Type. + PrivateKeyBlockType = "PRIVATE KEY" + // PublicKeyBlockType is a possible value for pem.Block.Type. + PublicKeyBlockType = "PUBLIC KEY" +) + // EncodePublicKeyPEM returns PEM-endcode public data func EncodePublicKeyPEM(key *rsa.PublicKey) ([]byte, error) { der, err := x509.MarshalPKIXPublicKey(key) @@ -31,7 +46,7 @@ func EncodePublicKeyPEM(key *rsa.PublicKey) ([]byte, error) { return []byte{}, err } block := pem.Block{ - Type: "PUBLIC KEY", + Type: PublicKeyBlockType, Bytes: der, } return pem.EncodeToMemory(&block), nil @@ -40,7 +55,7 @@ func EncodePublicKeyPEM(key *rsa.PublicKey) ([]byte, error) { // EncodePrivateKeyPEM returns PEM-encoded private key data func EncodePrivateKeyPEM(key *rsa.PrivateKey) []byte { block := pem.Block{ - Type: "RSA PRIVATE KEY", + Type: RSAPrivateKeyBlockType, Bytes: x509.MarshalPKCS1PrivateKey(key), } return pem.EncodeToMemory(&block) @@ -49,7 +64,7 @@ func EncodePrivateKeyPEM(key *rsa.PrivateKey) []byte { // EncodeCertPEM returns PEM-endcoded certificate data func EncodeCertPEM(cert *x509.Certificate) []byte { block := pem.Block{ - Type: "CERTIFICATE", + Type: CertificateBlockType, Bytes: cert.Raw, } return pem.EncodeToMemory(&block) @@ -66,17 +81,17 @@ func ParsePrivateKeyPEM(keyData []byte) (interface{}, error) { } switch privateKeyPemBlock.Type { - case "EC PRIVATE KEY": + case ECPrivateKeyBlockType: // ECDSA Private Key in ASN.1 format if key, err := x509.ParseECPrivateKey(privateKeyPemBlock.Bytes); err == nil { return key, nil } - case "RSA PRIVATE KEY": + case RSAPrivateKeyBlockType: // RSA Private Key in PKCS#1 format if key, err := x509.ParsePKCS1PrivateKey(privateKeyPemBlock.Bytes); err == nil { return key, nil } - case "PRIVATE KEY": + case PrivateKeyBlockType: // RSA or ECDSA Private Key in unencrypted PKCS#8 format if key, err := x509.ParsePKCS8PrivateKey(privateKeyPemBlock.Bytes); err == nil { return key, nil @@ -103,7 +118,7 @@ func ParseCertsPEM(pemCerts []byte) ([]*x509.Certificate, error) { break } // Only use PEM "CERTIFICATE" blocks without extra headers - if block.Type != "CERTIFICATE" || len(block.Headers) != 0 { + if block.Type != CertificateBlockType || len(block.Headers) != 0 { continue }