mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-24 20:24:09 +00:00
Merge pull request #41021 from jcbsmpsn/rotate-certificate
Automatic merge from submit-queue (batch tested with PRs 41043, 39058, 41021, 41603, 41414) Rotate the kubelet certificate when about to expire. Changes the kubelet so it doesn't use the cert/key files directly for starting the TLS server. Instead the TLS server reads the cert/key from the new CertificateManager component, which is responsible for requesting new certificates from the Certificate Signing Request API on the API Server.
This commit is contained in:
commit
eb8e1dd5f1
@ -234,6 +234,7 @@ filegroup(
|
||||
":package-srcs",
|
||||
"//pkg/kubelet/api:all-srcs",
|
||||
"//pkg/kubelet/cadvisor:all-srcs",
|
||||
"//pkg/kubelet/certificate:all-srcs",
|
||||
"//pkg/kubelet/client:all-srcs",
|
||||
"//pkg/kubelet/cm:all-srcs",
|
||||
"//pkg/kubelet/config:all-srcs",
|
||||
|
59
pkg/kubelet/certificate/BUILD
Normal file
59
pkg/kubelet/certificate/BUILD
Normal file
@ -0,0 +1,59 @@
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
licenses(["notice"])
|
||||
|
||||
load(
|
||||
"@io_bazel_rules_go//go:def.bzl",
|
||||
"go_library",
|
||||
"go_test",
|
||||
)
|
||||
|
||||
go_library(
|
||||
name = "go_default_library",
|
||||
srcs = [
|
||||
"certificate_manager.go",
|
||||
"certificate_store.go",
|
||||
],
|
||||
tags = ["automanaged"],
|
||||
deps = [
|
||||
"//pkg/apis/certificates/v1beta1:go_default_library",
|
||||
"//pkg/client/clientset_generated/clientset/typed/certificates/v1beta1:go_default_library",
|
||||
"//pkg/util:go_default_library",
|
||||
"//vendor:github.com/golang/glog",
|
||||
"//vendor:k8s.io/apimachinery/pkg/apis/meta/v1",
|
||||
"//vendor:k8s.io/apimachinery/pkg/fields",
|
||||
"//vendor:k8s.io/apimachinery/pkg/util/wait",
|
||||
"//vendor:k8s.io/apimachinery/pkg/watch",
|
||||
"//vendor:k8s.io/client-go/util/cert",
|
||||
],
|
||||
)
|
||||
|
||||
go_test(
|
||||
name = "go_default_test",
|
||||
srcs = [
|
||||
"certificate_manager_test.go",
|
||||
"certificate_store_test.go",
|
||||
],
|
||||
library = ":go_default_library",
|
||||
tags = ["automanaged"],
|
||||
deps = [
|
||||
"//pkg/apis/certificates/v1beta1:go_default_library",
|
||||
"//pkg/client/clientset_generated/clientset/typed/certificates/v1beta1:go_default_library",
|
||||
"//vendor:k8s.io/apimachinery/pkg/apis/meta/v1",
|
||||
"//vendor:k8s.io/apimachinery/pkg/watch",
|
||||
"//vendor:k8s.io/client-go/util/cert",
|
||||
],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "package-srcs",
|
||||
srcs = glob(["**"]),
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:private"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "all-srcs",
|
||||
srcs = [":package-srcs"],
|
||||
tags = ["automanaged"],
|
||||
)
|
276
pkg/kubelet/certificate/certificate_manager.go
Normal file
276
pkg/kubelet/certificate/certificate_manager.go
Normal file
@ -0,0 +1,276 @@
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package certificate
|
||||
|
||||
import (
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
cryptorand "crypto/rand"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/golang/glog"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/fields"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
"k8s.io/client-go/util/cert"
|
||||
certificates "k8s.io/kubernetes/pkg/apis/certificates/v1beta1"
|
||||
certificatesclient "k8s.io/kubernetes/pkg/client/clientset_generated/clientset/typed/certificates/v1beta1"
|
||||
)
|
||||
|
||||
const (
|
||||
syncPeriod = 1 * time.Hour
|
||||
)
|
||||
|
||||
// Manager maintains and updates the certificates in use by this certificate
|
||||
// manager. In the background it communicates with the API server to get new
|
||||
// certificates for certificates about to expire.
|
||||
type Manager interface {
|
||||
// Start the API server status sync loop.
|
||||
Start()
|
||||
// GetCertificate gets the current certificate from the certificate
|
||||
// manager. This function matches the signature required by
|
||||
// tls.Config.GetCertificate so it can be passed as TLS configuration. A
|
||||
// TLS server will automatically call back here to get the correct
|
||||
// certificate when establishing each new connection.
|
||||
GetCertificate(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error)
|
||||
}
|
||||
|
||||
// Store is responsible for getting and updating the current certificate.
|
||||
// Depending on the concrete implementation, the backing store for this
|
||||
// behavior may vary.
|
||||
type Store interface {
|
||||
// Current returns the currently selected certificate.
|
||||
Current() (*tls.Certificate, error)
|
||||
// Update accepts the PEM data for the cert/key pair and makes the new
|
||||
// cert/key pair the 'current' pair, that will be returned by future calls
|
||||
// to Current().
|
||||
Update(cert, key []byte) (*tls.Certificate, error)
|
||||
}
|
||||
|
||||
type manager struct {
|
||||
certSigningRequestClient certificatesclient.CertificateSigningRequestInterface
|
||||
template *x509.CertificateRequest
|
||||
usages []certificates.KeyUsage
|
||||
certStore Store
|
||||
certAccessLock sync.RWMutex
|
||||
cert *tls.Certificate
|
||||
shouldRotatePercent uint
|
||||
}
|
||||
|
||||
// NewManager returns a new certificate manager. A certificate manager is
|
||||
// responsible for being the authoritative source of certificates in the
|
||||
// Kubelet and handling updates due to rotation.
|
||||
func NewManager(
|
||||
certSigningRequestClient certificatesclient.CertificateSigningRequestInterface,
|
||||
template *x509.CertificateRequest,
|
||||
usages []certificates.KeyUsage,
|
||||
certificateStore Store,
|
||||
certRotationPercent uint) (Manager, error) {
|
||||
|
||||
cert, err := certificateStore.Current()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if certRotationPercent > 100 {
|
||||
certRotationPercent = 100
|
||||
}
|
||||
|
||||
m := manager{
|
||||
certSigningRequestClient: certSigningRequestClient,
|
||||
template: template,
|
||||
usages: usages,
|
||||
certStore: certificateStore,
|
||||
cert: cert,
|
||||
shouldRotatePercent: certRotationPercent,
|
||||
}
|
||||
|
||||
return &m, nil
|
||||
}
|
||||
|
||||
// GetCertificate returns the certificate that should be used with TLS
|
||||
// connections. The value returned by this function will change over time as
|
||||
// the certificate is rotated. If a reference to this method is passed directly
|
||||
// into the TLS options for a connection, certificate rotation will be handled
|
||||
// correctly by the underlying go libraries.
|
||||
//
|
||||
// tlsOptions := &server.TLSOptions{
|
||||
// ...
|
||||
// GetCertificate: certificateManager.GetCertificate
|
||||
// ...
|
||||
// }
|
||||
//
|
||||
func (m *manager) GetCertificate(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) {
|
||||
m.certAccessLock.RLock()
|
||||
defer m.certAccessLock.RUnlock()
|
||||
return m.cert, nil
|
||||
}
|
||||
|
||||
// Start will start the background work of rotating the certificates.
|
||||
func (m *manager) Start() {
|
||||
if m.shouldRotatePercent < 1 {
|
||||
glog.V(2).Infof("Certificate rotation is not enabled.")
|
||||
return
|
||||
}
|
||||
|
||||
// Certificate rotation depends on access to the API server certificate
|
||||
// signing API, so don't start the certificate manager if we don't have a
|
||||
// client. This will happen on the master, where the kubelet is responsible
|
||||
// for bootstrapping the pods of the master components.
|
||||
if m.certSigningRequestClient == nil {
|
||||
glog.V(2).Infof("Certificate rotation is not enabled, no connection to the apiserver.")
|
||||
return
|
||||
}
|
||||
|
||||
glog.V(2).Infof("Certificate rotation is enabled.")
|
||||
go wait.Forever(func() {
|
||||
for range time.Tick(syncPeriod) {
|
||||
err := m.rotateCerts()
|
||||
if err != nil {
|
||||
glog.Errorf("Could not rotate certificates: %v", err)
|
||||
}
|
||||
}
|
||||
}, 0)
|
||||
}
|
||||
|
||||
// shouldRotate looks at how close the current certificate is to expiring and
|
||||
// decides if it is time to rotate or not.
|
||||
func (m *manager) shouldRotate() bool {
|
||||
m.certAccessLock.RLock()
|
||||
defer m.certAccessLock.RUnlock()
|
||||
notAfter := m.cert.Leaf.NotAfter
|
||||
total := notAfter.Sub(m.cert.Leaf.NotBefore)
|
||||
remaining := notAfter.Sub(time.Now())
|
||||
return remaining < 0 || uint(remaining*100/total) < m.shouldRotatePercent
|
||||
}
|
||||
|
||||
func (m *manager) rotateCerts() error {
|
||||
if !m.shouldRotate() {
|
||||
return nil
|
||||
}
|
||||
|
||||
csrPEM, keyPEM, err := m.generateCSR()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Call the Certificate Signing Request API to get a certificate for the
|
||||
// new private key.
|
||||
crtPEM, err := requestCertificate(m.certSigningRequestClient, csrPEM, m.usages)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to get a new key signed: %v", err)
|
||||
}
|
||||
|
||||
cert, err := m.certStore.Update(crtPEM, keyPEM)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to store the new cert/key pair: %v", err)
|
||||
}
|
||||
|
||||
m.certAccessLock.Lock()
|
||||
defer m.certAccessLock.Unlock()
|
||||
m.cert = cert
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *manager) generateCSR() (csrPEM []byte, keyPEM []byte, err error) {
|
||||
// Generate a new private key.
|
||||
privateKey, err := ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("unable to generate a new private key: %v", err)
|
||||
}
|
||||
der, err := x509.MarshalECPrivateKey(privateKey)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("unable to marshal the new key to DER: %v", err)
|
||||
}
|
||||
|
||||
keyPEM = pem.EncodeToMemory(&pem.Block{Type: cert.ECPrivateKeyBlockType, Bytes: der})
|
||||
|
||||
csrPEM, err = cert.MakeCSRFromTemplate(privateKey, m.template)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("unable to create a csr from the private key: %v", err)
|
||||
}
|
||||
return csrPEM, keyPEM, nil
|
||||
}
|
||||
|
||||
// requestCertificate will create a certificate signing request using the PEM
|
||||
// encoded CSR and send it to API server, then it will watch the object's
|
||||
// status, once approved by API server, it will return the API server's issued
|
||||
// certificate (pem-encoded). If there is any errors, or the watch timeouts, it
|
||||
// will return an error.
|
||||
//
|
||||
// NOTE This is a copy of a function with the same name in
|
||||
// k8s.io/kubernetes/pkg/kubelet/util/csr/csr.go, changing only the package that
|
||||
// CertificateSigningRequestInterface and KeyUsage are imported from.
|
||||
func requestCertificate(client certificatesclient.CertificateSigningRequestInterface, csrData []byte, usages []certificates.KeyUsage) (certData []byte, err error) {
|
||||
req, err := client.Create(&certificates.CertificateSigningRequest{
|
||||
// Username, UID, Groups will be injected by API server.
|
||||
TypeMeta: metav1.TypeMeta{Kind: "CertificateSigningRequest"},
|
||||
ObjectMeta: metav1.ObjectMeta{GenerateName: "csr-"},
|
||||
|
||||
Spec: certificates.CertificateSigningRequestSpec{
|
||||
Request: csrData,
|
||||
Usages: usages,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot create certificate signing request: %v", err)
|
||||
}
|
||||
|
||||
// Make a default timeout = 3600s.
|
||||
var defaultTimeoutSeconds int64 = 3600
|
||||
certWatch, err := client.Watch(metav1.ListOptions{
|
||||
Watch: true,
|
||||
TimeoutSeconds: &defaultTimeoutSeconds,
|
||||
FieldSelector: fields.OneTermEqualSelector("metadata.name", req.Name).String(),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot watch on the certificate signing request: %v", err)
|
||||
}
|
||||
defer certWatch.Stop()
|
||||
ch := certWatch.ResultChan()
|
||||
|
||||
for {
|
||||
event, ok := <-ch
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
|
||||
if event.Type == watch.Modified || event.Type == watch.Added {
|
||||
if event.Object.(*certificates.CertificateSigningRequest).UID != req.UID {
|
||||
continue
|
||||
}
|
||||
status := event.Object.(*certificates.CertificateSigningRequest).Status
|
||||
for _, c := range status.Conditions {
|
||||
if c.Type == certificates.CertificateDenied {
|
||||
return nil, fmt.Errorf("certificate signing request is not approved, reason: %v, message: %v", c.Reason, c.Message)
|
||||
}
|
||||
if c.Type == certificates.CertificateApproved && status.Certificate != nil {
|
||||
return status.Certificate, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("watch channel closed")
|
||||
}
|
259
pkg/kubelet/certificate/certificate_manager_test.go
Normal file
259
pkg/kubelet/certificate/certificate_manager_test.go
Normal file
@ -0,0 +1,259 @@
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package certificate
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
watch "k8s.io/apimachinery/pkg/watch"
|
||||
certificates "k8s.io/kubernetes/pkg/apis/certificates/v1beta1"
|
||||
certificatesclient "k8s.io/kubernetes/pkg/client/clientset_generated/clientset/typed/certificates/v1beta1"
|
||||
)
|
||||
|
||||
const (
|
||||
privateKeyData = `-----BEGIN RSA PRIVATE KEY-----
|
||||
MIIEowIBAAKCAQEA03ppJ1S3xK2UaXIatBPMbstHm8U9fwIFAj3a2WDV6FHo6zi2
|
||||
YHVwCwSVnHL6D+Q5mmlbhnUpSD8SGTLk4EESAe2h203iBOBPBhymhTWA/gAEFk23
|
||||
aP1/KlubjYN1+eyksA0lOVcO3sCuRZ64yjYJ369IfV1w8APZ4BXoFtU3uuYpjxyF
|
||||
XlydkbLqQZLrBa1B5E8hEkDn4ywNDptGjRN3gT2GMQwnaCkWiLjGK6AxTCleXnjG
|
||||
/JyEwbczv0zAE43utcYPW7qk1m5QsKMUAu4/K8y8oGBFy2ygpY1qckcgr5haehOS
|
||||
IbFEvVd2oqW8NBicKNmSlh0OcAvQQZtaXhLg/QIDAQABAoIBAFkBmUZLerjVkbQ7
|
||||
qQ+HkbBD8FSYVESjVfZWkEiTYBRSfSSbDu9UHh8VA97/6U1M8g2SMEpL/17/5J8k
|
||||
c34LBQg4urmxcuI4gioBXviLx0mgOhglB3+xyZbLTZHm9X2F4t6R+cvDX2fTUsXM
|
||||
gtvgmJFDlc/lxwXNqSKONct+W+FV/9D2H1Vzf8fQHfa+lltAy8e8MrbmGQTgev+5
|
||||
vz/UR/bZz/CHRxXVA6txgvf4AL8BYibxgx6ihW9zKHy6GykqtQ2p0T5XCkObt41S
|
||||
6KwUmIHP8CHY23MJ9BPIxYH2+lOXFLizB1VFuxRE1W+je7wVWxzQgFS4IMOLVYDD
|
||||
LtprVQUCgYEA4g9ODbyW5vvyp8mmAWAvgeunOR1aP79IIyHiwefEIup4FNo+K2wZ
|
||||
QhRPf0LsVvnthJXFWeW9arAWZRWKCFWwISq/cIIB6KXCIIsjiTUe8SYE/8bxAkvL
|
||||
0lJhWugTpOnFd8oVuRivrsIWL+SXTNiO5JOP3/qfo+HFk3dqjDhXg4MCgYEA73y1
|
||||
Cy+8vHweHKr8HTkPF13GAB1I43SvzTnGT2BT9q6Ia+zQDF1dHjnMrswD1v0+6Xmq
|
||||
lKc5M69WBVuLIAfWfMQy0WANpsEMm5MYHShJ3YEYAqBiSTUWi23nLH/Poos4IUDV
|
||||
nTAgFuoKFaG/9cLKA736zqJaiJCE/IR2/gqcYX8CgYA5PCjF/5axWt8ALmTyejjt
|
||||
Cw4mvtDHzRVll8HC2HxnXrgSh4MwGUl32o6aKQaPqu3BIO57qVhA995jr4VoQNG8
|
||||
RAd+Y9w53CX/eVsA9UslQTwIyoTg0PIFCUiO7K10lp+hia/gUmjAtXFKpPTNxxK+
|
||||
usG1ss3Sf2o3wQdgAy/dIwKBgQCcHa1fZ3UfYcG3ancDDckasFR8ipqTO+PGYt01
|
||||
rVPOwSPJRwywosQrCf62C+SM53V1eYyLbx9I5AmtYGmnLbTSjIucFYOQqtPvLspP
|
||||
Z44PSTI/tBGeK29Q4QoL5h2SljK26q7V0yN4DIUaaODb8mkCW3v967QcxikK+8ce
|
||||
AAjFPQKBgHnfVRX+00xSeNE0zya1FtQH3db9+fm3IYGK10NI/jTNF6RhUwHJ6X3+
|
||||
TR6OhnTQ2j8eAo+6IlLqlDeC1X7GDvaxqstPvGi0lZjoQQGnQqw2m58AMJu3s9fW
|
||||
2iddptVycNU0+187DIO39cM3o5s0822VUWDbmymD9cW4i8G6Yto9
|
||||
-----END RSA PRIVATE KEY-----`
|
||||
certificateData = `-----BEGIN CERTIFICATE-----
|
||||
MIIDEzCCAfugAwIBAgIBATANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhrLWEt
|
||||
bm9kZS12YzFzQDE0ODYzMzM1NDgwHhcNMTcwMjA1MjIyNTQ4WhcNMTgwMjA1MjIy
|
||||
NTQ4WjAjMSEwHwYDVQQDDBhrLWEtbm9kZS12YzFzQDE0ODYzMzM1NDgwggEiMA0G
|
||||
CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDTemknVLfErZRpchq0E8xuy0ebxT1/
|
||||
AgUCPdrZYNXoUejrOLZgdXALBJWccvoP5DmaaVuGdSlIPxIZMuTgQRIB7aHbTeIE
|
||||
4E8GHKaFNYD+AAQWTbdo/X8qW5uNg3X57KSwDSU5Vw7ewK5FnrjKNgnfr0h9XXDw
|
||||
A9ngFegW1Te65imPHIVeXJ2RsupBkusFrUHkTyESQOfjLA0Om0aNE3eBPYYxDCdo
|
||||
KRaIuMYroDFMKV5eeMb8nITBtzO/TMATje61xg9buqTWblCwoxQC7j8rzLygYEXL
|
||||
bKCljWpyRyCvmFp6E5IhsUS9V3aipbw0GJwo2ZKWHQ5wC9BBm1peEuD9AgMBAAGj
|
||||
UjBQMA4GA1UdDwEB/wQEAwICpDATBgNVHSUEDDAKBggrBgEFBQcDATAPBgNVHRMB
|
||||
Af8EBTADAQH/MBgGA1UdEQQRMA+CDWstYS1ub2RlLXZjMXMwDQYJKoZIhvcNAQEL
|
||||
BQADggEBAAHap+dwrAuejnIK8X/CA2kp2CNZgK8cQbTz6gHcAF7FESv5fL7BiYbJ
|
||||
eljhZauh1MSU7hCeXNOK92I1ba7fa8gSdQoSblf9MOmeuNJ4tTwT0y5Cv0dE7anr
|
||||
EEPWhp5BeHM10lvw/S2uPiN5CNo9pSniMamDcSC4JPXqfRbpqNQkeFOjByb/Y+ez
|
||||
t+4mGQIouLdHDbx53xc0mmDXEfxwfE5K0gcF8T9EOE/azKlVA8Fk84vjMpVR2gka
|
||||
O1eRCsCGPAnUCviFgNeH15ug+6N54DTTR6ZV/TTV64FDOcsox9nrhYcmH9sYuITi
|
||||
0WC0XoXDL9tMOyzRR1ax/a26ks3Q3IY=
|
||||
-----END CERTIFICATE-----`
|
||||
)
|
||||
|
||||
func TestNewManagerNoRotation(t *testing.T) {
|
||||
cert, err := tls.X509KeyPair([]byte(certificateData), []byte(privateKeyData))
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to initialize a certificate: %v", err)
|
||||
}
|
||||
|
||||
store := &fakeStore{cert: &cert}
|
||||
if _, err := NewManager(nil, &x509.CertificateRequest{}, []certificates.KeyUsage{}, store, 0); err != nil {
|
||||
t.Fatalf("Failed to initialize the certificate manager: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldRotate(t *testing.T) {
|
||||
now := time.Now()
|
||||
tests := []struct {
|
||||
name string
|
||||
notBefore time.Time
|
||||
notAfter time.Time
|
||||
shouldRotate bool
|
||||
}{
|
||||
{"half way", now.Add(-24 * time.Hour), now.Add(24 * time.Hour), false},
|
||||
{"nearly there", now.Add(-100 * time.Hour), now.Add(1 * time.Hour), true},
|
||||
{"just started", now.Add(-1 * time.Hour), now.Add(100 * time.Hour), false},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
m := manager{
|
||||
cert: &tls.Certificate{
|
||||
Leaf: &x509.Certificate{
|
||||
NotAfter: test.notAfter,
|
||||
NotBefore: test.notBefore,
|
||||
},
|
||||
},
|
||||
template: &x509.CertificateRequest{},
|
||||
usages: []certificates.KeyUsage{},
|
||||
shouldRotatePercent: 10,
|
||||
}
|
||||
|
||||
if m.shouldRotate() != test.shouldRotate {
|
||||
t.Errorf("For test case %s, time %v, a certificate issued for (%v, %v) should rotate should be %t.",
|
||||
test.name,
|
||||
now,
|
||||
m.cert.Leaf.NotBefore,
|
||||
m.cert.Leaf.NotAfter,
|
||||
test.shouldRotate)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRotateCertCreateCSRError(t *testing.T) {
|
||||
now := time.Now()
|
||||
m := manager{
|
||||
cert: &tls.Certificate{
|
||||
Leaf: &x509.Certificate{
|
||||
NotAfter: now.Add(-1 * time.Hour),
|
||||
NotBefore: now.Add(-2 * time.Hour),
|
||||
},
|
||||
},
|
||||
template: &x509.CertificateRequest{},
|
||||
usages: []certificates.KeyUsage{},
|
||||
certSigningRequestClient: fakeClient{
|
||||
failureType: createError,
|
||||
},
|
||||
}
|
||||
|
||||
if err := m.rotateCerts(); err == nil {
|
||||
t.Errorf("Expected an error from 'rotateCerts'.")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRotateCertWaitingForResultError(t *testing.T) {
|
||||
now := time.Now()
|
||||
m := manager{
|
||||
cert: &tls.Certificate{
|
||||
Leaf: &x509.Certificate{
|
||||
NotAfter: now.Add(-1 * time.Hour),
|
||||
NotBefore: now.Add(-2 * time.Hour),
|
||||
},
|
||||
},
|
||||
template: &x509.CertificateRequest{},
|
||||
usages: []certificates.KeyUsage{},
|
||||
certSigningRequestClient: fakeClient{
|
||||
failureType: watchError,
|
||||
},
|
||||
}
|
||||
|
||||
if err := m.rotateCerts(); err == nil {
|
||||
t.Errorf("Expected an error receiving results from the CSR request but nothing was received.")
|
||||
}
|
||||
}
|
||||
|
||||
type fakeClientFailureType int
|
||||
|
||||
const (
|
||||
none fakeClientFailureType = iota
|
||||
createError
|
||||
watchError
|
||||
certificateSigningRequestDenied
|
||||
)
|
||||
|
||||
type fakeClient struct {
|
||||
certificatesclient.CertificateSigningRequestInterface
|
||||
failureType fakeClientFailureType
|
||||
}
|
||||
|
||||
func (c fakeClient) Create(*certificates.CertificateSigningRequest) (*certificates.CertificateSigningRequest, error) {
|
||||
if c.failureType == createError {
|
||||
return nil, fmt.Errorf("Create error")
|
||||
}
|
||||
csr := certificates.CertificateSigningRequest{}
|
||||
csr.UID = "fake-uid"
|
||||
return &csr, nil
|
||||
}
|
||||
|
||||
func (c fakeClient) Watch(opts v1.ListOptions) (watch.Interface, error) {
|
||||
if c.failureType == watchError {
|
||||
return nil, fmt.Errorf("Watch error")
|
||||
}
|
||||
return &fakeWatch{
|
||||
failureType: c.failureType,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type fakeWatch struct {
|
||||
failureType fakeClientFailureType
|
||||
}
|
||||
|
||||
func (w *fakeWatch) Stop() {
|
||||
}
|
||||
|
||||
func (w *fakeWatch) ResultChan() <-chan watch.Event {
|
||||
var condition certificates.CertificateSigningRequestCondition
|
||||
if w.failureType == certificateSigningRequestDenied {
|
||||
condition = certificates.CertificateSigningRequestCondition{
|
||||
Type: certificates.CertificateDenied,
|
||||
}
|
||||
} else {
|
||||
condition = certificates.CertificateSigningRequestCondition{
|
||||
Type: certificates.CertificateApproved,
|
||||
}
|
||||
}
|
||||
|
||||
csr := certificates.CertificateSigningRequest{
|
||||
Status: certificates.CertificateSigningRequestStatus{
|
||||
Conditions: []certificates.CertificateSigningRequestCondition{
|
||||
condition,
|
||||
},
|
||||
Certificate: []byte(certificateData),
|
||||
},
|
||||
}
|
||||
csr.UID = "fake-uid"
|
||||
|
||||
c := make(chan watch.Event, 1)
|
||||
c <- watch.Event{
|
||||
Type: watch.Added,
|
||||
Object: &csr,
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
type fakeStore struct {
|
||||
cert *tls.Certificate
|
||||
}
|
||||
|
||||
func (s *fakeStore) Current() (*tls.Certificate, error) {
|
||||
return s.cert, nil
|
||||
}
|
||||
|
||||
// Accepts the PEM data for the cert/key pair and makes the new cert/key
|
||||
// pair the 'current' pair, that will be returned by future calls to
|
||||
// Current().
|
||||
func (s *fakeStore) Update(certPEM, keyPEM []byte) (*tls.Certificate, error) {
|
||||
cert, err := tls.X509KeyPair(certPEM, keyPEM)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.cert = &cert
|
||||
return s.cert, nil
|
||||
}
|
304
pkg/kubelet/certificate/certificate_store.go
Normal file
304
pkg/kubelet/certificate/certificate_store.go
Normal file
@ -0,0 +1,304 @@
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package certificate
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/golang/glog"
|
||||
|
||||
"k8s.io/kubernetes/pkg/util"
|
||||
)
|
||||
|
||||
const (
|
||||
keyExtension = ".key"
|
||||
certExtension = ".crt"
|
||||
pemExtension = ".pem"
|
||||
currentPair = "current"
|
||||
updatedPair = "updated"
|
||||
)
|
||||
|
||||
type fileStore struct {
|
||||
pairNamePrefix string
|
||||
certDirectory string
|
||||
keyDirectory string
|
||||
certFile string
|
||||
keyFile string
|
||||
}
|
||||
|
||||
// NewFileStore returns a concrete implementation of a Store that is based on
|
||||
// storing the cert/key pairs in a single file per pair on disk in the
|
||||
// designated directory. When starting up it will look for the currently
|
||||
// selected cert/key pair in:
|
||||
//
|
||||
// 1. ${certDirectory}/${pairNamePrefix}-current.pem - both cert and key are in the same file.
|
||||
// 2. ${certFile}, ${keyFile}
|
||||
// 3. ${certDirectory}/${pairNamePrefix}.crt, ${keyDirectory}/${pairNamePrefix}.key
|
||||
//
|
||||
// The first one found will be used. If rotation is enabled, future cert/key
|
||||
// updates will be written to the ${certDirectory} directory and
|
||||
// ${certDirectory}/${pairNamePrefix}-current.pem will be created as a soft
|
||||
// link to the currently selected cert/key pair.
|
||||
func NewFileStore(
|
||||
pairNamePrefix string,
|
||||
certDirectory string,
|
||||
keyDirectory string,
|
||||
certFile string,
|
||||
keyFile string) (Store, error) {
|
||||
|
||||
s := fileStore{
|
||||
pairNamePrefix: pairNamePrefix,
|
||||
certDirectory: certDirectory,
|
||||
keyDirectory: keyDirectory,
|
||||
certFile: certFile,
|
||||
keyFile: keyFile,
|
||||
}
|
||||
if err := s.recover(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s, nil
|
||||
}
|
||||
|
||||
// recover checks if there is a certificate rotation that was interrupted while
|
||||
// progress, and if so, attempts to recover to a good state.
|
||||
func (s *fileStore) recover() error {
|
||||
// If the 'current' file doesn't exist, continue on with the recovery process.
|
||||
currentPath := filepath.Join(s.certDirectory, s.filename(currentPair))
|
||||
if exists, err := util.FileExists(currentPath); err != nil {
|
||||
return err
|
||||
} else if exists {
|
||||
return nil
|
||||
}
|
||||
|
||||
// If the 'updated' file exists, and it is a symbolic link, continue on
|
||||
// with the recovery process.
|
||||
updatedPath := filepath.Join(s.certDirectory, s.filename(updatedPair))
|
||||
if fi, err := os.Lstat(updatedPath); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
} else if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
|
||||
return fmt.Errorf("expected %q to be a symlink but it is a file.", updatedPath)
|
||||
}
|
||||
|
||||
// Move the 'updated' symlink to 'current'.
|
||||
if err := os.Rename(updatedPath, currentPath); err != nil {
|
||||
return fmt.Errorf("unable to rename %q to %q: %v", updatedPath, currentPath, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *fileStore) Current() (*tls.Certificate, error) {
|
||||
pairFile := filepath.Join(s.certDirectory, s.filename(currentPair))
|
||||
if pairFileExists, err := util.FileExists(pairFile); err != nil {
|
||||
return nil, err
|
||||
} else if pairFileExists {
|
||||
glog.Infof("Loading cert/key pair from %q.", pairFile)
|
||||
return loadFile(pairFile)
|
||||
}
|
||||
|
||||
certFileExists, err := util.FileExists(s.certFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keyFileExists, err := util.FileExists(s.keyFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if certFileExists && keyFileExists {
|
||||
glog.Infof("Loading cert/key pair from (%q, %q).", s.certFile, s.keyFile)
|
||||
return loadX509KeyPair(s.certFile, s.keyFile)
|
||||
}
|
||||
|
||||
c := filepath.Join(s.certDirectory, s.pairNamePrefix+certExtension)
|
||||
k := filepath.Join(s.keyDirectory, s.pairNamePrefix+keyExtension)
|
||||
certFileExists, err = util.FileExists(c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keyFileExists, err = util.FileExists(k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if certFileExists && keyFileExists {
|
||||
glog.Infof("Loading cert/key pair from (%q, %q).", c, k)
|
||||
return loadX509KeyPair(c, k)
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("no cert/key files read at %q, (%q, %q) or (%q, %q)",
|
||||
pairFile,
|
||||
s.certFile,
|
||||
s.keyFile,
|
||||
s.certDirectory,
|
||||
s.keyDirectory)
|
||||
}
|
||||
|
||||
func loadFile(pairFile string) (*tls.Certificate, error) {
|
||||
certBlock, keyBlock, err := loadCertKeyBlocks(pairFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cert, err := tls.X509KeyPair(pem.EncodeToMemory(certBlock), pem.EncodeToMemory(keyBlock))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not convert data from %q into cert/key pair: %v", pairFile, err)
|
||||
}
|
||||
certs, err := x509.ParseCertificates(cert.Certificate[0])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to parse certificate data: %v", err)
|
||||
}
|
||||
cert.Leaf = certs[0]
|
||||
return &cert, nil
|
||||
}
|
||||
|
||||
func loadCertKeyBlocks(pairFile string) (cert *pem.Block, key *pem.Block, err error) {
|
||||
data, err := ioutil.ReadFile(pairFile)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("could not load cert/key pair from %q: %v", pairFile, err)
|
||||
}
|
||||
certBlock, rest := pem.Decode(data)
|
||||
if certBlock == nil {
|
||||
return nil, nil, fmt.Errorf("could not decode the first block from %q from expected PEM format", pairFile)
|
||||
}
|
||||
keyBlock, rest := pem.Decode(rest)
|
||||
if keyBlock == nil {
|
||||
return nil, nil, fmt.Errorf("could not decode the second block from %q from expected PEM format", pairFile)
|
||||
}
|
||||
return certBlock, keyBlock, nil
|
||||
}
|
||||
|
||||
func (s *fileStore) Update(certData, keyData []byte) (*tls.Certificate, error) {
|
||||
ts := time.Now().Format("2006-01-02-15-04-05")
|
||||
pemFilename := s.filename(ts)
|
||||
|
||||
certPath := filepath.Join(s.certDirectory, pemFilename)
|
||||
|
||||
f, err := os.OpenFile(certPath, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0600)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not open %q: %v", certPath, err)
|
||||
}
|
||||
defer f.Close()
|
||||
certBlock, _ := pem.Decode(certData)
|
||||
if certBlock == nil {
|
||||
return nil, fmt.Errorf("invalid certificate data")
|
||||
}
|
||||
pem.Encode(f, certBlock)
|
||||
keyBlock, _ := pem.Decode(keyData)
|
||||
if keyBlock == nil {
|
||||
return nil, fmt.Errorf("invalid key data")
|
||||
}
|
||||
pem.Encode(f, keyBlock)
|
||||
|
||||
cert, err := loadFile(certPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := s.updateSymlink(certPath); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return cert, nil
|
||||
}
|
||||
|
||||
// updateSymLink updates the current symlink to point to the file that is
|
||||
// passed it. It will fail if there is a non-symlink file exists where the
|
||||
// symlink is expected to be.
|
||||
func (s *fileStore) updateSymlink(filename string) error {
|
||||
// If the 'current' file either doesn't exist, or is already a symlink,
|
||||
// proceed. Otherwise, this is an unrecoverable error.
|
||||
currentPath := filepath.Join(s.certDirectory, s.filename(currentPair))
|
||||
currentPathExists := false
|
||||
if fi, err := os.Lstat(currentPath); err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
} else if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
|
||||
return fmt.Errorf("expected %q to be a symlink but it is a file.", currentPath)
|
||||
} else {
|
||||
currentPathExists = true
|
||||
}
|
||||
|
||||
// If the 'updated' file doesn't exist, proceed. If it exists but it is a
|
||||
// symlink, delete it. Otherwise, this is an unrecoverable error.
|
||||
updatedPath := filepath.Join(s.certDirectory, s.filename(updatedPair))
|
||||
if fi, err := os.Lstat(updatedPath); err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
} else if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
|
||||
return fmt.Errorf("expected %q to be a symlink but it is a file.", updatedPath)
|
||||
} else {
|
||||
if err := os.Remove(updatedPath); err != nil {
|
||||
return fmt.Errorf("unable to remove %q: %v", updatedPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Check that the new cert/key pair file exists to avoid rotating to an
|
||||
// invalid cert/key.
|
||||
if filenameExists, err := util.FileExists(filename); err != nil {
|
||||
return err
|
||||
} else if !filenameExists {
|
||||
return fmt.Errorf("file %q does not exist so it can not be used as the currently selected cert/key", filename)
|
||||
}
|
||||
|
||||
// Create the 'updated' symlink pointing to the requested file name.
|
||||
if err := os.Symlink(filename, updatedPath); err != nil {
|
||||
return fmt.Errorf("unable to create a symlink from %q to %q: %v", updatedPath, filename, err)
|
||||
}
|
||||
|
||||
// Replace the 'current' symlink.
|
||||
if currentPathExists {
|
||||
if err := os.Remove(currentPath); err != nil {
|
||||
return fmt.Errorf("unable to remove %q: %v", currentPath, err)
|
||||
}
|
||||
}
|
||||
if err := os.Rename(updatedPath, currentPath); err != nil {
|
||||
return fmt.Errorf("unable to rename %q to %q: %v", updatedPath, currentPath, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *fileStore) filename(qualifier string) string {
|
||||
return s.pairNamePrefix + "-" + qualifier + pemExtension
|
||||
}
|
||||
|
||||
// withoutExt returns the given filename after removing the extension. The
|
||||
// extension to remove will be the result of filepath.Ext().
|
||||
func withoutExt(filename string) string {
|
||||
return strings.TrimSuffix(filename, filepath.Ext(filename))
|
||||
}
|
||||
|
||||
func loadX509KeyPair(certFile, keyFile string) (*tls.Certificate, error) {
|
||||
cert, err := tls.LoadX509KeyPair(certFile, keyFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
certs, err := x509.ParseCertificates(cert.Certificate[0])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to parse certificate data: %v", err)
|
||||
}
|
||||
cert.Leaf = certs[0]
|
||||
return &cert, nil
|
||||
}
|
471
pkg/kubelet/certificate/certificate_store_test.go
Normal file
471
pkg/kubelet/certificate/certificate_store_test.go
Normal file
@ -0,0 +1,471 @@
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package certificate
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"k8s.io/client-go/util/cert"
|
||||
)
|
||||
|
||||
func TestUpdateSymlinkExistingFileError(t *testing.T) {
|
||||
dir, err := ioutil.TempDir("", "k8s-test-update-symlink")
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to create the test directory %q: %v", dir, err)
|
||||
}
|
||||
defer func() {
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
t.Errorf("Unable to clean up test directory %q: %v", dir, err)
|
||||
}
|
||||
}()
|
||||
pairFile := filepath.Join(dir, "kubelet-current.pem")
|
||||
if err := ioutil.WriteFile(pairFile, nil, 0600); err != nil {
|
||||
t.Fatalf("Unable to create the file %q: %v", pairFile, err)
|
||||
}
|
||||
|
||||
s := fileStore{
|
||||
certDirectory: dir,
|
||||
pairNamePrefix: "kubelet",
|
||||
}
|
||||
if err := s.updateSymlink(pairFile); err == nil {
|
||||
t.Errorf("Got no error, wanted to fail updating the symlink because there is a file there.")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateSymlinkNewFileNotExist(t *testing.T) {
|
||||
dir, err := ioutil.TempDir("", "k8s-test-update-symlink")
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to create the test directory %q: %v", dir, err)
|
||||
}
|
||||
defer func() {
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
t.Errorf("Unable to clean up test directory %q: %v", dir, err)
|
||||
}
|
||||
}()
|
||||
oldPairFile := filepath.Join(dir, "kubelet-oldpair.pem")
|
||||
if err := ioutil.WriteFile(oldPairFile, nil, 0600); err != nil {
|
||||
t.Fatalf("Unable to create the file %q: %v", oldPairFile, err)
|
||||
}
|
||||
|
||||
s := fileStore{
|
||||
certDirectory: dir,
|
||||
pairNamePrefix: "kubelet",
|
||||
}
|
||||
if err := s.updateSymlink(oldPairFile); err != nil {
|
||||
t.Errorf("Got %v, wanted successful update of the symlink to point to %q", err, oldPairFile)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(oldPairFile); err != nil {
|
||||
t.Errorf("Got %v, wanted file %q to be there.", oldPairFile, err)
|
||||
}
|
||||
|
||||
currentPairFile := filepath.Join(dir, "kubelet-current.pem")
|
||||
if fi, err := os.Lstat(currentPairFile); err != nil {
|
||||
t.Errorf("Got %v, wanted file %q to be there", currentPairFile, err)
|
||||
} else if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
|
||||
t.Errorf("Got %q not a symlink.", currentPairFile)
|
||||
}
|
||||
|
||||
newPairFile := filepath.Join(dir, "kubelet-newpair.pem")
|
||||
if err := s.updateSymlink(newPairFile); err == nil {
|
||||
t.Errorf("Got no error, wanted to fail updating the symlink the file %q does not exist.", newPairFile)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateSymlinkNoSymlink(t *testing.T) {
|
||||
dir, err := ioutil.TempDir("", "k8s-test-update-symlink")
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to create the test directory %q: %v", dir, err)
|
||||
}
|
||||
defer func() {
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
t.Errorf("Unable to clean up test directory %q: %v", dir, err)
|
||||
}
|
||||
}()
|
||||
pairFile := filepath.Join(dir, "kubelet-newfile.pem")
|
||||
if err := ioutil.WriteFile(pairFile, nil, 0600); err != nil {
|
||||
t.Fatalf("Unable to create the file %q: %v", pairFile, err)
|
||||
}
|
||||
|
||||
s := fileStore{
|
||||
certDirectory: dir,
|
||||
pairNamePrefix: "kubelet",
|
||||
}
|
||||
if err := s.updateSymlink(pairFile); err != nil {
|
||||
t.Errorf("Got error %v, wanted a new symlink to be created", err)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(pairFile); err != nil {
|
||||
t.Errorf("Got error %v, wanted file %q to be there", pairFile, err)
|
||||
}
|
||||
currentPairFile := filepath.Join(dir, "kubelet-current.pem")
|
||||
if fi, err := os.Lstat(currentPairFile); err != nil {
|
||||
t.Errorf("Got %v, wanted %q to be there", currentPairFile, err)
|
||||
} else if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
|
||||
t.Errorf("%q not a symlink, wanted a symlink.", currentPairFile)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateSymlinkReplaceExistingSymlink(t *testing.T) {
|
||||
prefix := "kubelet"
|
||||
dir, err := ioutil.TempDir("", "k8s-test-update-symlink")
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to create the test directory %q: %v", dir, err)
|
||||
}
|
||||
defer func() {
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
t.Errorf("Unable to clean up test directory %q: %v", dir, err)
|
||||
}
|
||||
}()
|
||||
oldPairFile := filepath.Join(dir, prefix+"-oldfile.pem")
|
||||
if err := ioutil.WriteFile(oldPairFile, nil, 0600); err != nil {
|
||||
t.Fatalf("Unable to create the file %q: %v", oldPairFile, err)
|
||||
}
|
||||
newPairFile := filepath.Join(dir, prefix+"-newfile.pem")
|
||||
if err := ioutil.WriteFile(newPairFile, nil, 0600); err != nil {
|
||||
t.Fatalf("Unable to create the file %q: %v", newPairFile, err)
|
||||
}
|
||||
currentPairFile := filepath.Join(dir, prefix+"-current.pem")
|
||||
if err := os.Symlink(oldPairFile, currentPairFile); err != nil {
|
||||
t.Fatalf("unable to create a symlink from %q to %q: %v", currentPairFile, oldPairFile, err)
|
||||
}
|
||||
if resolved, err := os.Readlink(currentPairFile); err != nil {
|
||||
t.Fatalf("Got %v when attempting to resolve symlink %q", err, currentPairFile)
|
||||
} else if resolved != oldPairFile {
|
||||
t.Fatalf("Got %q as resolution of symlink %q, wanted %q", resolved, currentPairFile, oldPairFile)
|
||||
}
|
||||
|
||||
s := fileStore{
|
||||
certDirectory: dir,
|
||||
pairNamePrefix: prefix,
|
||||
}
|
||||
if err := s.updateSymlink(newPairFile); err != nil {
|
||||
t.Errorf("Got error %v, wanted a new symlink to be created", err)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(oldPairFile); err != nil {
|
||||
t.Errorf("Got error %v, wanted file %q to be there", oldPairFile, err)
|
||||
}
|
||||
if _, err := os.Stat(newPairFile); err != nil {
|
||||
t.Errorf("Got error %v, wanted file %q to be there", newPairFile, err)
|
||||
}
|
||||
if fi, err := os.Lstat(currentPairFile); err != nil {
|
||||
t.Errorf("Got %v, wanted %q to be there", currentPairFile, err)
|
||||
} else if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
|
||||
t.Errorf("%q not a symlink, wanted a symlink.", currentPairFile)
|
||||
}
|
||||
if resolved, err := os.Readlink(currentPairFile); err != nil {
|
||||
t.Fatalf("Got %v when attempting to resolve symlink %q", err, currentPairFile)
|
||||
} else if resolved != newPairFile {
|
||||
t.Fatalf("Got %q as resolution of symlink %q, wanted %q", resolved, currentPairFile, newPairFile)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadCertKeyBlocksNoFile(t *testing.T) {
|
||||
dir, err := ioutil.TempDir("", "k8s-test-load-cert-key-blocks")
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to created the test directory %q: %v", dir, err)
|
||||
}
|
||||
defer func() {
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
t.Errorf("Unable to clean up test directory %q: %v", dir, err)
|
||||
}
|
||||
}()
|
||||
|
||||
pairFile := filepath.Join(dir, "kubelet-pair.pem")
|
||||
|
||||
if _, _, err := loadCertKeyBlocks(pairFile); err == nil {
|
||||
t.Errorf("Got no error, but expected %q not found.", pairFile)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadCertKeyBlocksEmptyFile(t *testing.T) {
|
||||
dir, err := ioutil.TempDir("", "k8s-test-load-cert-key-blocks")
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to created the test directory %q: %v", dir, err)
|
||||
}
|
||||
defer func() {
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
t.Errorf("Unable to clean up test directory %q: %v", dir, err)
|
||||
}
|
||||
}()
|
||||
|
||||
pairFile := filepath.Join(dir, "kubelet-pair.pem")
|
||||
if err := ioutil.WriteFile(pairFile, nil, 0600); err != nil {
|
||||
t.Fatalf("Unable to create the file %q: %v", pairFile, err)
|
||||
}
|
||||
|
||||
if _, _, err := loadCertKeyBlocks(pairFile); err == nil {
|
||||
t.Errorf("Got no error, but expected %q not found.", pairFile)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadCertKeyBlocksPartialFile(t *testing.T) {
|
||||
dir, err := ioutil.TempDir("", "k8s-test-load-cert-key-blocks")
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to created the test directory %q: %v", dir, err)
|
||||
}
|
||||
defer func() {
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
t.Errorf("Unable to clean up test directory %q: %v", dir, err)
|
||||
}
|
||||
}()
|
||||
|
||||
pairFile := filepath.Join(dir, "kubelet-pair.pem")
|
||||
if err := ioutil.WriteFile(pairFile, []byte(certificateData), 0600); err != nil {
|
||||
t.Fatalf("Unable to create the file %q: %v", pairFile, err)
|
||||
}
|
||||
|
||||
if _, _, err := loadCertKeyBlocks(pairFile); err == nil {
|
||||
t.Errorf("Got no error, but expected %q invalid.", pairFile)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadCertKeyBlocks(t *testing.T) {
|
||||
dir, err := ioutil.TempDir("", "k8s-test-load-cert-key-blocks")
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to created the test directory %q: %v", dir, err)
|
||||
}
|
||||
defer func() {
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
t.Errorf("Unable to clean up test directory %q: %v", dir, err)
|
||||
}
|
||||
}()
|
||||
|
||||
pairFile := filepath.Join(dir, "kubelet-pair.pem")
|
||||
if err := ioutil.WriteFile(pairFile, []byte(certificateData+"\n"+privateKeyData), 0600); err != nil {
|
||||
t.Fatalf("Unable to create the file %q: %v", pairFile, err)
|
||||
}
|
||||
|
||||
certBlock, keyBlock, err := loadCertKeyBlocks(pairFile)
|
||||
if err != nil {
|
||||
t.Errorf("Got %v, but expected no error.", pairFile)
|
||||
}
|
||||
if certBlock.Type != cert.CertificateBlockType {
|
||||
t.Errorf("Got %q loaded from the pair file, expected a %q.", certBlock.Type, cert.CertificateBlockType)
|
||||
}
|
||||
if keyBlock.Type != cert.RSAPrivateKeyBlockType {
|
||||
t.Errorf("Got %q loaded from the pair file, expected a %q.", keyBlock.Type, cert.RSAPrivateKeyBlockType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFile(t *testing.T) {
|
||||
dir, err := ioutil.TempDir("", "k8s-test-load-cert-key-blocks")
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to created the test directory %q: %v", dir, err)
|
||||
}
|
||||
defer func() {
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
t.Errorf("Unable to clean up test directory %q: %v", dir, err)
|
||||
}
|
||||
}()
|
||||
|
||||
pairFile := filepath.Join(dir, "kubelet-pair.pem")
|
||||
if err := ioutil.WriteFile(pairFile, []byte(certificateData+"\n"+privateKeyData), 0600); err != nil {
|
||||
t.Fatalf("Unable to create the file %q: %v", pairFile, err)
|
||||
}
|
||||
|
||||
cert, err := loadFile(pairFile)
|
||||
if err != nil {
|
||||
t.Fatalf("Could not load certificate from disk: %v", err)
|
||||
}
|
||||
if cert == nil {
|
||||
t.Fatalf("There was no error, but no certificate data was returned.")
|
||||
}
|
||||
if cert.Leaf == nil {
|
||||
t.Fatalf("Got an empty leaf, expected private data.")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateNoRotation(t *testing.T) {
|
||||
prefix := "kubelet-server"
|
||||
dir, err := ioutil.TempDir("", "k8s-test-certstore-current")
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to created the test directory %q: %v", dir, err)
|
||||
}
|
||||
defer func() {
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
t.Errorf("Unable to clean up test directory %q: %v", dir, err)
|
||||
}
|
||||
}()
|
||||
keyFile := filepath.Join(dir, "kubelet.key")
|
||||
if err := ioutil.WriteFile(keyFile, []byte(privateKeyData), 0600); err != nil {
|
||||
t.Fatalf("Unable to create the file %q: %v", keyFile, err)
|
||||
}
|
||||
certFile := filepath.Join(dir, "kubelet.crt")
|
||||
if err := ioutil.WriteFile(certFile, []byte(certificateData), 0600); err != nil {
|
||||
t.Fatalf("Unable to create the file %q: %v", certFile, err)
|
||||
}
|
||||
|
||||
s, err := NewFileStore(prefix, dir, dir, certFile, keyFile)
|
||||
if err != nil {
|
||||
t.Fatalf("Got %v while creating a new store.", err)
|
||||
}
|
||||
|
||||
cert, err := s.Update([]byte(certificateData), []byte(privateKeyData))
|
||||
if err != nil {
|
||||
t.Errorf("Got %v while updating certificate store.", err)
|
||||
}
|
||||
if cert == nil {
|
||||
t.Errorf("Got nil certificate, expected something real.")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateRotation(t *testing.T) {
|
||||
prefix := "kubelet-server"
|
||||
dir, err := ioutil.TempDir("", "k8s-test-certstore-current")
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to created the test directory %q: %v", dir, err)
|
||||
}
|
||||
defer func() {
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
t.Errorf("Unable to clean up test directory %q: %v", dir, err)
|
||||
}
|
||||
}()
|
||||
keyFile := filepath.Join(dir, "kubelet.key")
|
||||
if err := ioutil.WriteFile(keyFile, []byte(privateKeyData), 0600); err != nil {
|
||||
t.Fatalf("Unable to create the file %q: %v", keyFile, err)
|
||||
}
|
||||
certFile := filepath.Join(dir, "kubelet.crt")
|
||||
if err := ioutil.WriteFile(certFile, []byte(certificateData), 0600); err != nil {
|
||||
t.Fatalf("Unable to create the file %q: %v", certFile, err)
|
||||
}
|
||||
|
||||
s, err := NewFileStore(prefix, dir, dir, certFile, keyFile)
|
||||
if err != nil {
|
||||
t.Fatalf("Got %v while creating a new store.", err)
|
||||
}
|
||||
|
||||
cert, err := s.Update([]byte(certificateData), []byte(privateKeyData))
|
||||
if err != nil {
|
||||
t.Fatalf("Got %v while updating certificate store.", err)
|
||||
}
|
||||
if cert == nil {
|
||||
t.Fatalf("Got nil certificate, expected something real.")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateWithBadCertKeyData(t *testing.T) {
|
||||
prefix := "kubelet-server"
|
||||
dir, err := ioutil.TempDir("", "k8s-test-certstore-current")
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to created the test directory %q: %v", dir, err)
|
||||
}
|
||||
defer func() {
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
t.Errorf("Unable to clean up test directory %q: %v", dir, err)
|
||||
}
|
||||
}()
|
||||
keyFile := filepath.Join(dir, "kubelet.key")
|
||||
if err := ioutil.WriteFile(keyFile, []byte(privateKeyData), 0600); err != nil {
|
||||
t.Fatalf("Unable to create the file %q: %v", keyFile, err)
|
||||
}
|
||||
certFile := filepath.Join(dir, "kubelet.crt")
|
||||
if err := ioutil.WriteFile(certFile, []byte(certificateData), 0600); err != nil {
|
||||
t.Fatalf("Unable to create the file %q: %v", certFile, err)
|
||||
}
|
||||
|
||||
s, err := NewFileStore(prefix, dir, dir, certFile, keyFile)
|
||||
if err != nil {
|
||||
t.Fatalf("Got %v while creating a new store.", err)
|
||||
}
|
||||
|
||||
cert, err := s.Update([]byte{0, 0}, []byte(privateKeyData))
|
||||
if err == nil {
|
||||
t.Fatalf("Got no error while updating certificate store with invalid data.")
|
||||
}
|
||||
if cert != nil {
|
||||
t.Fatalf("Got %v certificate returned from the update, expected nil.", cert)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCurrentPairFile(t *testing.T) {
|
||||
prefix := "kubelet-server"
|
||||
dir, err := ioutil.TempDir("", "k8s-test-certstore-current")
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to created the test directory %q: %v", dir, err)
|
||||
}
|
||||
defer func() {
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
t.Errorf("Unable to clean up test directory %q: %v", dir, err)
|
||||
}
|
||||
}()
|
||||
pairFile := filepath.Join(dir, prefix+"-pair.pem")
|
||||
if err := ioutil.WriteFile(pairFile, []byte(certificateData+"\n"+privateKeyData), 0600); err != nil {
|
||||
t.Fatalf("Unable to create the file %q: %v", pairFile, err)
|
||||
}
|
||||
currentFile := filepath.Join(dir, prefix+"-current.pem")
|
||||
if err := os.Symlink(pairFile, currentFile); err != nil {
|
||||
t.Fatalf("unable to create a symlink from %q to %q: %v", currentFile, pairFile, err)
|
||||
}
|
||||
|
||||
store, err := NewFileStore("kubelet-server", dir, dir, "", "")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to initialize certificate store: %v", err)
|
||||
}
|
||||
|
||||
cert, err := store.Current()
|
||||
if err != nil {
|
||||
t.Fatalf("Could not load certificate from disk: %v", err)
|
||||
}
|
||||
if cert == nil {
|
||||
t.Fatalf("There was no error, but no certificate data was returned.")
|
||||
}
|
||||
if cert.Leaf == nil {
|
||||
t.Fatalf("Got an empty leaf, expected private data.")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCurrentCertKeyFiles(t *testing.T) {
|
||||
prefix := "kubelet-server"
|
||||
dir, err := ioutil.TempDir("", "k8s-test-certstore-current")
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to created the test directory %q: %v", dir, err)
|
||||
}
|
||||
defer func() {
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
t.Errorf("Unable to clean up test directory %q: %v", dir, err)
|
||||
}
|
||||
}()
|
||||
certFile := filepath.Join(dir, "kubelet.crt")
|
||||
if err := ioutil.WriteFile(certFile, []byte(certificateData), 0600); err != nil {
|
||||
t.Fatalf("Unable to create the file %q: %v", certFile, err)
|
||||
}
|
||||
keyFile := filepath.Join(dir, "kubelet.key")
|
||||
if err := ioutil.WriteFile(keyFile, []byte(privateKeyData), 0600); err != nil {
|
||||
t.Fatalf("Unable to create the file %q: %v", keyFile, err)
|
||||
}
|
||||
|
||||
store, err := NewFileStore(prefix, dir, dir, certFile, keyFile)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to initialize certificate store: %v", err)
|
||||
}
|
||||
|
||||
cert, err := store.Current()
|
||||
if err != nil {
|
||||
t.Fatalf("Could not load certificate from disk: %v", err)
|
||||
}
|
||||
if cert == nil {
|
||||
t.Fatalf("There was no error, but no certificate data was returned.")
|
||||
}
|
||||
if cert.Leaf == nil {
|
||||
t.Fatalf("Got an empty leaf, expected private data.")
|
||||
}
|
||||
}
|
@ -128,7 +128,7 @@ func MakeEllipticPrivateKeyPEM() ([]byte, error) {
|
||||
}
|
||||
|
||||
privateKeyPemBlock := &pem.Block{
|
||||
Type: "EC PRIVATE KEY",
|
||||
Type: ECPrivateKeyBlockType,
|
||||
Bytes: derBytes,
|
||||
}
|
||||
return pem.EncodeToMemory(privateKeyPemBlock), nil
|
||||
@ -173,13 +173,13 @@ func GenerateSelfSignedCertKey(host string, alternateIPs []net.IP, alternateDNS
|
||||
|
||||
// Generate cert
|
||||
certBuffer := bytes.Buffer{}
|
||||
if err := pem.Encode(&certBuffer, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
|
||||
if err := pem.Encode(&certBuffer, &pem.Block{Type: CertificateBlockType, Bytes: derBytes}); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// Generate key
|
||||
keyBuffer := bytes.Buffer{}
|
||||
if err := pem.Encode(&keyBuffer, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil {
|
||||
if err := pem.Encode(&keyBuffer, &pem.Block{Type: RSAPrivateKeyBlockType, Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
|
@ -24,6 +24,21 @@ import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
const (
|
||||
// ECPrivateKeyBlockType is a possible value for pem.Block.Type.
|
||||
ECPrivateKeyBlockType = "EC PRIVATE KEY"
|
||||
// RSAPrivateKeyBlockType is a possible value for pem.Block.Type.
|
||||
RSAPrivateKeyBlockType = "RSA PRIVATE KEY"
|
||||
// CertificateBlockType is a possible value for pem.Block.Type.
|
||||
CertificateBlockType = "CERTIFICATE"
|
||||
// CertificateRequestBlockType is a possible value for pem.Block.Type.
|
||||
CertificateRequestBlockType = "CERTIFICATE REQUEST"
|
||||
// PrivateKeyBlockType is a possible value for pem.Block.Type.
|
||||
PrivateKeyBlockType = "PRIVATE KEY"
|
||||
// PublicKeyBlockType is a possible value for pem.Block.Type.
|
||||
PublicKeyBlockType = "PUBLIC KEY"
|
||||
)
|
||||
|
||||
// EncodePublicKeyPEM returns PEM-endcode public data
|
||||
func EncodePublicKeyPEM(key *rsa.PublicKey) ([]byte, error) {
|
||||
der, err := x509.MarshalPKIXPublicKey(key)
|
||||
@ -31,7 +46,7 @@ func EncodePublicKeyPEM(key *rsa.PublicKey) ([]byte, error) {
|
||||
return []byte{}, err
|
||||
}
|
||||
block := pem.Block{
|
||||
Type: "PUBLIC KEY",
|
||||
Type: PublicKeyBlockType,
|
||||
Bytes: der,
|
||||
}
|
||||
return pem.EncodeToMemory(&block), nil
|
||||
@ -40,7 +55,7 @@ func EncodePublicKeyPEM(key *rsa.PublicKey) ([]byte, error) {
|
||||
// EncodePrivateKeyPEM returns PEM-encoded private key data
|
||||
func EncodePrivateKeyPEM(key *rsa.PrivateKey) []byte {
|
||||
block := pem.Block{
|
||||
Type: "RSA PRIVATE KEY",
|
||||
Type: RSAPrivateKeyBlockType,
|
||||
Bytes: x509.MarshalPKCS1PrivateKey(key),
|
||||
}
|
||||
return pem.EncodeToMemory(&block)
|
||||
@ -49,7 +64,7 @@ func EncodePrivateKeyPEM(key *rsa.PrivateKey) []byte {
|
||||
// EncodeCertPEM returns PEM-endcoded certificate data
|
||||
func EncodeCertPEM(cert *x509.Certificate) []byte {
|
||||
block := pem.Block{
|
||||
Type: "CERTIFICATE",
|
||||
Type: CertificateBlockType,
|
||||
Bytes: cert.Raw,
|
||||
}
|
||||
return pem.EncodeToMemory(&block)
|
||||
@ -66,17 +81,17 @@ func ParsePrivateKeyPEM(keyData []byte) (interface{}, error) {
|
||||
}
|
||||
|
||||
switch privateKeyPemBlock.Type {
|
||||
case "EC PRIVATE KEY":
|
||||
case ECPrivateKeyBlockType:
|
||||
// ECDSA Private Key in ASN.1 format
|
||||
if key, err := x509.ParseECPrivateKey(privateKeyPemBlock.Bytes); err == nil {
|
||||
return key, nil
|
||||
}
|
||||
case "RSA PRIVATE KEY":
|
||||
case RSAPrivateKeyBlockType:
|
||||
// RSA Private Key in PKCS#1 format
|
||||
if key, err := x509.ParsePKCS1PrivateKey(privateKeyPemBlock.Bytes); err == nil {
|
||||
return key, nil
|
||||
}
|
||||
case "PRIVATE KEY":
|
||||
case PrivateKeyBlockType:
|
||||
// RSA or ECDSA Private Key in unencrypted PKCS#8 format
|
||||
if key, err := x509.ParsePKCS8PrivateKey(privateKeyPemBlock.Bytes); err == nil {
|
||||
return key, nil
|
||||
@ -103,7 +118,7 @@ func ParseCertsPEM(pemCerts []byte) ([]*x509.Certificate, error) {
|
||||
break
|
||||
}
|
||||
// Only use PEM "CERTIFICATE" blocks without extra headers
|
||||
if block.Type != "CERTIFICATE" || len(block.Headers) != 0 {
|
||||
if block.Type != CertificateBlockType || len(block.Headers) != 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user