From 329b6b522e1b4b0e76e3d55069dd75b34cf545c4 Mon Sep 17 00:00:00 2001 From: XianglongLuo <1981826068@qq.com> Date: Thu, 12 Dec 2019 14:00:21 +0800 Subject: [PATCH] Fix for deleting unused etcd and kubelet certs --- pki/services.go | 33 +++++++++- pki/services_test.go | 142 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 174 insertions(+), 1 deletion(-) create mode 100644 pki/services_test.go diff --git a/pki/services.go b/pki/services.go index 8542f96e..22de0051 100644 --- a/pki/services.go +++ b/pki/services.go @@ -6,6 +6,7 @@ import ( "fmt" "reflect" "sort" + "strings" "github.com/rancher/rke/hosts" "github.com/rancher/rke/log" @@ -370,7 +371,6 @@ func GenerateEtcdCertificates(ctx context.Context, certs map[string]CertificateP ips = append(ips, ip.String()) } sort.Strings(ips) - for _, host := range etcdHosts { etcdName := GetCrtNameForHost(host, EtcdCertName) if _, ok := certs[etcdName]; ok && certs[etcdName].CertificatePEM != "" && !rotate { @@ -403,6 +403,8 @@ func GenerateEtcdCertificates(ctx context.Context, certs map[string]CertificateP } certs[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey, nil) } + log.Debugf(ctx, "Checking and deleting unused etcd certificates, current etcd nodes are: %v", etcdHosts) + deleteUnusedCerts(ctx, certs, EtcdCertName, etcdHosts) return nil } @@ -516,6 +518,8 @@ func GenerateKubeletCertificate(ctx context.Context, certs map[string]Certificat } certs[kubeletName] = ToCertObject(kubeletName, "", "", kubeletCrt, kubeletKey, nil) } + log.Debugf(ctx, "Checking and deleting unused kubelet certificates, current nodes are : %v", allHosts) + deleteUnusedCerts(ctx, certs, KubeletCertName, allHosts) return nil } @@ -555,6 +559,15 @@ func GenerateRKEServicesCerts(ctx context.Context, certs map[string]CertificateP } if IsKubeletGenerateServingCertificateEnabledinConfig(&rkeConfig) { RKECerts = append(RKECerts, GenerateKubeletCertificate) + } else { + //Clean up kubelet certs when GenerateServingCertificate is disabled + log.Infof(ctx, "[certificates] GenerateServingCertificate is disabled, checking if there are unused kubelet certificates") + for k := range certs { + if strings.HasPrefix(k, KubeletCertName) { + log.Infof(ctx, "[certificates] Deleting unused kubelet certificate: %s", k) + delete(certs, k) + } + } } for _, gen := range RKECerts { if err := gen(ctx, certs, rkeConfig, configPath, configDir, rotate); err != nil { @@ -588,3 +601,21 @@ func GenerateRKEServicesCSRs(ctx context.Context, certs map[string]CertificatePK } return nil } + +func deleteUnusedCerts(ctx context.Context, certs map[string]CertificatePKI, certName string, hosts []*hosts.Host) { + log.Infof(ctx, "[certificates] Checking and deleting unused %s certificates", certName) + unusedCerts := make(map[string]bool) + for k := range certs { + if strings.HasPrefix(k, certName) { + unusedCerts[k] = true + } + } + for _, host := range hosts { + Name := GetCrtNameForHost(host, certName) + delete(unusedCerts, Name) + } + for k := range unusedCerts { + log.Infof(ctx, "[certificates] Deleting unused certificate: %s", k) + delete(certs, k) + } +} diff --git a/pki/services_test.go b/pki/services_test.go new file mode 100644 index 00000000..f1fd18e8 --- /dev/null +++ b/pki/services_test.go @@ -0,0 +1,142 @@ +package pki + +import ( + "context" + "github.com/rancher/rke/hosts" + v3 "github.com/rancher/types/apis/management.cattle.io/v3" + "github.com/stretchr/testify/assert" + "reflect" + "testing" +) + +func TestDeleteUnusedCerts(t *testing.T) { + tests := []struct { + ctx context.Context + name string + certs map[string]CertificatePKI + certName string + hosts []*hosts.Host + expectLeftCerts map[string]CertificatePKI + }{ + { + ctx: context.Background(), + name: "Keep valid etcd certs", + certs: map[string]CertificatePKI{ + "kube-etcd-172-17-0-3": CertificatePKI{}, + "kube-etcd-172-17-0-4": CertificatePKI{}, + "kube-node": CertificatePKI{}, + "kube-kubelet-172-17-0-4": CertificatePKI{}, + "kube-apiserver": CertificatePKI{}, + "kube-proxy": CertificatePKI{}, + }, + certName: EtcdCertName, + hosts: []*hosts.Host{ + {RKEConfigNode: v3.RKEConfigNode{ + Address: "172.17.0.3", + }}, + {RKEConfigNode: v3.RKEConfigNode{ + Address: "172.17.0.4", + }}, + }, + expectLeftCerts: map[string]CertificatePKI{ + "kube-etcd-172-17-0-3": CertificatePKI{}, + "kube-etcd-172-17-0-4": CertificatePKI{}, + "kube-node": CertificatePKI{}, + "kube-kubelet-172-17-0-4": CertificatePKI{}, + "kube-apiserver": CertificatePKI{}, + "kube-proxy": CertificatePKI{}, + }, + }, + { + ctx: context.Background(), + name: "Keep valid kubelet certs", + certs: map[string]CertificatePKI{ + "kube-kubelet-172-17-0-5": CertificatePKI{}, + "kube-kubelet-172-17-0-6": CertificatePKI{}, + "kube-node": CertificatePKI{}, + "kube-apiserver": CertificatePKI{}, + "kube-proxy": CertificatePKI{}, + "kube-etcd-172-17-0-6": CertificatePKI{}, + }, + certName: KubeletCertName, + hosts: []*hosts.Host{ + {RKEConfigNode: v3.RKEConfigNode{ + Address: "172.17.0.5", + }}, + {RKEConfigNode: v3.RKEConfigNode{ + Address: "172.17.0.6", + }}, + }, + expectLeftCerts: map[string]CertificatePKI{ + "kube-kubelet-172-17-0-5": CertificatePKI{}, + "kube-kubelet-172-17-0-6": CertificatePKI{}, + "kube-node": CertificatePKI{}, + "kube-apiserver": CertificatePKI{}, + "kube-proxy": CertificatePKI{}, + "kube-etcd-172-17-0-6": CertificatePKI{}, + }, + }, + { + ctx: context.Background(), + name: "Remove unused etcd certs", + certs: map[string]CertificatePKI{ + "kube-etcd-172-17-0-11": CertificatePKI{}, + "kube-etcd-172-17-0-10": CertificatePKI{}, + "kube-kubelet-172-17-0-11": CertificatePKI{}, + "kube-node": CertificatePKI{}, + "kube-apiserver": CertificatePKI{}, + "kube-proxy": CertificatePKI{}, + }, + certName: EtcdCertName, + hosts: []*hosts.Host{ + {RKEConfigNode: v3.RKEConfigNode{ + Address: "172.17.0.11", + }}, + {RKEConfigNode: v3.RKEConfigNode{ + Address: "172.17.0.12", + }}, + }, + expectLeftCerts: map[string]CertificatePKI{ + "kube-etcd-172-17-0-11": CertificatePKI{}, + "kube-kubelet-172-17-0-11": CertificatePKI{}, + "kube-node": CertificatePKI{}, + "kube-apiserver": CertificatePKI{}, + "kube-proxy": CertificatePKI{}, + }, + }, + { + ctx: context.Background(), + name: "Remove unused kubelet certs", + certs: map[string]CertificatePKI{ + "kube-kubelet-172-17-0-11": CertificatePKI{}, + "kube-kubelet-172-17-0-10": CertificatePKI{}, + "kube-etcd-172-17-0-10": CertificatePKI{}, + "kube-node": CertificatePKI{}, + "kube-apiserver": CertificatePKI{}, + "kube-proxy": CertificatePKI{}, + }, + certName: KubeletCertName, + hosts: []*hosts.Host{ + {RKEConfigNode: v3.RKEConfigNode{ + Address: "172.17.0.11", + }}, + {RKEConfigNode: v3.RKEConfigNode{ + Address: "172.17.0.12", + }}, + }, + expectLeftCerts: map[string]CertificatePKI{ + "kube-kubelet-172-17-0-11": CertificatePKI{}, + "kube-etcd-172-17-0-10": CertificatePKI{}, + "kube-node": CertificatePKI{}, + "kube-apiserver": CertificatePKI{}, + "kube-proxy": CertificatePKI{}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + deleteUnusedCerts(tt.ctx, tt.certs, tt.certName, tt.hosts) + assert.Equal(t, true, reflect.DeepEqual(tt.certs, tt.expectLeftCerts)) + }) + } +}