From 9bbf3fd5393455b544f6d7d0ed622d551c8e6741 Mon Sep 17 00:00:00 2001 From: leigh schrandt Date: Fri, 2 Mar 2018 16:42:34 -0700 Subject: [PATCH] Update liveness probes to exec etcdctl /w mTLS for kubeadm etcd static pods --- cmd/kubeadm/app/phases/etcd/local.go | 5 +- cmd/kubeadm/app/util/staticpod/utils.go | 18 +++ cmd/kubeadm/app/util/staticpod/utils_test.go | 113 ++++++++++++------- 3 files changed, 93 insertions(+), 43 deletions(-) diff --git a/cmd/kubeadm/app/phases/etcd/local.go b/cmd/kubeadm/app/phases/etcd/local.go index 8dfe2bd42f1..d12682cd720 100644 --- a/cmd/kubeadm/app/phases/etcd/local.go +++ b/cmd/kubeadm/app/phases/etcd/local.go @@ -65,7 +65,10 @@ func GetEtcdPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.Pod { staticpodutil.NewVolumeMount(etcdVolumeName, cfg.Etcd.DataDir, false), staticpodutil.NewVolumeMount(certsVolumeName, cfg.CertificatesDir+"/etcd", false), }, - LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.Etcd, 2379, "/health", v1.URISchemeHTTP), + LivenessProbe: staticpodutil.EtcdProbe( + cfg, kubeadmconstants.Etcd, 2379, cfg.CertificatesDir, + kubeadmconstants.EtcdCACertName, kubeadmconstants.EtcdHealthcheckClientCertName, kubeadmconstants.EtcdHealthcheckClientKeyName, + ), }, etcdMounts) } diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go index 1ea1b79ad19..08291f92709 100644 --- a/cmd/kubeadm/app/util/staticpod/utils.go +++ b/cmd/kubeadm/app/util/staticpod/utils.go @@ -97,6 +97,24 @@ func ComponentProbe(cfg *kubeadmapi.MasterConfiguration, componentName string, p } } +// EtcdProbe is a helper function for building a shell-based, etcdctl v1.Probe object to healthcheck etcd +func EtcdProbe(cfg *kubeadmapi.MasterConfiguration, componentName string, port int, certsDir string, CACertName string, CertName string, KeyName string) *v1.Probe { + tlsFlags := fmt.Sprintf("--cacert=%[1]s/%[2]s --cert=%[1]s/%[3]s --key=%[1]s/%[4]s", certsDir, CACertName, CertName, KeyName) + // etcd pod is alive if a linearizable get succeeds. + cmd := fmt.Sprintf("ETCDCTL_API=3 etcdctl --endpoints=%s:%d %s get foo", GetProbeAddress(cfg, componentName), port, tlsFlags) + + return &v1.Probe{ + Handler: v1.Handler{ + Exec: &v1.ExecAction{ + Command: []string{"/bin/sh", "-ec", cmd}, + }, + }, + InitialDelaySeconds: 15, + TimeoutSeconds: 15, + FailureThreshold: 8, + } +} + // NewVolume creates a v1.Volume with a hostPath mount to the specified location func NewVolume(name, path string, pathType *v1.HostPathType) v1.Volume { return v1.Volume{ diff --git a/cmd/kubeadm/app/util/staticpod/utils_test.go b/cmd/kubeadm/app/util/staticpod/utils_test.go index 10a3f9ce5b4..1cf4da2b071 100644 --- a/cmd/kubeadm/app/util/staticpod/utils_test.go +++ b/cmd/kubeadm/app/util/staticpod/utils_test.go @@ -161,48 +161,6 @@ func TestComponentProbe(t *testing.T) { scheme: v1.URISchemeHTTP, expected: "2001:db8::1", }, - { - name: "valid etcd probe using listen-client-urls IPv4 addresses", - cfg: &kubeadmapi.MasterConfiguration{ - Etcd: kubeadmapi.Etcd{ - ExtraArgs: map[string]string{ - "listen-client-urls": "http://1.2.3.4:2379,http://4.3.2.1:2379"}, - }, - }, - component: kubeadmconstants.Etcd, - port: 1, - path: "foo", - scheme: v1.URISchemeHTTP, - expected: "1.2.3.4", - }, - { - name: "valid etcd probe using listen-client-urls IPv6 addresses", - cfg: &kubeadmapi.MasterConfiguration{ - Etcd: kubeadmapi.Etcd{ - ExtraArgs: map[string]string{ - "listen-client-urls": "http://[2001:db8::1]:2379,http://[2001:db8::2]:2379"}, - }, - }, - component: kubeadmconstants.Etcd, - port: 1, - path: "foo", - scheme: v1.URISchemeHTTP, - expected: "2001:db8::1", - }, - { - name: "valid IPv4 etcd probe using hostname for listen-client-urls", - cfg: &kubeadmapi.MasterConfiguration{ - Etcd: kubeadmapi.Etcd{ - ExtraArgs: map[string]string{ - "listen-client-urls": "http://localhost:2379"}, - }, - }, - component: kubeadmconstants.Etcd, - port: 1, - path: "foo", - scheme: v1.URISchemeHTTP, - expected: "127.0.0.1", - }, } for _, rt := range tests { actual := ComponentProbe(rt.cfg, rt.component, rt.port, rt.path, rt.scheme) @@ -229,6 +187,77 @@ func TestComponentProbe(t *testing.T) { } } +func TestEtcdProbe(t *testing.T) { + var tests = []struct { + name string + cfg *kubeadmapi.MasterConfiguration + component string + port int + certsDir string + cacert string + cert string + key string + expected string + }{ + { + name: "valid etcd probe using listen-client-urls IPv4 addresses", + cfg: &kubeadmapi.MasterConfiguration{ + Etcd: kubeadmapi.Etcd{ + ExtraArgs: map[string]string{ + "listen-client-urls": "http://1.2.3.4:2379,http://4.3.2.1:2379"}, + }, + }, + component: kubeadmconstants.Etcd, + port: 1, + certsDir: "secretsA", + cacert: "ca1", + cert: "cert1", + key: "key1", + expected: "ETCDCTL_API=3 etcdctl --endpoints=1.2.3.4:1 --cacert=secretsA/ca1 --cert=secretsA/cert1 --key=secretsA/key1 get foo", + }, + { + name: "valid etcd probe using listen-client-urls IPv6 addresses", + cfg: &kubeadmapi.MasterConfiguration{ + Etcd: kubeadmapi.Etcd{ + ExtraArgs: map[string]string{ + "listen-client-urls": "http://[2001:db8::1]:2379,http://[2001:db8::2]:2379"}, + }, + }, + component: kubeadmconstants.Etcd, + port: 1, + certsDir: "secretsB", + cacert: "ca2", + cert: "cert2", + key: "key2", + expected: "ETCDCTL_API=3 etcdctl --endpoints=2001:db8::1:1 --cacert=secretsB/ca2 --cert=secretsB/cert2 --key=secretsB/key2 get foo", + }, + { + name: "valid IPv4 etcd probe using hostname for listen-client-urls", + cfg: &kubeadmapi.MasterConfiguration{ + Etcd: kubeadmapi.Etcd{ + ExtraArgs: map[string]string{ + "listen-client-urls": "http://localhost:2379"}, + }, + }, + component: kubeadmconstants.Etcd, + port: 1, + certsDir: "secretsC", + cacert: "ca3", + cert: "cert3", + key: "key3", + expected: "ETCDCTL_API=3 etcdctl --endpoints=127.0.0.1:1 --cacert=secretsC/ca3 --cert=secretsC/cert3 --key=secretsC/key3 get foo", + }, + } + for _, rt := range tests { + actual := EtcdProbe(rt.cfg, rt.component, rt.port, rt.certsDir, rt.cacert, rt.cert, rt.key) + if actual.Handler.Exec.Command[2] != rt.expected { + t.Errorf("%s test case failed:\n\texpected: %s\n\t actual: %s", + rt.name, rt.expected, + actual.Handler.Exec.Command[2]) + } + } +} + func TestComponentPod(t *testing.T) { var tests = []struct { name string