Update liveness probes to exec etcdctl /w mTLS for kubeadm etcd static pods

cmd/kubeadm/app/phases/etcd/local.go

@@ -65,7 +65,10 @@ func GetEtcdPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.Pod {
 			staticpodutil.NewVolumeMount(etcdVolumeName, cfg.Etcd.DataDir, false),
 			staticpodutil.NewVolumeMount(certsVolumeName, cfg.CertificatesDir+"/etcd", false),
 		},
-		LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.Etcd, 2379, "/health", v1.URISchemeHTTP),
+		LivenessProbe: staticpodutil.EtcdProbe(
+			cfg, kubeadmconstants.Etcd, 2379, cfg.CertificatesDir,
+			kubeadmconstants.EtcdCACertName, kubeadmconstants.EtcdHealthcheckClientCertName, kubeadmconstants.EtcdHealthcheckClientKeyName,
+		),
 	}, etcdMounts)
 }
 

cmd/kubeadm/app/util/staticpod/utils.go

@@ -97,6 +97,24 @@ func ComponentProbe(cfg *kubeadmapi.MasterConfiguration, componentName string, p
 	}
 }
 
+// EtcdProbe is a helper function for building a shell-based, etcdctl v1.Probe object to healthcheck etcd
+func EtcdProbe(cfg *kubeadmapi.MasterConfiguration, componentName string, port int, certsDir string, CACertName string, CertName string, KeyName string) *v1.Probe {
+	tlsFlags := fmt.Sprintf("--cacert=%[1]s/%[2]s --cert=%[1]s/%[3]s --key=%[1]s/%[4]s", certsDir, CACertName, CertName, KeyName)
+	// etcd pod is alive if a linearizable get succeeds.
+	cmd := fmt.Sprintf("ETCDCTL_API=3 etcdctl --endpoints=%s:%d %s get foo", GetProbeAddress(cfg, componentName), port, tlsFlags)
+
+	return &v1.Probe{
+		Handler: v1.Handler{
+			Exec: &v1.ExecAction{
+				Command: []string{"/bin/sh", "-ec", cmd},
+			},
+		},
+		InitialDelaySeconds: 15,
+		TimeoutSeconds:      15,
+		FailureThreshold:    8,
+	}
+}
+
 // NewVolume creates a v1.Volume with a hostPath mount to the specified location
 func NewVolume(name, path string, pathType *v1.HostPathType) v1.Volume {
 	return v1.Volume{

cmd/kubeadm/app/util/staticpod/utils_test.go

@@ -161,48 +161,6 @@ func TestComponentProbe(t *testing.T) {
 			scheme:    v1.URISchemeHTTP,
 			expected:  "2001:db8::1",
 		},
-		{
-			name: "valid etcd probe using listen-client-urls IPv4 addresses",
-			cfg: &kubeadmapi.MasterConfiguration{
-				Etcd: kubeadmapi.Etcd{
-					ExtraArgs: map[string]string{
-						"listen-client-urls": "http://1.2.3.4:2379,http://4.3.2.1:2379"},
-				},
-			},
-			component: kubeadmconstants.Etcd,
-			port:      1,
-			path:      "foo",
-			scheme:    v1.URISchemeHTTP,
-			expected:  "1.2.3.4",
-		},
-		{
-			name: "valid etcd probe using listen-client-urls IPv6 addresses",
-			cfg: &kubeadmapi.MasterConfiguration{
-				Etcd: kubeadmapi.Etcd{
-					ExtraArgs: map[string]string{
-						"listen-client-urls": "http://[2001:db8::1]:2379,http://[2001:db8::2]:2379"},
-				},
-			},
-			component: kubeadmconstants.Etcd,
-			port:      1,
-			path:      "foo",
-			scheme:    v1.URISchemeHTTP,
-			expected:  "2001:db8::1",
-		},
-		{
-			name: "valid IPv4 etcd probe using hostname for listen-client-urls",
-			cfg: &kubeadmapi.MasterConfiguration{
-				Etcd: kubeadmapi.Etcd{
-					ExtraArgs: map[string]string{
-						"listen-client-urls": "http://localhost:2379"},
-				},
-			},
-			component: kubeadmconstants.Etcd,
-			port:      1,
-			path:      "foo",
-			scheme:    v1.URISchemeHTTP,
-			expected:  "127.0.0.1",
-		},
 	}
 	for _, rt := range tests {
 		actual := ComponentProbe(rt.cfg, rt.component, rt.port, rt.path, rt.scheme)
@@ -229,6 +187,77 @@ func TestComponentProbe(t *testing.T) {
 	}
 }
 
+func TestEtcdProbe(t *testing.T) {
+	var tests = []struct {
+		name      string
+		cfg       *kubeadmapi.MasterConfiguration
+		component string
+		port      int
+		certsDir  string
+		cacert    string
+		cert      string
+		key       string
+		expected  string
+	}{
+		{
+			name: "valid etcd probe using listen-client-urls IPv4 addresses",
+			cfg: &kubeadmapi.MasterConfiguration{
+				Etcd: kubeadmapi.Etcd{
+					ExtraArgs: map[string]string{
+						"listen-client-urls": "http://1.2.3.4:2379,http://4.3.2.1:2379"},
+				},
+			},
+			component: kubeadmconstants.Etcd,
+			port:      1,
+			certsDir:  "secretsA",
+			cacert:    "ca1",
+			cert:      "cert1",
+			key:       "key1",
+			expected:  "ETCDCTL_API=3 etcdctl --endpoints=1.2.3.4:1 --cacert=secretsA/ca1 --cert=secretsA/cert1 --key=secretsA/key1 get foo",
+		},
+		{
+			name: "valid etcd probe using listen-client-urls IPv6 addresses",
+			cfg: &kubeadmapi.MasterConfiguration{
+				Etcd: kubeadmapi.Etcd{
+					ExtraArgs: map[string]string{
+						"listen-client-urls": "http://[2001:db8::1]:2379,http://[2001:db8::2]:2379"},
+				},
+			},
+			component: kubeadmconstants.Etcd,
+			port:      1,
+			certsDir:  "secretsB",
+			cacert:    "ca2",
+			cert:      "cert2",
+			key:       "key2",
+			expected:  "ETCDCTL_API=3 etcdctl --endpoints=2001:db8::1:1 --cacert=secretsB/ca2 --cert=secretsB/cert2 --key=secretsB/key2 get foo",
+		},
+		{
+			name: "valid IPv4 etcd probe using hostname for listen-client-urls",
+			cfg: &kubeadmapi.MasterConfiguration{
+				Etcd: kubeadmapi.Etcd{
+					ExtraArgs: map[string]string{
+						"listen-client-urls": "http://localhost:2379"},
+				},
+			},
+			component: kubeadmconstants.Etcd,
+			port:      1,
+			certsDir:  "secretsC",
+			cacert:    "ca3",
+			cert:      "cert3",
+			key:       "key3",
+			expected:  "ETCDCTL_API=3 etcdctl --endpoints=127.0.0.1:1 --cacert=secretsC/ca3 --cert=secretsC/cert3 --key=secretsC/key3 get foo",
+		},
+	}
+	for _, rt := range tests {
+		actual := EtcdProbe(rt.cfg, rt.component, rt.port, rt.certsDir, rt.cacert, rt.cert, rt.key)
+		if actual.Handler.Exec.Command[2] != rt.expected {
+			t.Errorf("%s test case failed:\n\texpected: %s\n\t  actual: %s",
+				rt.name, rt.expected,
+				actual.Handler.Exec.Command[2])
+		}
+	}
+}
+
 func TestComponentPod(t *testing.T) {
 	var tests = []struct {
 		name     string