Merge pull request #64670 from stealthybox/feature/kubeadm_882-etcd-zero-probe

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

kubeadm: When etcd is listening on all interfaces, set the etcd probe to use loopback

**What this PR does / why we need it**:
When constructing the etcd liveness probe, if the user passes an IPv4 or IPv6 address,
we set the `etcdctl` liveness probe to use the respective IPv4 or IPv6 loopback address for `--endpoints`.

The etcd probe is now always formatted with the https:// protocol and square brackets around the IP (required for IPv6 / compatible with IPv4).

`::1` is now also included in the etcd serving cert SAN by default.

/kind bug
/area kubeadm
/area etcd
/priority important-soon

/sig cluster-lifecycle
/assign @fabriziopandini

**Which issue(s) this PR fixes**
Fixes https://github.com/kubernetes/kubeadm/issues/882

**Special notes for your reviewer**:
```bash
root@vagrant:~# /vagrant/bin/882_kubeadm init --config /dev/stdin << EOF |& tail -n5
etcd:
  extraArgs:
    listen-client-urls: https://[::]:2379
EOF
I0603 19:52:15.666594   24743 tlsbootstrap.go:50] [bootstraptoken] configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
I0603 19:52:15.671424   24743 tlsbootstrap.go:72] [bootstraptoken] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
I0603 19:52:15.674607   24743 tlsbootstrap.go:95] [bootstraptoken] configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
I0603 19:52:15.677551   24743 clusterinfo.go:43] [bootstraptoken] creating the "cluster-info" ConfigMap in the "kube-public" namespace
[addons] Applied essential addon: CoreDNS
[addons] Applied essential addon: kube-proxy
root@vagrant:~# cat /etc/kubernetes/manifests/etcd.yaml |grep -C4 listen
spec:
  containers:
  - command:
    - etcd
    - --listen-client-urls=https://[::]:2379
    - --advertise-client-urls=https://127.0.0.1:2379
    - --cert-file=/etc/kubernetes/pki/etcd/server.crt
    - --client-cert-auth=true
    - --data-dir=/var/lib/etcd
root@vagrant:~# cat /etc/kubernetes/manifests/etcd.yaml |grep -C4 etcdctl
      exec:
        command:
        - /bin/sh
        - -ec
        - ETCDCTL_API=3 etcdctl --endpoints=https://[::1]:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt
          --cert=/etc/kubernetes/pki/etcd/healthcheck-client.crt --key=/etc/kubernetes/pki/etcd/healthcheck-client.key
          get foo
      failureThreshold: 8
      initialDelaySeconds: 15
```

**Release note**:
```release-note
kubeadm now configures the etcd liveness probe correctly when etcd is listening on all interfaces
```

cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers.go

@@ -313,7 +313,7 @@ func GetEtcdAltNames(cfg *kubeadmapi.MasterConfiguration) (*certutil.AltNames, e
 	// create AltNames with defaults DNSNames/IPs
 	altNames := &certutil.AltNames{
 		DNSNames: []string{"localhost"},
-		IPs:      []net.IP{net.IPv4(127, 0, 0, 1)},
+		IPs:      []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback},
 	}
 
 	if cfg.Etcd.Local != nil {

cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers_test.go

@@ -539,7 +539,7 @@ func TestGetEtcdAltNames(t *testing.T) {
 		}
 	}
 
-	expectedIPAddresses := []string{"127.0.0.1", proxyIP}
+	expectedIPAddresses := []string{"127.0.0.1", net.IPv6loopback.String(), proxyIP}
 	for _, IPAddress := range expectedIPAddresses {
 		found := false
 		for _, val := range altNames.IPs {

cmd/kubeadm/app/util/etcd/etcd_test.go

@@ -84,6 +84,64 @@ spec:
       type: DirectoryOrCreate
     name: etcd-certs
 status: {}
+`
+	secureExposedEtcdPod = `
+apiVersion: v1
+kind: Pod
+metadata:
+  annotations:
+    scheduler.alpha.kubernetes.io/critical-pod: ""
+  creationTimestamp: null
+  labels:
+    component: etcd
+    tier: control-plane
+  name: etcd
+  namespace: kube-system
+spec:
+  containers:
+  - command:
+    - etcd
+    - --advertise-client-urls=https://10.0.5.5:2379
+    - --data-dir=/var/lib/etcd
+    - --peer-key-file=/etc/kubernetes/pki/etcd/peer.key
+    - --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt
+    - --listen-client-urls=https://[::0:0]:2379
+    - --peer-client-cert-auth=true
+    - --cert-file=/etc/kubernetes/pki/etcd/server.crt
+    - --key-file=/etc/kubernetes/pki/etcd/server.key
+    - --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt
+    - --peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt
+    - --client-cert-auth=true
+    image: k8s.gcr.io/etcd-amd64:3.1.12
+    livenessProbe:
+      exec:
+        command:
+        - /bin/sh
+        - -ec
+        - ETCDCTL_API=3 etcdctl --endpoints=https://[::1]:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt
+          --cert=/etc/kubernetes/pki/etcd/healthcheck-client.crt --key=/etc/kubernetes/pki/etcd/healthcheck-client.key
+          get foo
+      failureThreshold: 8
+      initialDelaySeconds: 15
+      timeoutSeconds: 15
+    name: etcd
+    resources: {}
+    volumeMounts:
+    - mountPath: /var/lib/etcd
+      name: etcd-data
+    - mountPath: /etc/kubernetes/pki/etcd
+      name: etcd-certs
+  hostNetwork: true
+  volumes:
+  - hostPath:
+      path: /var/lib/etcd
+      type: DirectoryOrCreate
+    name: etcd-data
+  - hostPath:
+      path: /etc/kubernetes/pki/etcd
+      type: DirectoryOrCreate
+    name: etcd-certs
+status: {}
 `
 	insecureEtcdPod = `# generated by kubeadm v1.9.6
 apiVersion: v1
@@ -145,6 +203,13 @@ func TestPodManifestHasTLS(t *testing.T) {
 			writeManifest: true,
 			expectErr:     false,
 		},
+		{
+			description:   "secure exposed etcd returns true",
+			podYaml:       secureExposedEtcdPod,
+			hasTLS:        true,
+			writeManifest: true,
+			expectErr:     false,
+		},
 		{
 			description:   "insecure etcd returns false",
 			podYaml:       insecureEtcdPod,

cmd/kubeadm/app/util/staticpod/utils.go

@@ -102,7 +102,7 @@ func ComponentProbe(cfg *kubeadmapi.MasterConfiguration, componentName string, p
 func EtcdProbe(cfg *kubeadmapi.MasterConfiguration, componentName string, port int, certsDir string, CACertName string, CertName string, KeyName string) *v1.Probe {
 	tlsFlags := fmt.Sprintf("--cacert=%[1]s/%[2]s --cert=%[1]s/%[3]s --key=%[1]s/%[4]s", certsDir, CACertName, CertName, KeyName)
 	// etcd pod is alive if a linearizable get succeeds.
-	cmd := fmt.Sprintf("ETCDCTL_API=3 etcdctl --endpoints=%s:%d %s get foo", GetProbeAddress(cfg, componentName), port, tlsFlags)
+	cmd := fmt.Sprintf("ETCDCTL_API=3 etcdctl --endpoints=https://[%s]:%d %s get foo", GetProbeAddress(cfg, componentName), port, tlsFlags)
 
 	return &v1.Probe{
 		Handler: v1.Handler{
@@ -254,6 +254,13 @@ func GetProbeAddress(cfg *kubeadmapi.MasterConfiguration, componentName string)
 				}
 				// Return the IP if the URL contains an address instead of a name.
 				if ip := net.ParseIP(parsedURL.Hostname()); ip != nil {
+					// etcdctl doesn't support auto-converting zero addresses into loopback addresses
+					if ip.Equal(net.IPv4zero) {
+						return "127.0.0.1"
+					}
+					if ip.Equal(net.IPv6zero) {
+						return net.IPv6loopback.String()
+					}
 					return ip.String()
 				}
 				// Use the local resolver to try resolving the name within the URL.

cmd/kubeadm/app/util/staticpod/utils_test.go

@@ -219,7 +219,79 @@ func TestEtcdProbe(t *testing.T) {
 			cacert:    "ca1",
 			cert:      "cert1",
 			key:       "key1",
-			expected:  "ETCDCTL_API=3 etcdctl --endpoints=1.2.3.4:1 --cacert=secretsA/ca1 --cert=secretsA/cert1 --key=secretsA/key1 get foo",
+			expected:  "ETCDCTL_API=3 etcdctl --endpoints=https://[1.2.3.4]:1 --cacert=secretsA/ca1 --cert=secretsA/cert1 --key=secretsA/key1 get foo",
+		},
+		{
+			name: "valid etcd probe using listen-client-urls unspecified IPv6 address",
+			cfg: &kubeadmapi.MasterConfiguration{
+				Etcd: kubeadmapi.Etcd{
+					Local: &kubeadmapi.LocalEtcd{
+						ExtraArgs: map[string]string{
+							"listen-client-urls": "http://[0:0:0:0:0:0:0:0]:2379"},
+					},
+				},
+			},
+			component: kubeadmconstants.Etcd,
+			port:      1,
+			certsDir:  "secretsB",
+			cacert:    "ca2",
+			cert:      "cert2",
+			key:       "key2",
+			expected:  "ETCDCTL_API=3 etcdctl --endpoints=https://[::1]:1 --cacert=secretsB/ca2 --cert=secretsB/cert2 --key=secretsB/key2 get foo",
+		},
+		{
+			name: "valid etcd probe using listen-client-urls unspecified IPv6 address 2",
+			cfg: &kubeadmapi.MasterConfiguration{
+				Etcd: kubeadmapi.Etcd{
+					Local: &kubeadmapi.LocalEtcd{
+						ExtraArgs: map[string]string{
+							"listen-client-urls": "http://[::0:0]:2379"},
+					},
+				},
+			},
+			component: kubeadmconstants.Etcd,
+			port:      1,
+			certsDir:  "secretsB",
+			cacert:    "ca2",
+			cert:      "cert2",
+			key:       "key2",
+			expected:  "ETCDCTL_API=3 etcdctl --endpoints=https://[::1]:1 --cacert=secretsB/ca2 --cert=secretsB/cert2 --key=secretsB/key2 get foo",
+		},
+		{
+			name: "valid etcd probe using listen-client-urls unspecified IPv6 address 3",
+			cfg: &kubeadmapi.MasterConfiguration{
+				Etcd: kubeadmapi.Etcd{
+					Local: &kubeadmapi.LocalEtcd{
+						ExtraArgs: map[string]string{
+							"listen-client-urls": "http://[::]:2379"},
+					},
+				},
+			},
+			component: kubeadmconstants.Etcd,
+			port:      1,
+			certsDir:  "secretsB",
+			cacert:    "ca2",
+			cert:      "cert2",
+			key:       "key2",
+			expected:  "ETCDCTL_API=3 etcdctl --endpoints=https://[::1]:1 --cacert=secretsB/ca2 --cert=secretsB/cert2 --key=secretsB/key2 get foo",
+		},
+		{
+			name: "valid etcd probe using listen-client-urls unspecified IPv4 address",
+			cfg: &kubeadmapi.MasterConfiguration{
+				Etcd: kubeadmapi.Etcd{
+					Local: &kubeadmapi.LocalEtcd{
+						ExtraArgs: map[string]string{
+							"listen-client-urls": "http://1.2.3.4:2379,http://4.3.2.1:2379"},
+					},
+				},
+			},
+			component: kubeadmconstants.Etcd,
+			port:      1,
+			certsDir:  "secretsA",
+			cacert:    "ca1",
+			cert:      "cert1",
+			key:       "key1",
+			expected:  "ETCDCTL_API=3 etcdctl --endpoints=https://[1.2.3.4]:1 --cacert=secretsA/ca1 --cert=secretsA/cert1 --key=secretsA/key1 get foo",
 		},
 		{
 			name: "valid etcd probe using listen-client-urls IPv6 addresses",
@@ -237,7 +309,7 @@ func TestEtcdProbe(t *testing.T) {
 			cacert:    "ca2",
 			cert:      "cert2",
 			key:       "key2",
-			expected:  "ETCDCTL_API=3 etcdctl --endpoints=2001:db8::1:1 --cacert=secretsB/ca2 --cert=secretsB/cert2 --key=secretsB/key2 get foo",
+			expected:  "ETCDCTL_API=3 etcdctl --endpoints=https://[2001:db8::1]:1 --cacert=secretsB/ca2 --cert=secretsB/cert2 --key=secretsB/key2 get foo",
 		},
 		{
 			name: "valid IPv4 etcd probe using hostname for listen-client-urls",
@@ -255,7 +327,7 @@ func TestEtcdProbe(t *testing.T) {
 			cacert:    "ca3",
 			cert:      "cert3",
 			key:       "key3",
-			expected:  "ETCDCTL_API=3 etcdctl --endpoints=127.0.0.1:1 --cacert=secretsC/ca3 --cert=secretsC/cert3 --key=secretsC/key3 get foo",
+			expected:  "ETCDCTL_API=3 etcdctl --endpoints=https://[127.0.0.1]:1 --cacert=secretsC/ca3 --cert=secretsC/cert3 --key=secretsC/key3 get foo",
 		},
 	}
 	for _, rt := range tests {