From 7f09d592154efc9ed608d8d460ce5d0a20f1f7fe Mon Sep 17 00:00:00 2001
From: Maciej Borsz <maciejborsz@google.com>
Date: Thu, 3 Dec 2020 09:02:43 +0100
Subject: [PATCH] Migrate etcd's livenessProbe to etcdctl endpoint health.

Change-Id: Ie19c844050c75e3d1c4b431d09ba0ac851c5317b
---
 cluster/gce/gci/configure-helper.sh | 10 ++++++----
 cluster/gce/manifests/etcd.manifest | 14 +++++++++-----
 cluster/images/etcd/Dockerfile      |  1 +
 3 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh
index f6908905b15..8b74a4a7e8a 100644
--- a/cluster/gce/gci/configure-helper.sh
+++ b/cluster/gce/gci/configure-helper.sh
@@ -1718,7 +1718,8 @@ function prepare-etcd-manifest {
   local etcd_apiserver_creds="${ETCD_APISERVER_CREDS:-}"
   local etcd_extra_args="${ETCD_EXTRA_ARGS:-}"
   local suffix="$1"
-  local etcd_livenessprobe_port="$2"
+  local etcd_listen_metrics_port="$2"
+  local etcdctl_certs=""
 
   if [[ -n "${INITIAL_ETCD_CLUSTER_STATE:-}" ]]; then
     cluster_state="${INITIAL_ETCD_CLUSTER_STATE}"
@@ -1731,9 +1732,10 @@ function prepare-etcd-manifest {
   # mTLS should only be enabled for etcd server but not etcd-events. if $1 suffix is empty, it's etcd server.
   if [[ -z "${suffix}" && -n "${ETCD_APISERVER_CA_KEY:-}" && -n "${ETCD_APISERVER_CA_CERT:-}" && -n "${ETCD_APISERVER_SERVER_KEY:-}" && -n "${ETCD_APISERVER_SERVER_CERT:-}" && -n "${ETCD_APISERVER_CLIENT_KEY:-}" && -n "${ETCD_APISERVER_CLIENT_CERT:-}" ]]; then
     etcd_apiserver_creds=" --client-cert-auth --trusted-ca-file ${ETCD_APISERVER_CA_CERT_PATH} --cert-file ${ETCD_APISERVER_SERVER_CERT_PATH} --key-file ${ETCD_APISERVER_SERVER_KEY_PATH} "
+    etcdctl_certs="--cacert ${ETCD_APISERVER_CA_CERT_PATH} --cert ${ETCD_APISERVER_CLIENT_CERT_PATH} --key ${ETCD_APISERVER_CLIENT_KEY_PATH}"
     etcd_apiserver_protocol="https"
-    etcd_livenessprobe_port="2382"
-    etcd_extra_args+=" --listen-metrics-urls=http://${ETCD_LISTEN_CLIENT_IP:-127.0.0.1}:${etcd_livenessprobe_port} "
+    etcd_listen_metrics_port="2382"
+    etcd_extra_args+=" --listen-metrics-urls=http://${ETCD_LISTEN_CLIENT_IP:-127.0.0.1}:${etcd_listen_metrics_port} "
   fi
 
   if [[ -n "${ETCD_PROGRESS_NOTIFY_INTERVAL:-}" ]]; then
@@ -1787,9 +1789,9 @@ function prepare-etcd-manifest {
   sed -i -e "s@{{ *etcd_protocol *}}@$etcd_protocol@g" "${temp_file}"
   sed -i -e "s@{{ *etcd_apiserver_protocol *}}@$etcd_apiserver_protocol@g" "${temp_file}"
   sed -i -e "s@{{ *etcd_creds *}}@$etcd_creds@g" "${temp_file}"
+  sed -i -e "s@{{ *etcdctl_certs *}}@$etcdctl_certs@g" "${temp_file}"
   sed -i -e "s@{{ *etcd_apiserver_creds *}}@$etcd_apiserver_creds@g" "${temp_file}"
   sed -i -e "s@{{ *etcd_extra_args *}}@$etcd_extra_args@g" "${temp_file}"
-  sed -i -e "s@{{ *etcd_livenessprobe_port *}}@$etcd_livenessprobe_port@g" "${temp_file}"
   if [[ -n "${ETCD_VERSION:-}" ]]; then
     sed -i -e "s@{{ *pillar\.get('etcd_version', '\(.*\)') *}}@${ETCD_VERSION}@g" "${temp_file}"
   else
diff --git a/cluster/gce/manifests/etcd.manifest b/cluster/gce/manifests/etcd.manifest
index 664883ddf92..f927c204e33 100644
--- a/cluster/gce/manifests/etcd.manifest
+++ b/cluster/gce/manifests/etcd.manifest
@@ -61,13 +61,17 @@
       }
         ],
     "livenessProbe": {
-      "httpGet": {
-        "host": "127.0.0.1",
-        "port": {{ etcd_livenessprobe_port }},
-        "path": "/health"
+      "exec": {
+        "command": [
+          "/bin/sh",
+          "-c",
+          "set -x; exec /usr/local/bin/etcdctl --endpoints=127.0.0.1:{{ port }} {{ etcdctl_certs }} --command-timeout=15s endpoint health"
+        ]
       },
       "initialDelaySeconds": {{ liveness_probe_initial_delay }},
-      "timeoutSeconds": 15
+      "timeoutSeconds": 15,
+      "periodSeconds": 5,
+      "failureThreshold": 5
     },
     "ports": [
       { "name": "serverport",
diff --git a/cluster/images/etcd/Dockerfile b/cluster/images/etcd/Dockerfile
index 4e8e4027c7d..61a1bfa5db3 100644
--- a/cluster/images/etcd/Dockerfile
+++ b/cluster/images/etcd/Dockerfile
@@ -29,6 +29,7 @@ WORKDIR /
 COPY --from=builder /sh /bin/
 
 EXPOSE 2379 2380 4001 7001
+# etcdctl is used by etcd.manifest for livenessProbe.
 COPY etcd* etcdctl* /usr/local/bin/
 COPY cp* /bin/
 COPY migrate-if-needed.sh migrate /usr/local/bin/