From 3e4744c7364808e6a78b4f92bff30692a0abc179 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bartek=20=C5=BBurawski?= <bartekzurawski1@gmail.com>
Date: Thu, 26 Mar 2020 19:07:23 +0100
Subject: [PATCH] Set ip alias route on kubernetes-master during booting

---
 cluster/gce/gci/kube-master-internal-route.sh | 27 +++++++++++++++++++
 cluster/gce/gci/master-helper.sh              |  1 +
 cluster/gce/gci/master.yaml                   | 19 +++++++++++++
 cluster/gce/util.sh                           |  4 ++-
 4 files changed, 50 insertions(+), 1 deletion(-)
 create mode 100644 cluster/gce/gci/kube-master-internal-route.sh

diff --git a/cluster/gce/gci/kube-master-internal-route.sh b/cluster/gce/gci/kube-master-internal-route.sh
new file mode 100644
index 00000000000..5ebf7b023af
--- /dev/null
+++ b/cluster/gce/gci/kube-master-internal-route.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+METADATA_ENDPOINT="http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-master-internal-ip"
+METADATA_HEADER="Metadata-Flavor: Google"
+ip=$(curl -s --fail ${METADATA_ENDPOINT} -H "${METADATA_HEADER}")
+if [ -n "$ip" ];
+then
+    # Check if route is already set if not set it
+    if ! sudo ip route show table local | grep -q "$(echo "$ip" | cut -d'/' -f 1)";
+    then
+            sudo ip route add to local "${ip}/32" dev "$(ip route | grep default | awk '{print $5}')"
+    fi
+fi
diff --git a/cluster/gce/gci/master-helper.sh b/cluster/gce/gci/master-helper.sh
index 26319c495f3..0baf588be10 100755
--- a/cluster/gce/gci/master-helper.sh
+++ b/cluster/gce/gci/master-helper.sh
@@ -157,6 +157,7 @@ function create-master-instance-internal() {
   metadata="${metadata},gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt"
   metadata="${metadata},kube-master-certs=${KUBE_TEMP}/kube-master-certs.yaml"
   metadata="${metadata},cluster-location=${KUBE_TEMP}/cluster-location.txt"
+  metadata="${metadata},kube-master-internal-route=${KUBE_ROOT}/cluster/gce/gci/kube-master-internal-route.sh"
   metadata="${metadata},${MASTER_EXTRA_METADATA}"
 
   local disk="name=${master_name}-pd"
diff --git a/cluster/gce/gci/master.yaml b/cluster/gce/gci/master.yaml
index fd0a88d081d..884133c8227 100644
--- a/cluster/gce/gci/master.yaml
+++ b/cluster/gce/gci/master.yaml
@@ -23,6 +23,24 @@ write_files:
       [Install]
       WantedBy=kubernetes.target
 
+  - path: /etc/systemd/system/kube-master-internal-route.service
+    permissions: 0644
+    owner: root
+    content: |
+      [Unit]
+      Description=Configure kube internal route
+      After=kube-master-installation.service
+
+      [Service]
+      Type=oneshot
+      RemainAfterExit=yes
+      ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/kubernetes/bin/kube-master-internal-route.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-master-internal-route
+      ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/kube-master-internal-route.sh
+      ExecStart=/home/kubernetes/bin/kube-master-internal-route.sh
+
+      [Install]
+      WantedBy=kubernetes.target
+
   - path: /etc/systemd/system/kube-master-configuration.service
     permissions: 0644
     owner: root
@@ -119,6 +137,7 @@ write_files:
 runcmd:
  - systemctl daemon-reload
  - systemctl enable kube-master-installation.service
+ - systemctl enable kube-master-internal-route.service
  - systemctl enable kube-master-configuration.service
  - systemctl enable kube-container-runtime-monitor.service
  - systemctl enable kubelet-monitor.service
diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh
index d16a77cbd16..e44d29fa0f2 100755
--- a/cluster/gce/util.sh
+++ b/cluster/gce/util.sh
@@ -2946,7 +2946,8 @@ function attach-internal-master-ip() {
   echo "Setting ${name}'s aliases to '${aliases}' (added ${ip})"
   # Attach ${ip} to ${name}
   gcloud compute instances network-interfaces update "${name}" --project "${PROJECT}" --zone "${zone}" --aliases="${aliases}"
-  run-gcloud-command "${name}" "${zone}" 'sudo ip route add to local '${ip}'/32 dev $(ip route | grep default | awk '\''{print $5}'\'')' || true
+  gcloud compute instances add-metadata "${name}" --zone "${zone}" --metadata=kube-master-internal-ip="${ip}"
+  run-gcloud-command "${name}" "${zone}" 'sudo /bin/bash /home/kubernetes/bin/kube-master-internal-route.sh' || true
   return $?
 }
 
@@ -2964,6 +2965,7 @@ function detach-internal-master-ip() {
   echo "Setting ${name}'s aliases to '${aliases}' (removed ${ip})"
   # Detach ${MASTER_NAME}-internal-ip from ${name}
   gcloud compute instances network-interfaces update "${name}" --project "${PROJECT}" --zone "${zone}" --aliases="${aliases}"
+  gcloud compute instances remove-metadata "${name}" --zone "${zone}" --keys=kube-master-internal-ip
   run-gcloud-command "${name}" "${zone}" 'sudo ip route del to local '${ip}'/32 dev $(ip route | grep default | awk '\''{print $5}'\'')' || true
   return $?
 }