Merge pull request #56132 from jingax10/upgrade_aliases_branch

Automatic merge from submit-queue (batch tested with PRs 58412, 56132, 58506, 58542, 58394). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

 Implement upgrade-aliases.sh to migrate a route-based k8s cluster to use IP aliases in GCE.

**What this PR does / why we need it**:
This PR implements upgrade-aliases.sh, which is used to migrate a route-based k8s cluster to use IP aliases in GCE.

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #

**Special notes for your reviewer**:

**Release note**:

```release-note
"NONE"
```

cluster/gce/config-default.sh

@@ -267,14 +267,19 @@ ENABLE_RESCHEDULER="${KUBE_ENABLE_RESCHEDULER:-true}"
 # IP_ALIAS_SUBNETWORK is the subnetwork to allocate from. If empty, a
 #   new subnetwork will be created for the cluster.
 ENABLE_IP_ALIASES=${KUBE_GCE_ENABLE_IP_ALIASES:-false}
+NODE_IPAM_MODE=${KUBE_GCE_NODE_IPAM_MODE:-RangeAllocator}
 if [ ${ENABLE_IP_ALIASES} = true ]; then
   # Size of ranges allocated to each node. Currently supports only /32 and /24.
   IP_ALIAS_SIZE=${KUBE_GCE_IP_ALIAS_SIZE:-/24}
   IP_ALIAS_SUBNETWORK=${KUBE_GCE_IP_ALIAS_SUBNETWORK:-${INSTANCE_PREFIX}-subnet-default}
   # Reserve the services IP space to avoid being allocated for other GCP resources.
   SERVICE_CLUSTER_IP_SUBNETWORK=${KUBE_GCE_SERVICE_CLUSTER_IP_SUBNETWORK:-${INSTANCE_PREFIX}-subnet-services}
+  NODE_IPAM_MODE=${KUBE_GCE_NODE_IPAM_MODE:-CloudAllocator}
+  SECONDARY_RANGE_NAME=${SECONDARY_RANGE_NAME:-}
   # Add to the provider custom variables.
   PROVIDER_VARS="${PROVIDER_VARS:-} ENABLE_IP_ALIASES"
+  PROVIDER_VARS="${PROVIDER_VARS:-} NODE_IPAM_MODE"
+  PROVIDER_VARS="${PROVIDER_VARS:-} SECONDARY_RANGE_NAME"
 fi
 
 # Enable GCE Alpha features.

cluster/gce/gci/configure-helper.sh

@@ -1835,7 +1835,7 @@ function start-kube-controller-manager {
     params+=" --terminated-pod-gc-threshold=${TERMINATED_POD_GC_THRESHOLD}"
   fi
   if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then
-    params+=" --cidr-allocator-type=CloudAllocator"
+    params+=" --cidr-allocator-type=${NODE_IPAM_MODE}"
     params+=" --configure-cloud-routes=false"
   fi
   if [[ -n "${FEATURE_GATES:-}" ]]; then

cluster/gce/gci/master-helper.sh

@@ -94,9 +94,16 @@ function create-master-instance-internal() {
     preemptible_master="--preemptible --maintenance-policy TERMINATE"
   fi
 
+  local enable_ip_aliases
+  if [[ "${NODE_IPAM_MODE:-}" == "CloudAllocator" ]]; then
+    enable_ip_aliases=true
+  else
+    enable_ip_aliases=false
+  fi
+
   local network=$(make-gcloud-network-argument \
     "${NETWORK_PROJECT}" "${REGION}" "${NETWORK}" "${SUBNETWORK:-}" \
-    "${address:-}" "${ENABLE_IP_ALIASES:-}" "${IP_ALIAS_SIZE:-}")
+    "${address:-}" "${enable_ip_aliases:-}" "${IP_ALIAS_SIZE:-}")
 
   local metadata="kube-env=${KUBE_TEMP}/master-kube-env.yaml"
   metadata="${metadata},user-data=${KUBE_ROOT}/cluster/gce/gci/master.yaml"

cluster/gce/upgrade-aliases.sh

@@ -0,0 +1,171 @@
+#!/bin/bash
+
+# Copyright 2018 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# !!!EXPERIMENTAL!!! Upgrade a K8s cluster from routes to IP aliases for
+# node connectivity on GCE. This is only for migration.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+if [[ "${KUBERNETES_PROVIDER:-gce}" != "gce" ]]; then
+  echo "ERR: KUBERNETES_PROVIDER must be gce" >&2
+  exit 1
+fi
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
+source "${KUBE_ROOT}/hack/lib/util.sh"
+source "${KUBE_ROOT}/cluster/kube-util.sh"
+
+# Print the number of routes used for K8s cluster node connectivity.
+#
+# Assumed vars:
+#   PROJECT
+function get-k8s-node-routes-count() {
+  local k8s_node_routes_count=$(gcloud compute routes list \
+    --project=${PROJECT} --filter='description=k8s-node-route' \
+    --format='value(name)' | wc -l)
+  echo -n "${k8s_node_routes_count}"
+}
+
+# Detect the subnetwork where the K8s cluster resides.
+#
+# Assumed vars:
+#  KUBE_MASTER
+#  PROJECT
+#  ZONE
+# Vars set:
+#  IP_ALIAS_SUBNETWORK
+function detect-k8s-subnetwork() {
+  local subnetwork_url=$(gcloud compute instances describe \
+    ${KUBE_MASTER} --project=${PROJECT} --zone=${ZONE} \
+    --format='value(networkInterfaces[0].subnetwork)')
+  if [ -n ${subnetwork_url} ]; then
+    IP_ALIAS_SUBNETWORK=$(echo ${subnetwork_url##*/})
+  fi
+}
+
+# Set IP_ALIAS_SUBNETWORK's allowSubnetCidrRoutesOverlap to a boolean value.
+# $1: true or false for the desired allowSubnetCidrRoutesOverlap.
+#
+# Assumed vars:
+#   IP_ALIAS_SUBNETWORK
+#   GCE_API_ENDPOINT
+#   PROJECT
+#   REGION
+function set-allow-subnet-cidr-routes-overlap() {
+  local allow_subnet_cidr_routes_overlap
+  allow_subnet_cidr_routes_overlap=$(gcloud beta compute networks subnets \
+    describe ${IP_ALIAS_SUBNETWORK} --project=${PROJECT} --region=${REGION} \
+    --format='value(allowSubnetCidrRoutesOverlap)')
+  local allow_overlap=$1
+  if [ ${allow_subnet_cidr_routes_overlap,,} = ${allow_overlap} ]; then
+    echo "Subnet ${IP_ALIAS_SUBNETWORK}'s allowSubnetCidrRoutesOverlap is already set as $1"
+    return
+  fi
+
+  echo "Setting subnet \"${IP_ALIAS_SUBNETWORK}\" allowSubnetCidrRoutesOverlap to $1"
+  local fingerprint=$(gcloud beta compute networks subnets describe \
+    ${IP_ALIAS_SUBNETWORK} --project=${PROJECT} --region=${REGION} \
+    --format='value(fingerprint)')
+  local access_token=$(gcloud auth print-access-token)
+  local request="{\"allowSubnetCidrRoutesOverlap\":$1, \"fingerprint\":\"${fingerprint}\"}"
+  local subnetwork_url="${GCE_API_ENDPOINT}projects/${PROJECT}/regions/${REGION}/subnetworks/${IP_ALIAS_SUBNETWORK}"
+  until curl -s --header "Content-Type: application/json" --header "Authorization: Bearer ${access_token}" \
+    -X PATCH -d "${request}" "${subnetwork_url}" --output /dev/null; do
+    printf "."
+    sleep 1
+  done
+}
+
+# Add secondary ranges to K8s subnet.
+#
+# Assumed vars:
+#   IP_ALIAS_SUBNETWORK
+#   PROJECT
+#   REGION
+#   CLUSTER_IP_RANGE
+#   SERVICE_CLUSTER_IP_RANGE
+function add-k8s-subnet-secondary-ranges() {
+  local secondary_ranges=$(gcloud beta compute networks subnets describe "${IP_ALIAS_SUBNETWORK}" \
+    --project="${PROJECT}" --region="${REGION}" \
+    --format='value(secondaryIpRanges)')
+  if [[ "${secondary_ranges}" =~ "pods-default" && "${secondary_ranges}" =~ "services-default" ]]; then
+    echo "${secondary_ranges} already contains both pods-default and services-default secondary ranges"
+    return
+  fi
+
+  echo "Adding secondary ranges: pods-default (${CLUSTER_IP_RANGE}), services-default (${SERVICE_CLUSTER_IP_RANGE})"
+  until gcloud beta compute networks subnets update ${IP_ALIAS_SUBNETWORK} \
+    --project=${PROJECT} --region=${REGION} \
+    --add-secondary-ranges="pods-default=${CLUSTER_IP_RANGE},services-default=${SERVICE_CLUSTER_IP_RANGE}"; do
+    printf "."
+    sleep 1
+  done
+}
+
+# Delete all K8s node routes.
+#
+# Assumed vars:
+#   PROJECT
+function delete-k8s-node-routes() {
+  local -a routes
+  local -r batch=200
+  routes=( $(gcloud compute routes list \
+    --project=${PROJECT} --filter='description=k8s-node-route' \
+    --format='value(name)') )
+  while (( "${#routes[@]}" > 0 )); do
+      echo Deleting k8s node routes "${routes[*]::${batch}}"
+      gcloud compute routes delete --project "${PROJECT}" --quiet "${routes[@]::${batch}}"
+      routes=( "${routes[@]:${batch}}" )
+  done
+}
+
+detect-project
+detect-master
+
+k8s_node_routes_count=$(get-k8s-node-routes-count)
+if [[ "${k8s_node_routes_count}" -eq 0 ]]; then
+  echo "No k8s node routes found and IP alias should already be enabled. Exiting..."
+  exit 0
+fi
+echo "Found ${k8s_node_routes_count} K8s node routes. Proceeding to upgrade them to IP aliases based connectivity..."
+
+detect-k8s-subnetwork
+if [ -z ${IP_ALIAS_SUBNETWORK} ]; then
+  echo "No k8s cluster subnetwork found. Exiting..."
+  exit 1
+fi
+echo "k8s cluster sits on subnetwork \"${IP_ALIAS_SUBNETWORK}\""
+
+set-allow-subnet-cidr-routes-overlap true
+add-k8s-subnet-secondary-ranges
+
+echo "Changing K8s master envs and restarting..."
+export KUBE_GCE_IP_ALIAS_SUBNETWORK=${IP_ALIAS_SUBNETWORK}
+export KUBE_GCE_NODE_IPAM_MODE="IPAMFromCluster"
+export KUBE_GCE_ENABLE_IP_ALIASES=true
+export SECONDARY_RANGE_NAME="pods-default"
+export STORAGE_BACKEND="etcd3"
+export STORAGE_MEDIA_TYPE="application/vnd.kubernetes.protobuf"
+export ETCD_IMAGE=3.1.10
+export ETCD_VERSION=3.1.10
+
+# Upgrade master with updated kube envs
+${KUBE_ROOT}/cluster/gce/upgrade.sh -M -l
+
+delete-k8s-node-routes
+set-allow-subnet-cidr-routes-overlap false