From 551ffbe7bfd8bacfcebdb2a0db0450d3b204d580 Mon Sep 17 00:00:00 2001
From: Jing Ai <jinga@google.com>
Date: Mon, 20 Nov 2017 21:11:50 -0800
Subject: [PATCH 01/16] Implement upgrade-aliases.sh to migrate a route-based
 k8s cluster to use IP aliases in GCE.

---
 cluster/gce/config-default.sh                 |   5 +
 .../gce/container-linux/configure-helper.sh   |   2 +-
 cluster/gce/container-linux/master-helper.sh  |   9 +-
 cluster/gce/gci/configure-helper.sh           |   2 +-
 cluster/gce/upgrade-aliases.sh                | 165 ++++++++++++++++++
 pkg/controller/node/ipam/sync/sync.go         |   8 +-
 6 files changed, 182 insertions(+), 9 deletions(-)
 create mode 100755 cluster/gce/upgrade-aliases.sh

diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh
index 3c9cb99af95..cf2f1f71da3 100755
--- a/cluster/gce/config-default.sh
+++ b/cluster/gce/config-default.sh
@@ -253,14 +253,19 @@ ENABLE_RESCHEDULER="${KUBE_ENABLE_RESCHEDULER:-true}"
 # IP_ALIAS_SUBNETWORK is the subnetwork to allocate from. If empty, a
 #   new subnetwork will be created for the cluster.
 ENABLE_IP_ALIASES=${KUBE_GCE_ENABLE_IP_ALIASES:-false}
+NODE_IPAM_MODE=${KUBE_GCE_NODE_IPAM_MODE:-RangeAllocator}
 if [ ${ENABLE_IP_ALIASES} = true ]; then
   # Size of ranges allocated to each node. Currently supports only /32 and /24.
   IP_ALIAS_SIZE=${KUBE_GCE_IP_ALIAS_SIZE:-/24}
   IP_ALIAS_SUBNETWORK=${KUBE_GCE_IP_ALIAS_SUBNETWORK:-${INSTANCE_PREFIX}-subnet-default}
   # Reserve the services IP space to avoid being allocated for other GCP resources.
   SERVICE_CLUSTER_IP_SUBNETWORK=${KUBE_GCE_SERVICE_CLUSTER_IP_SUBNETWORK:-${INSTANCE_PREFIX}-subnet-services}
+  NODE_IPAM_MODE=${KUBE_GCE_NODE_IPAM_MODE:-CloudAllocator}
+  SECONDARY_RANGE_NAME=${SECONDARY_RANGE_NAME:-}
   # Add to the provider custom variables.
   PROVIDER_VARS="${PROVIDER_VARS:-} ENABLE_IP_ALIASES"
+  PROVIDER_VARS="${PROVIDER_VARS:-} NODE_IPAM_MODE"
+  PROVIDER_VARS="${PROVIDER_VARS:-} SECONDARY_RANGE_NAME"
 fi
 
 # Enable GCE Alpha features.
diff --git a/cluster/gce/container-linux/configure-helper.sh b/cluster/gce/container-linux/configure-helper.sh
index ac54e662f3f..8e90a695e8b 100755
--- a/cluster/gce/container-linux/configure-helper.sh
+++ b/cluster/gce/container-linux/configure-helper.sh
@@ -1104,7 +1104,7 @@ function start-kube-controller-manager {
     params+=" --terminated-pod-gc-threshold=${TERMINATED_POD_GC_THRESHOLD}"
   fi
   if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then
-    params+=" --cidr-allocator-type=CloudAllocator"
+    params+=" --cidr-allocator-type=${NODE_IPAM_MODE}"
     params+=" --configure-cloud-routes=false"
   fi
   if [[ -n "${FEATURE_GATES:-}" ]]; then
diff --git a/cluster/gce/container-linux/master-helper.sh b/cluster/gce/container-linux/master-helper.sh
index e727349f2c7..f8177fd308f 100755
--- a/cluster/gce/container-linux/master-helper.sh
+++ b/cluster/gce/container-linux/master-helper.sh
@@ -79,9 +79,16 @@ function create-master-instance-internal() {
     preemptible_master="--preemptible --maintenance-policy TERMINATE"
   fi
 
+  local enable_ip_aliases
+  if [[ "${NODE_IPAM_MODE:-}" == "CloudAllocator" ]]; then
+    enable_ip_aliases=true
+  else
+    enable_ip_aliases=false
+  fi
+
   local network=$(make-gcloud-network-argument \
     "${NETWORK_PROJECT}" "${REGION}" "${NETWORK}" "${SUBNETWORK:-}" \
-    "${address:-}" "${ENABLE_IP_ALIASES:-}" "${IP_ALIAS_SIZE:-}")
+    "${address:-}" "${enable_ip_aliases:-}" "${IP_ALIAS_SIZE:-}")
 
   local metadata="kube-env=${KUBE_TEMP}/master-kube-env.yaml"
   metadata="${metadata},user-data=${KUBE_ROOT}/cluster/gce/container-linux/master.yaml"
diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh
index f5407936ec0..00bcf556d0e 100644
--- a/cluster/gce/gci/configure-helper.sh
+++ b/cluster/gce/gci/configure-helper.sh
@@ -1601,7 +1601,7 @@ function start-kube-controller-manager {
     params+=" --terminated-pod-gc-threshold=${TERMINATED_POD_GC_THRESHOLD}"
   fi
   if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then
-    params+=" --cidr-allocator-type=CloudAllocator"
+    params+=" --cidr-allocator-type=${NODE_IPAM_MODE}"
     params+=" --configure-cloud-routes=false"
   fi
   if [[ -n "${FEATURE_GATES:-}" ]]; then
diff --git a/cluster/gce/upgrade-aliases.sh b/cluster/gce/upgrade-aliases.sh
new file mode 100755
index 00000000000..311f629d4aa
--- /dev/null
+++ b/cluster/gce/upgrade-aliases.sh
@@ -0,0 +1,165 @@
+#!/bin/bash
+
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# !!!EXPERIMENTAL!!! Upgrade a k8s cluster from routes to IP aliases for
+# node connectivity on GCE. This is only for migration.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+if [[ "${KUBERNETES_PROVIDER:-gce}" != "gce" ]]; then
+  echo "!!! ${1} only works on GCE" >&2
+  exit 1
+fi
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
+source "${KUBE_ROOT}/hack/lib/util.sh"
+source "${KUBE_ROOT}/cluster/kube-util.sh"
+
+# Print the number of routes used for k8s cluster node connectivity.
+#
+# Assumed vars:
+#   PROJECT
+function get-k8s-node-routes-count() {
+  local k8s_node_routes_count=$(gcloud alpha compute routes list \
+    --project=${PROJECT} --filter='description=k8s-node-route' \
+    --format='value(name)' | wc -l)
+  echo -n "${k8s_node_routes_count}"
+}
+
+# Detect the subnetwork where the k8s cluster resides.
+#
+# Assumed vars:
+#  KUBE_MASTER
+#  PROJECT
+#  ZONE
+# Vars set:
+#  IP_ALIAS_SUBNETWORK
+function detect-k8s-subnetwork() {
+  local subnetwork_url=$(gcloud alpha compute instances describe \
+    ${KUBE_MASTER} --project=${PROJECT} --zone=${ZONE} \
+    --format='value(networkInterfaces[0].subnetwork)')
+  if [ -n ${subnetwork_url} ]; then
+    IP_ALIAS_SUBNETWORK=$(echo ${subnetwork_url##*/})
+  fi
+}
+
+# Set IP_ALIAS_SUBNETWORK's allowSubnetCidrRoutesOverlap to a boolean value.
+# $1: true or false for the desired allowSubnetCidrRoutesOverlap.
+#
+# Assumed vars:
+#   IP_ALIAS_SUBNETWORK
+#   GCE_API_ENDPOINT
+#   PROJECT
+#   REGION
+function set-allow-subnet-cidr-routes-overlap() {
+  local allow_subnet_cidr_routes_overlap
+  allow_subnet_cidr_routes_overlap=$(gcloud alpha compute networks subnets \
+    describe ${IP_ALIAS_SUBNETWORK} --project=${PROJECT} --region=${REGION} \
+    --format='value(allowSubnetCidrRoutesOverlap)')
+  if [ ${allow_subnet_cidr_routes_overlap,,} = $1 ]; then
+    echo "Subnet ${IP_ALIAS_SUBNETWORK}'s allowSubnetCidrRoutesOverlap is already set as $1"
+    return
+  fi
+
+  echo "Setting subnet \"${IP_ALIAS_SUBNETWORK}\" allowSubnetCidrRoutesOverlap to $1"
+  local fingerprint=$(gcloud alpha compute networks subnets describe \
+    ${IP_ALIAS_SUBNETWORK} --project=${PROJECT} --region=${REGION} \
+    --format='value(fingerprint)')
+  local access_token=$(gcloud auth print-access-token)
+  local request="{\"allowSubnetCidrRoutesOverlap\":$1, \"fingerprint\":\"${fingerprint}\"}"
+  local subnetwork_url="${GCE_API_ENDPOINT}projects/${PROJECT}/regions/${REGION}/subnetworks/${IP_ALIAS_SUBNETWORK}"
+  until curl --header "Content-Type: application/json" --header "Authorization: Bearer ${access_token}" \
+    -X PATCH -d "${request}" "${subnetwork_url}" --output /dev/null; do
+    printf "."
+    sleep 1
+  done
+}
+
+# Add secondary ranges to k8s subnet.
+#
+# Assumed vars:
+#   IP_ALIAS_SUBNETWORK
+#   PROJECT
+#   REGION
+#   CLUSTER_IP_RANGE
+#   SERVICE_CLUSTER_IP_RANGE
+function add-k8s-subnet-secondary-ranges() {
+  local secondary_ranges=$(gcloud alpha compute networks subnets describe "${IP_ALIAS_SUBNETWORK}" \
+    --project="${PROJECT}" --region="${REGION}" \
+    --format='value(secondaryIpRanges)')
+  if [[ "${secondary_ranges}" =~ "pods-default" && "${secondary_ranges}" =~ "services-default" ]]; then
+    echo "${secondary_ranges} already contains both pods-default and services-default secondary ranges"
+    return
+  fi
+
+  echo "Adding secondary ranges: pods-default (${CLUSTER_IP_RANGE}), services-default (${SERVICE_CLUSTER_IP_RANGE})"
+  gcloud alpha compute networks subnets update ${IP_ALIAS_SUBNETWORK} \
+    --project=${PROJECT} --region=${REGION} \
+    --add-secondary-ranges="pods-default=${CLUSTER_IP_RANGE},services-default=${SERVICE_CLUSTER_IP_RANGE}"
+}
+
+# Delete all k8s node routes.
+#
+# Assumed vars:
+#   PROJECT
+function delete-k8s-node-routes() {
+  local -a routes
+  local -r batch=200
+  routes=( $(gcloud alpha compute routes list \
+    --project=${PROJECT} --filter='description=k8s-node-route' \
+    --format='value(name)') )
+  while (( "${#routes[@]}" > 0 )); do
+      echo Deleting k8s node routes "${routes[*]::${batch}}"
+      gcloud compute routes delete --project "${PROJECT}" --quiet "${routes[@]::${batch}}"
+      routes=( "${routes[@]:${batch}}" )
+  done
+}
+
+detect-project
+detect-master
+
+k8s_node_routes_count=$(get-k8s-node-routes-count)
+if [[ "${k8s_node_routes_count}" -eq 0 ]]; then
+  echo "No k8s node routes found and IP alias should already be enabled. Exiting..."
+  exit 0
+fi
+echo "Found ${k8s_node_routes_count} k8s node routes. Proceeding to upgrade them to IP aliases based connectivity..."
+
+detect-k8s-subnetwork
+if [ -z ${IP_ALIAS_SUBNETWORK} ]; then
+  echo "No k8s cluster subnetwork found. Exiting..."
+  exit 1
+fi
+echo "k8s cluster sits on subnetwork \"${IP_ALIAS_SUBNETWORK}\""
+
+set-allow-subnet-cidr-routes-overlap true
+add-k8s-subnet-secondary-ranges
+
+echo "Changing k8s master envs and restarting..."
+export KUBE_GCE_IP_ALIAS_SUBNETWORK=${IP_ALIAS_SUBNETWORK}
+export KUBE_GCE_NODE_IPAM_MODE="IPAMFromCluster"
+export KUBE_GCE_ENABLE_IP_ALIASES=true
+export SECONDARY_RANGE_NAME="pods-default"
+export STORAGE_BACKEND="etcd3"
+export STORAGE_MEDIA_TYPE="application/vnd.kubernetes.protobuf"
+
+# Upgrade master with updated kube envs
+${KUBE_ROOT}/cluster/gce/upgrade.sh -M -l
+
+delete-k8s-node-routes
+set-allow-subnet-cidr-routes-overlap false
diff --git a/pkg/controller/node/ipam/sync/sync.go b/pkg/controller/node/ipam/sync/sync.go
index 4995f425543..737826547c2 100644
--- a/pkg/controller/node/ipam/sync/sync.go
+++ b/pkg/controller/node/ipam/sync/sync.go
@@ -244,9 +244,7 @@ func (op *updateOp) validateRange(ctx context.Context, sync *NodeSync, node *v1.
 // alias.
 func (op *updateOp) updateNodeFromAlias(ctx context.Context, sync *NodeSync, node *v1.Node, aliasRange *net.IPNet) error {
 	if sync.mode != SyncFromCloud {
-		sync.kubeAPI.EmitNodeWarningEvent(node.Name, InvalidModeEvent,
-			"Cannot sync from cloud in mode %q", sync.mode)
-		return fmt.Errorf("cannot sync from cloud in mode %q", sync.mode)
+		fmt.Errorf("Detect mode %q while expect mode %q when syncing from cloud", sync.mode, SyncFromCloud)
 	}
 
 	glog.V(2).Infof("Updating node spec with alias range, node.PodCIDR = %v", aliasRange)
@@ -276,9 +274,7 @@ func (op *updateOp) updateNodeFromAlias(ctx context.Context, sync *NodeSync, nod
 // updateAliasFromNode updates the cloud alias given the node allocation.
 func (op *updateOp) updateAliasFromNode(ctx context.Context, sync *NodeSync, node *v1.Node) error {
 	if sync.mode != SyncFromCluster {
-		sync.kubeAPI.EmitNodeWarningEvent(
-			node.Name, InvalidModeEvent, "Cannot sync to cloud in mode %q", sync.mode)
-		return fmt.Errorf("cannot sync to cloud in mode %q", sync.mode)
+		fmt.Errorf("Detect mode %q while expect mode %q when syncing from cluster", sync.mode, SyncFromCluster)
 	}
 
 	_, aliasRange, err := net.ParseCIDR(node.Spec.PodCIDR)

From 302c59f50f4977c1c1d4ab61e1e05327476543f1 Mon Sep 17 00:00:00 2001
From: Jing Ai <jinga@google.com>
Date: Tue, 21 Nov 2017 09:59:48 -0800
Subject: [PATCH 02/16] Fix test failure in sync_test.go.

---
 pkg/controller/node/ipam/sync/sync_test.go | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/pkg/controller/node/ipam/sync/sync_test.go b/pkg/controller/node/ipam/sync/sync_test.go
index d3268480439..94b9775ccbd 100644
--- a/pkg/controller/node/ipam/sync/sync_test.go
+++ b/pkg/controller/node/ipam/sync/sync_test.go
@@ -145,11 +145,10 @@ func TestNodeSyncUpdate(t *testing.T) {
 			events: []fakeEvent{{"node1", "CloudCIDRAllocatorMismatch"}},
 		},
 		{
-			desc:      "update alias from node",
-			mode:      SyncFromCloud,
-			node:      nodeWithCIDRRange,
-			events:    []fakeEvent{{"node1", "CloudCIDRAllocatorInvalidMode"}},
-			wantError: true,
+			desc:   "update alias from node",
+			mode:   SyncFromCloud,
+			node:   nodeWithCIDRRange,
+			events: nil,
 		},
 		{
 			desc: "update alias from node",
@@ -165,12 +164,11 @@ func TestNodeSyncUpdate(t *testing.T) {
 			// XXX/bowei -- validation
 		},
 		{
-			desc:      "update node from alias",
-			mode:      SyncFromCluster,
-			node:      nodeWithoutCIDRRange,
-			fake:      fakeAPIs{aliasRange: test.MustParseCIDR("10.1.2.3/16")},
-			events:    []fakeEvent{{"node1", "CloudCIDRAllocatorInvalidMode"}},
-			wantError: true,
+			desc:   "update node from alias",
+			mode:   SyncFromCluster,
+			node:   nodeWithoutCIDRRange,
+			fake:   fakeAPIs{aliasRange: test.MustParseCIDR("10.1.2.3/16")},
+			events: nil,
 		},
 		{
 			desc:      "allocate range",

From 9acc22ba1f8042b8e0fd78c3a8f3c9adb4a7df94 Mon Sep 17 00:00:00 2001
From: Jing Ai <jinga@google.com>
Date: Wed, 22 Nov 2017 16:15:42 -0800
Subject: [PATCH 03/16] Addressed bowei's comments.

---
 cluster/gce/upgrade-aliases.sh | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/cluster/gce/upgrade-aliases.sh b/cluster/gce/upgrade-aliases.sh
index 311f629d4aa..6157b2a966a 100755
--- a/cluster/gce/upgrade-aliases.sh
+++ b/cluster/gce/upgrade-aliases.sh
@@ -22,7 +22,7 @@ set -o nounset
 set -o pipefail
 
 if [[ "${KUBERNETES_PROVIDER:-gce}" != "gce" ]]; then
-  echo "!!! ${1} only works on GCE" >&2
+  echo "ERR: KUBERNETES_PROVIDER must be gce" >&2
   exit 1
 fi
 
@@ -35,7 +35,7 @@ source "${KUBE_ROOT}/cluster/kube-util.sh"
 # Assumed vars:
 #   PROJECT
 function get-k8s-node-routes-count() {
-  local k8s_node_routes_count=$(gcloud alpha compute routes list \
+  local k8s_node_routes_count=$(gcloud compute routes list \
     --project=${PROJECT} --filter='description=k8s-node-route' \
     --format='value(name)' | wc -l)
   echo -n "${k8s_node_routes_count}"
@@ -50,7 +50,7 @@ function get-k8s-node-routes-count() {
 # Vars set:
 #  IP_ALIAS_SUBNETWORK
 function detect-k8s-subnetwork() {
-  local subnetwork_url=$(gcloud alpha compute instances describe \
+  local subnetwork_url=$(gcloud compute instances describe \
     ${KUBE_MASTER} --project=${PROJECT} --zone=${ZONE} \
     --format='value(networkInterfaces[0].subnetwork)')
   if [ -n ${subnetwork_url} ]; then
@@ -71,7 +71,8 @@ function set-allow-subnet-cidr-routes-overlap() {
   allow_subnet_cidr_routes_overlap=$(gcloud alpha compute networks subnets \
     describe ${IP_ALIAS_SUBNETWORK} --project=${PROJECT} --region=${REGION} \
     --format='value(allowSubnetCidrRoutesOverlap)')
-  if [ ${allow_subnet_cidr_routes_overlap,,} = $1 ]; then
+  local allow_overlap=$1
+  if [ ${allow_subnet_cidr_routes_overlap,,} = ${allow_overlap} ]; then
     echo "Subnet ${IP_ALIAS_SUBNETWORK}'s allowSubnetCidrRoutesOverlap is already set as $1"
     return
   fi

From 17c63ee7843fa546346cb43ca8f4f85a499699ba Mon Sep 17 00:00:00 2001
From: Jing Ai <jinga@google.com>
Date: Wed, 22 Nov 2017 18:16:00 -0800
Subject: [PATCH 04/16] Synced the changed made in PR #56260.

---
 pkg/controller/node/ipam/sync/sync.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/controller/node/ipam/sync/sync.go b/pkg/controller/node/ipam/sync/sync.go
index 737826547c2..71489b8bac5 100644
--- a/pkg/controller/node/ipam/sync/sync.go
+++ b/pkg/controller/node/ipam/sync/sync.go
@@ -244,7 +244,7 @@ func (op *updateOp) validateRange(ctx context.Context, sync *NodeSync, node *v1.
 // alias.
 func (op *updateOp) updateNodeFromAlias(ctx context.Context, sync *NodeSync, node *v1.Node, aliasRange *net.IPNet) error {
 	if sync.mode != SyncFromCloud {
-		fmt.Errorf("Detect mode %q while expect mode %q when syncing from cloud", sync.mode, SyncFromCloud)
+		glog.Warningf("Detect mode %q while expect to sync from cloud", sync.mode
 	}
 
 	glog.V(2).Infof("Updating node spec with alias range, node.PodCIDR = %v", aliasRange)
@@ -274,7 +274,7 @@ func (op *updateOp) updateNodeFromAlias(ctx context.Context, sync *NodeSync, nod
 // updateAliasFromNode updates the cloud alias given the node allocation.
 func (op *updateOp) updateAliasFromNode(ctx context.Context, sync *NodeSync, node *v1.Node) error {
 	if sync.mode != SyncFromCluster {
-		fmt.Errorf("Detect mode %q while expect mode %q when syncing from cluster", sync.mode, SyncFromCluster)
+		glog.Warningf("Detect mode %q while expect to sync from cluster", sync.mode)
 	}
 
 	_, aliasRange, err := net.ParseCIDR(node.Spec.PodCIDR)

From 1229b9a537137d367050c4a9bad0b5d5186f2a4f Mon Sep 17 00:00:00 2001
From: Jing Ai <jinga@google.com>
Date: Wed, 22 Nov 2017 18:29:59 -0800
Subject: [PATCH 05/16] Add a missing ) to glog.Waringf.

---
 pkg/controller/node/ipam/sync/sync.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/controller/node/ipam/sync/sync.go b/pkg/controller/node/ipam/sync/sync.go
index 71489b8bac5..eeff4a6d5fc 100644
--- a/pkg/controller/node/ipam/sync/sync.go
+++ b/pkg/controller/node/ipam/sync/sync.go
@@ -244,7 +244,7 @@ func (op *updateOp) validateRange(ctx context.Context, sync *NodeSync, node *v1.
 // alias.
 func (op *updateOp) updateNodeFromAlias(ctx context.Context, sync *NodeSync, node *v1.Node, aliasRange *net.IPNet) error {
 	if sync.mode != SyncFromCloud {
-		glog.Warningf("Detect mode %q while expect to sync from cloud", sync.mode
+		glog.Warningf("Detect mode %q while expect to sync from cloud", sync.mode)
 	}
 
 	glog.V(2).Infof("Updating node spec with alias range, node.PodCIDR = %v", aliasRange)

From a538c1e82ea3244ede613ef3e7a741d064d67eee Mon Sep 17 00:00:00 2001
From: Jing Ai <jinga@google.com>
Date: Wed, 22 Nov 2017 22:12:42 -0800
Subject: [PATCH 06/16] Retry on adding secondary ranges to a subnet.

---
 cluster/gce/upgrade-aliases.sh | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/cluster/gce/upgrade-aliases.sh b/cluster/gce/upgrade-aliases.sh
index 6157b2a966a..fc9c1902e96 100755
--- a/cluster/gce/upgrade-aliases.sh
+++ b/cluster/gce/upgrade-aliases.sh
@@ -109,9 +109,12 @@ function add-k8s-subnet-secondary-ranges() {
   fi
 
   echo "Adding secondary ranges: pods-default (${CLUSTER_IP_RANGE}), services-default (${SERVICE_CLUSTER_IP_RANGE})"
-  gcloud alpha compute networks subnets update ${IP_ALIAS_SUBNETWORK} \
+  until gcloud alpha compute networks subnets update ${IP_ALIAS_SUBNETWORK} \
     --project=${PROJECT} --region=${REGION} \
-    --add-secondary-ranges="pods-default=${CLUSTER_IP_RANGE},services-default=${SERVICE_CLUSTER_IP_RANGE}"
+    --add-secondary-ranges="pods-default=${CLUSTER_IP_RANGE},services-default=${SERVICE_CLUSTER_IP_RANGE}"; do
+    printf "."
+    sleep 1
+  done
 }
 
 # Delete all k8s node routes.

From 79e9a9e065ea0e9b44cc5d14fecfc75987bbb1ee Mon Sep 17 00:00:00 2001
From: Jing Ai <jinga@google.com>
Date: Fri, 24 Nov 2017 16:59:23 -0800
Subject: [PATCH 07/16] A couple of more changes. Specifically, a) make the
 same changes to master-helper.sh for gci as container-linux.sh does; b)
 revert changes to sync.go and sync_test.go.

---
 cluster/gce/gci/master-helper.sh           |  9 ++++++++-
 pkg/controller/node/ipam/sync/sync.go      |  8 ++++++--
 pkg/controller/node/ipam/sync/sync_test.go | 20 +++++++++++---------
 3 files changed, 25 insertions(+), 12 deletions(-)

diff --git a/cluster/gce/gci/master-helper.sh b/cluster/gce/gci/master-helper.sh
index def861d20bb..b93bd86ab8c 100755
--- a/cluster/gce/gci/master-helper.sh
+++ b/cluster/gce/gci/master-helper.sh
@@ -94,9 +94,16 @@ function create-master-instance-internal() {
     preemptible_master="--preemptible --maintenance-policy TERMINATE"
   fi
 
+  local enable_ip_aliases
+  if [[ "${NODE_IPAM_MODE:-}" == "CloudAllocator" ]]; then
+    enable_ip_aliases=true
+  else
+    enable_ip_aliases=false
+  fi
+
   local network=$(make-gcloud-network-argument \
     "${NETWORK_PROJECT}" "${REGION}" "${NETWORK}" "${SUBNETWORK:-}" \
-    "${address:-}" "${ENABLE_IP_ALIASES:-}" "${IP_ALIAS_SIZE:-}")
+    "${address:-}" "${enable_ip_aliases:-}" "${IP_ALIAS_SIZE:-}")
 
   local metadata="kube-env=${KUBE_TEMP}/master-kube-env.yaml"
   metadata="${metadata},user-data=${KUBE_ROOT}/cluster/gce/gci/master.yaml"
diff --git a/pkg/controller/node/ipam/sync/sync.go b/pkg/controller/node/ipam/sync/sync.go
index eeff4a6d5fc..4995f425543 100644
--- a/pkg/controller/node/ipam/sync/sync.go
+++ b/pkg/controller/node/ipam/sync/sync.go
@@ -244,7 +244,9 @@ func (op *updateOp) validateRange(ctx context.Context, sync *NodeSync, node *v1.
 // alias.
 func (op *updateOp) updateNodeFromAlias(ctx context.Context, sync *NodeSync, node *v1.Node, aliasRange *net.IPNet) error {
 	if sync.mode != SyncFromCloud {
-		glog.Warningf("Detect mode %q while expect to sync from cloud", sync.mode)
+		sync.kubeAPI.EmitNodeWarningEvent(node.Name, InvalidModeEvent,
+			"Cannot sync from cloud in mode %q", sync.mode)
+		return fmt.Errorf("cannot sync from cloud in mode %q", sync.mode)
 	}
 
 	glog.V(2).Infof("Updating node spec with alias range, node.PodCIDR = %v", aliasRange)
@@ -274,7 +276,9 @@ func (op *updateOp) updateNodeFromAlias(ctx context.Context, sync *NodeSync, nod
 // updateAliasFromNode updates the cloud alias given the node allocation.
 func (op *updateOp) updateAliasFromNode(ctx context.Context, sync *NodeSync, node *v1.Node) error {
 	if sync.mode != SyncFromCluster {
-		glog.Warningf("Detect mode %q while expect to sync from cluster", sync.mode)
+		sync.kubeAPI.EmitNodeWarningEvent(
+			node.Name, InvalidModeEvent, "Cannot sync to cloud in mode %q", sync.mode)
+		return fmt.Errorf("cannot sync to cloud in mode %q", sync.mode)
 	}
 
 	_, aliasRange, err := net.ParseCIDR(node.Spec.PodCIDR)
diff --git a/pkg/controller/node/ipam/sync/sync_test.go b/pkg/controller/node/ipam/sync/sync_test.go
index 94b9775ccbd..d3268480439 100644
--- a/pkg/controller/node/ipam/sync/sync_test.go
+++ b/pkg/controller/node/ipam/sync/sync_test.go
@@ -145,10 +145,11 @@ func TestNodeSyncUpdate(t *testing.T) {
 			events: []fakeEvent{{"node1", "CloudCIDRAllocatorMismatch"}},
 		},
 		{
-			desc:   "update alias from node",
-			mode:   SyncFromCloud,
-			node:   nodeWithCIDRRange,
-			events: nil,
+			desc:      "update alias from node",
+			mode:      SyncFromCloud,
+			node:      nodeWithCIDRRange,
+			events:    []fakeEvent{{"node1", "CloudCIDRAllocatorInvalidMode"}},
+			wantError: true,
 		},
 		{
 			desc: "update alias from node",
@@ -164,11 +165,12 @@ func TestNodeSyncUpdate(t *testing.T) {
 			// XXX/bowei -- validation
 		},
 		{
-			desc:   "update node from alias",
-			mode:   SyncFromCluster,
-			node:   nodeWithoutCIDRRange,
-			fake:   fakeAPIs{aliasRange: test.MustParseCIDR("10.1.2.3/16")},
-			events: nil,
+			desc:      "update node from alias",
+			mode:      SyncFromCluster,
+			node:      nodeWithoutCIDRRange,
+			fake:      fakeAPIs{aliasRange: test.MustParseCIDR("10.1.2.3/16")},
+			events:    []fakeEvent{{"node1", "CloudCIDRAllocatorInvalidMode"}},
+			wantError: true,
 		},
 		{
 			desc:      "allocate range",

From ba577bb5dd69c2d6839caa64b206b2536c8557db Mon Sep 17 00:00:00 2001
From: Jing Ai <jinga@google.com>
Date: Fri, 24 Nov 2017 17:50:33 -0800
Subject: [PATCH 08/16] Add -s (--slient) option to curl.

---
 cluster/gce/upgrade-aliases.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cluster/gce/upgrade-aliases.sh b/cluster/gce/upgrade-aliases.sh
index fc9c1902e96..8de5b0efe3a 100755
--- a/cluster/gce/upgrade-aliases.sh
+++ b/cluster/gce/upgrade-aliases.sh
@@ -84,7 +84,7 @@ function set-allow-subnet-cidr-routes-overlap() {
   local access_token=$(gcloud auth print-access-token)
   local request="{\"allowSubnetCidrRoutesOverlap\":$1, \"fingerprint\":\"${fingerprint}\"}"
   local subnetwork_url="${GCE_API_ENDPOINT}projects/${PROJECT}/regions/${REGION}/subnetworks/${IP_ALIAS_SUBNETWORK}"
-  until curl --header "Content-Type: application/json" --header "Authorization: Bearer ${access_token}" \
+  until curl -s --header "Content-Type: application/json" --header "Authorization: Bearer ${access_token}" \
     -X PATCH -d "${request}" "${subnetwork_url}" --output /dev/null; do
     printf "."
     sleep 1

From ce5d158f224263fa79c2bcb11050e91534eede12 Mon Sep 17 00:00:00 2001
From: Jing Ai <jinga@google.com>
Date: Fri, 24 Nov 2017 18:23:56 -0800
Subject: [PATCH 09/16] Merge PR #56260 again.

---
 pkg/controller/node/ipam/sync/sync.go      |  8 ++------
 pkg/controller/node/ipam/sync/sync_test.go | 18 +++++++-----------
 2 files changed, 9 insertions(+), 17 deletions(-)

diff --git a/pkg/controller/node/ipam/sync/sync.go b/pkg/controller/node/ipam/sync/sync.go
index 4995f425543..eeff4a6d5fc 100644
--- a/pkg/controller/node/ipam/sync/sync.go
+++ b/pkg/controller/node/ipam/sync/sync.go
@@ -244,9 +244,7 @@ func (op *updateOp) validateRange(ctx context.Context, sync *NodeSync, node *v1.
 // alias.
 func (op *updateOp) updateNodeFromAlias(ctx context.Context, sync *NodeSync, node *v1.Node, aliasRange *net.IPNet) error {
 	if sync.mode != SyncFromCloud {
-		sync.kubeAPI.EmitNodeWarningEvent(node.Name, InvalidModeEvent,
-			"Cannot sync from cloud in mode %q", sync.mode)
-		return fmt.Errorf("cannot sync from cloud in mode %q", sync.mode)
+		glog.Warningf("Detect mode %q while expect to sync from cloud", sync.mode)
 	}
 
 	glog.V(2).Infof("Updating node spec with alias range, node.PodCIDR = %v", aliasRange)
@@ -276,9 +274,7 @@ func (op *updateOp) updateNodeFromAlias(ctx context.Context, sync *NodeSync, nod
 // updateAliasFromNode updates the cloud alias given the node allocation.
 func (op *updateOp) updateAliasFromNode(ctx context.Context, sync *NodeSync, node *v1.Node) error {
 	if sync.mode != SyncFromCluster {
-		sync.kubeAPI.EmitNodeWarningEvent(
-			node.Name, InvalidModeEvent, "Cannot sync to cloud in mode %q", sync.mode)
-		return fmt.Errorf("cannot sync to cloud in mode %q", sync.mode)
+		glog.Warningf("Detect mode %q while expect to sync from cluster", sync.mode)
 	}
 
 	_, aliasRange, err := net.ParseCIDR(node.Spec.PodCIDR)
diff --git a/pkg/controller/node/ipam/sync/sync_test.go b/pkg/controller/node/ipam/sync/sync_test.go
index d3268480439..18e1335366e 100644
--- a/pkg/controller/node/ipam/sync/sync_test.go
+++ b/pkg/controller/node/ipam/sync/sync_test.go
@@ -145,11 +145,9 @@ func TestNodeSyncUpdate(t *testing.T) {
 			events: []fakeEvent{{"node1", "CloudCIDRAllocatorMismatch"}},
 		},
 		{
-			desc:      "update alias from node",
-			mode:      SyncFromCloud,
-			node:      nodeWithCIDRRange,
-			events:    []fakeEvent{{"node1", "CloudCIDRAllocatorInvalidMode"}},
-			wantError: true,
+			desc: "update alias from node",
+			mode: SyncFromCloud,
+			node: nodeWithCIDRRange,
 		},
 		{
 			desc: "update alias from node",
@@ -165,12 +163,10 @@ func TestNodeSyncUpdate(t *testing.T) {
 			// XXX/bowei -- validation
 		},
 		{
-			desc:      "update node from alias",
-			mode:      SyncFromCluster,
-			node:      nodeWithoutCIDRRange,
-			fake:      fakeAPIs{aliasRange: test.MustParseCIDR("10.1.2.3/16")},
-			events:    []fakeEvent{{"node1", "CloudCIDRAllocatorInvalidMode"}},
-			wantError: true,
+			desc: "update node from alias",
+			mode: SyncFromCluster,
+			node: nodeWithoutCIDRRange,
+			fake: fakeAPIs{aliasRange: test.MustParseCIDR("10.1.2.3/16")},
 		},
 		{
 			desc:      "allocate range",

From 492544b03fc8f40cd8bf67f0ae337ef1e02b6605 Mon Sep 17 00:00:00 2001
From: Jing Ai <jinga@google.com>
Date: Thu, 30 Nov 2017 15:38:54 -0800
Subject: [PATCH 10/16] Revert changes in sync.go and sync_test.go as the error
 it intends to resolve becomes transient.

---
 pkg/controller/node/ipam/sync/sync.go      |  8 ++++++--
 pkg/controller/node/ipam/sync/sync_test.go | 18 +++++++++++-------
 2 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/pkg/controller/node/ipam/sync/sync.go b/pkg/controller/node/ipam/sync/sync.go
index eeff4a6d5fc..4995f425543 100644
--- a/pkg/controller/node/ipam/sync/sync.go
+++ b/pkg/controller/node/ipam/sync/sync.go
@@ -244,7 +244,9 @@ func (op *updateOp) validateRange(ctx context.Context, sync *NodeSync, node *v1.
 // alias.
 func (op *updateOp) updateNodeFromAlias(ctx context.Context, sync *NodeSync, node *v1.Node, aliasRange *net.IPNet) error {
 	if sync.mode != SyncFromCloud {
-		glog.Warningf("Detect mode %q while expect to sync from cloud", sync.mode)
+		sync.kubeAPI.EmitNodeWarningEvent(node.Name, InvalidModeEvent,
+			"Cannot sync from cloud in mode %q", sync.mode)
+		return fmt.Errorf("cannot sync from cloud in mode %q", sync.mode)
 	}
 
 	glog.V(2).Infof("Updating node spec with alias range, node.PodCIDR = %v", aliasRange)
@@ -274,7 +276,9 @@ func (op *updateOp) updateNodeFromAlias(ctx context.Context, sync *NodeSync, nod
 // updateAliasFromNode updates the cloud alias given the node allocation.
 func (op *updateOp) updateAliasFromNode(ctx context.Context, sync *NodeSync, node *v1.Node) error {
 	if sync.mode != SyncFromCluster {
-		glog.Warningf("Detect mode %q while expect to sync from cluster", sync.mode)
+		sync.kubeAPI.EmitNodeWarningEvent(
+			node.Name, InvalidModeEvent, "Cannot sync to cloud in mode %q", sync.mode)
+		return fmt.Errorf("cannot sync to cloud in mode %q", sync.mode)
 	}
 
 	_, aliasRange, err := net.ParseCIDR(node.Spec.PodCIDR)
diff --git a/pkg/controller/node/ipam/sync/sync_test.go b/pkg/controller/node/ipam/sync/sync_test.go
index 18e1335366e..d3268480439 100644
--- a/pkg/controller/node/ipam/sync/sync_test.go
+++ b/pkg/controller/node/ipam/sync/sync_test.go
@@ -145,9 +145,11 @@ func TestNodeSyncUpdate(t *testing.T) {
 			events: []fakeEvent{{"node1", "CloudCIDRAllocatorMismatch"}},
 		},
 		{
-			desc: "update alias from node",
-			mode: SyncFromCloud,
-			node: nodeWithCIDRRange,
+			desc:      "update alias from node",
+			mode:      SyncFromCloud,
+			node:      nodeWithCIDRRange,
+			events:    []fakeEvent{{"node1", "CloudCIDRAllocatorInvalidMode"}},
+			wantError: true,
 		},
 		{
 			desc: "update alias from node",
@@ -163,10 +165,12 @@ func TestNodeSyncUpdate(t *testing.T) {
 			// XXX/bowei -- validation
 		},
 		{
-			desc: "update node from alias",
-			mode: SyncFromCluster,
-			node: nodeWithoutCIDRRange,
-			fake: fakeAPIs{aliasRange: test.MustParseCIDR("10.1.2.3/16")},
+			desc:      "update node from alias",
+			mode:      SyncFromCluster,
+			node:      nodeWithoutCIDRRange,
+			fake:      fakeAPIs{aliasRange: test.MustParseCIDR("10.1.2.3/16")},
+			events:    []fakeEvent{{"node1", "CloudCIDRAllocatorInvalidMode"}},
+			wantError: true,
 		},
 		{
 			desc:      "allocate range",

From beb6cf6fdb2a6994b47517ac9d6a28c0ee45df82 Mon Sep 17 00:00:00 2001
From: Jing Ai <jinga@google.com>
Date: Wed, 6 Dec 2017 14:21:24 -0800
Subject: [PATCH 11/16] Use gcloud beta instead of alpha for alias ops.

---
 cluster/gce/upgrade-aliases.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/cluster/gce/upgrade-aliases.sh b/cluster/gce/upgrade-aliases.sh
index 8de5b0efe3a..b5da11dc6ce 100755
--- a/cluster/gce/upgrade-aliases.sh
+++ b/cluster/gce/upgrade-aliases.sh
@@ -68,7 +68,7 @@ function detect-k8s-subnetwork() {
 #   REGION
 function set-allow-subnet-cidr-routes-overlap() {
   local allow_subnet_cidr_routes_overlap
-  allow_subnet_cidr_routes_overlap=$(gcloud alpha compute networks subnets \
+  allow_subnet_cidr_routes_overlap=$(gcloud beta compute networks subnets \
     describe ${IP_ALIAS_SUBNETWORK} --project=${PROJECT} --region=${REGION} \
     --format='value(allowSubnetCidrRoutesOverlap)')
   local allow_overlap=$1
@@ -78,7 +78,7 @@ function set-allow-subnet-cidr-routes-overlap() {
   fi
 
   echo "Setting subnet \"${IP_ALIAS_SUBNETWORK}\" allowSubnetCidrRoutesOverlap to $1"
-  local fingerprint=$(gcloud alpha compute networks subnets describe \
+  local fingerprint=$(gcloud beta compute networks subnets describe \
     ${IP_ALIAS_SUBNETWORK} --project=${PROJECT} --region=${REGION} \
     --format='value(fingerprint)')
   local access_token=$(gcloud auth print-access-token)
@@ -100,7 +100,7 @@ function set-allow-subnet-cidr-routes-overlap() {
 #   CLUSTER_IP_RANGE
 #   SERVICE_CLUSTER_IP_RANGE
 function add-k8s-subnet-secondary-ranges() {
-  local secondary_ranges=$(gcloud alpha compute networks subnets describe "${IP_ALIAS_SUBNETWORK}" \
+  local secondary_ranges=$(gcloud beta compute networks subnets describe "${IP_ALIAS_SUBNETWORK}" \
     --project="${PROJECT}" --region="${REGION}" \
     --format='value(secondaryIpRanges)')
   if [[ "${secondary_ranges}" =~ "pods-default" && "${secondary_ranges}" =~ "services-default" ]]; then
@@ -109,7 +109,7 @@ function add-k8s-subnet-secondary-ranges() {
   fi
 
   echo "Adding secondary ranges: pods-default (${CLUSTER_IP_RANGE}), services-default (${SERVICE_CLUSTER_IP_RANGE})"
-  until gcloud alpha compute networks subnets update ${IP_ALIAS_SUBNETWORK} \
+  until gcloud beta compute networks subnets update ${IP_ALIAS_SUBNETWORK} \
     --project=${PROJECT} --region=${REGION} \
     --add-secondary-ranges="pods-default=${CLUSTER_IP_RANGE},services-default=${SERVICE_CLUSTER_IP_RANGE}"; do
     printf "."
@@ -124,7 +124,7 @@ function add-k8s-subnet-secondary-ranges() {
 function delete-k8s-node-routes() {
   local -a routes
   local -r batch=200
-  routes=( $(gcloud alpha compute routes list \
+  routes=( $(gcloud compute routes list \
     --project=${PROJECT} --filter='description=k8s-node-route' \
     --format='value(name)') )
   while (( "${#routes[@]}" > 0 )); do

From 06378ce0f0653b807f0c35cb04e2f0b869bee221 Mon Sep 17 00:00:00 2001
From: Jing Ai <jinga@google.com>
Date: Thu, 18 Jan 2018 17:51:06 -0800
Subject: [PATCH 12/16] A couple of minior changes: a) fetch the subnetwork url
 from subnets describe command rather than compose it from env vars; b)
 explicit specify etcd version env vars before running upgrade.sh to avoid
 prompt.

---
 cluster/gce/upgrade-aliases.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/cluster/gce/upgrade-aliases.sh b/cluster/gce/upgrade-aliases.sh
index b5da11dc6ce..a58498de1eb 100755
--- a/cluster/gce/upgrade-aliases.sh
+++ b/cluster/gce/upgrade-aliases.sh
@@ -83,7 +83,9 @@ function set-allow-subnet-cidr-routes-overlap() {
     --format='value(fingerprint)')
   local access_token=$(gcloud auth print-access-token)
   local request="{\"allowSubnetCidrRoutesOverlap\":$1, \"fingerprint\":\"${fingerprint}\"}"
-  local subnetwork_url="${GCE_API_ENDPOINT}projects/${PROJECT}/regions/${REGION}/subnetworks/${IP_ALIAS_SUBNETWORK}"
+  local subnetwork_url=$(gcloud beta compute networks subnets describe \
+    ${IP_ALIAS_SUBNETWORK} --project=${PROJECT} --region=${REGION} \
+    --format='value(selfLink)')
   until curl -s --header "Content-Type: application/json" --header "Authorization: Bearer ${access_token}" \
     -X PATCH -d "${request}" "${subnetwork_url}" --output /dev/null; do
     printf "."
@@ -161,6 +163,8 @@ export KUBE_GCE_ENABLE_IP_ALIASES=true
 export SECONDARY_RANGE_NAME="pods-default"
 export STORAGE_BACKEND="etcd3"
 export STORAGE_MEDIA_TYPE="application/vnd.kubernetes.protobuf"
+export ETCD_IMAGE=3.0.17
+export ETCD_VERSION=3.0.17
 
 # Upgrade master with updated kube envs
 ${KUBE_ROOT}/cluster/gce/upgrade.sh -M -l

From a4549a706717813f3d1888133ae0e4b1a41ad65f Mon Sep 17 00:00:00 2001
From: Jing Ai <jinga@google.com>
Date: Thu, 18 Jan 2018 17:58:39 -0800
Subject: [PATCH 13/16] Minior changes on comments.

---
 cluster/gce/upgrade-aliases.sh | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/cluster/gce/upgrade-aliases.sh b/cluster/gce/upgrade-aliases.sh
index a58498de1eb..841ddb8e6ef 100755
--- a/cluster/gce/upgrade-aliases.sh
+++ b/cluster/gce/upgrade-aliases.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2017 The Kubernetes Authors.
+# Copyright 2018 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# !!!EXPERIMENTAL!!! Upgrade a k8s cluster from routes to IP aliases for
+# !!!EXPERIMENTAL!!! Upgrade a K8s cluster from routes to IP aliases for
 # node connectivity on GCE. This is only for migration.
 
 set -o errexit
@@ -30,7 +30,7 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
 source "${KUBE_ROOT}/hack/lib/util.sh"
 source "${KUBE_ROOT}/cluster/kube-util.sh"
 
-# Print the number of routes used for k8s cluster node connectivity.
+# Print the number of routes used for K8s cluster node connectivity.
 #
 # Assumed vars:
 #   PROJECT
@@ -41,7 +41,7 @@ function get-k8s-node-routes-count() {
   echo -n "${k8s_node_routes_count}"
 }
 
-# Detect the subnetwork where the k8s cluster resides.
+# Detect the subnetwork where the K8s cluster resides.
 #
 # Assumed vars:
 #  KUBE_MASTER
@@ -93,7 +93,7 @@ function set-allow-subnet-cidr-routes-overlap() {
   done
 }
 
-# Add secondary ranges to k8s subnet.
+# Add secondary ranges to K8s subnet.
 #
 # Assumed vars:
 #   IP_ALIAS_SUBNETWORK
@@ -119,7 +119,7 @@ function add-k8s-subnet-secondary-ranges() {
   done
 }
 
-# Delete all k8s node routes.
+# Delete all K8s node routes.
 #
 # Assumed vars:
 #   PROJECT
@@ -156,7 +156,7 @@ echo "k8s cluster sits on subnetwork \"${IP_ALIAS_SUBNETWORK}\""
 set-allow-subnet-cidr-routes-overlap true
 add-k8s-subnet-secondary-ranges
 
-echo "Changing k8s master envs and restarting..."
+echo "Changing K8s master envs and restarting..."
 export KUBE_GCE_IP_ALIAS_SUBNETWORK=${IP_ALIAS_SUBNETWORK}
 export KUBE_GCE_NODE_IPAM_MODE="IPAMFromCluster"
 export KUBE_GCE_ENABLE_IP_ALIASES=true

From d3fb77e237658c9553fa5b9ee77ec07379673ca0 Mon Sep 17 00:00:00 2001
From: Jing Ai <jinga@google.com>
Date: Thu, 18 Jan 2018 18:00:53 -0800
Subject: [PATCH 14/16] Minior changes on comments.

---
 cluster/gce/upgrade-aliases.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cluster/gce/upgrade-aliases.sh b/cluster/gce/upgrade-aliases.sh
index 841ddb8e6ef..2fd8449e44c 100755
--- a/cluster/gce/upgrade-aliases.sh
+++ b/cluster/gce/upgrade-aliases.sh
@@ -144,7 +144,7 @@ if [[ "${k8s_node_routes_count}" -eq 0 ]]; then
   echo "No k8s node routes found and IP alias should already be enabled. Exiting..."
   exit 0
 fi
-echo "Found ${k8s_node_routes_count} k8s node routes. Proceeding to upgrade them to IP aliases based connectivity..."
+echo "Found ${k8s_node_routes_count} K8s node routes. Proceeding to upgrade them to IP aliases based connectivity..."
 
 detect-k8s-subnetwork
 if [ -z ${IP_ALIAS_SUBNETWORK} ]; then
@@ -167,7 +167,7 @@ export ETCD_IMAGE=3.0.17
 export ETCD_VERSION=3.0.17
 
 # Upgrade master with updated kube envs
-${KUBE_ROOT}/cluster/gce/upgrade.sh -M -l
+# ${KUBE_ROOT}/cluster/gce/upgrade.sh -M -l
 
 delete-k8s-node-routes
 set-allow-subnet-cidr-routes-overlap false

From 2225be249248217c807e0e3939f839c8718ed9bc Mon Sep 17 00:00:00 2001
From: Jing Ai <jinga@google.com>
Date: Thu, 18 Jan 2018 20:26:32 -0800
Subject: [PATCH 15/16] Uncomment the call to upgrade.sh

---
 cluster/gce/upgrade-aliases.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cluster/gce/upgrade-aliases.sh b/cluster/gce/upgrade-aliases.sh
index 2fd8449e44c..6d1fd9b139d 100755
--- a/cluster/gce/upgrade-aliases.sh
+++ b/cluster/gce/upgrade-aliases.sh
@@ -167,7 +167,7 @@ export ETCD_IMAGE=3.0.17
 export ETCD_VERSION=3.0.17
 
 # Upgrade master with updated kube envs
-# ${KUBE_ROOT}/cluster/gce/upgrade.sh -M -l
+${KUBE_ROOT}/cluster/gce/upgrade.sh -M -l
 
 delete-k8s-node-routes
 set-allow-subnet-cidr-routes-overlap false

From 4a627eb3a3b6aee89f66ec31a62ae8ee826aa41c Mon Sep 17 00:00:00 2001
From: Jing Ai <jinga@google.com>
Date: Fri, 19 Jan 2018 15:51:50 -0800
Subject: [PATCH 16/16] A couple of more changes: 1) revert the changes on
 assigning subnetwork_url from selfLink as it may break if using an overrided
 api endpoint; 2) update etcd version to the latest.

---
 cluster/gce/upgrade-aliases.sh | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/cluster/gce/upgrade-aliases.sh b/cluster/gce/upgrade-aliases.sh
index 6d1fd9b139d..d7ff3c69aa7 100755
--- a/cluster/gce/upgrade-aliases.sh
+++ b/cluster/gce/upgrade-aliases.sh
@@ -83,9 +83,7 @@ function set-allow-subnet-cidr-routes-overlap() {
     --format='value(fingerprint)')
   local access_token=$(gcloud auth print-access-token)
   local request="{\"allowSubnetCidrRoutesOverlap\":$1, \"fingerprint\":\"${fingerprint}\"}"
-  local subnetwork_url=$(gcloud beta compute networks subnets describe \
-    ${IP_ALIAS_SUBNETWORK} --project=${PROJECT} --region=${REGION} \
-    --format='value(selfLink)')
+  local subnetwork_url="${GCE_API_ENDPOINT}projects/${PROJECT}/regions/${REGION}/subnetworks/${IP_ALIAS_SUBNETWORK}"
   until curl -s --header "Content-Type: application/json" --header "Authorization: Bearer ${access_token}" \
     -X PATCH -d "${request}" "${subnetwork_url}" --output /dev/null; do
     printf "."
@@ -163,8 +161,8 @@ export KUBE_GCE_ENABLE_IP_ALIASES=true
 export SECONDARY_RANGE_NAME="pods-default"
 export STORAGE_BACKEND="etcd3"
 export STORAGE_MEDIA_TYPE="application/vnd.kubernetes.protobuf"
-export ETCD_IMAGE=3.0.17
-export ETCD_VERSION=3.0.17
+export ETCD_IMAGE=3.1.10
+export ETCD_VERSION=3.1.10
 
 # Upgrade master with updated kube envs
 ${KUBE_ROOT}/cluster/gce/upgrade.sh -M -l