mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-24 12:15:52 +00:00
Merge pull request #56132 from jingax10/upgrade_aliases_branch
Automatic merge from submit-queue (batch tested with PRs 58412, 56132, 58506, 58542, 58394). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. Implement upgrade-aliases.sh to migrate a route-based k8s cluster to use IP aliases in GCE. **What this PR does / why we need it**: This PR implements upgrade-aliases.sh, which is used to migrate a route-based k8s cluster to use IP aliases in GCE. **Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*: Fixes # **Special notes for your reviewer**: **Release note**: ```release-note "NONE" ```
This commit is contained in:
Kubernetes Submit Queue
GitHub
commit
854adeb2d7
@ -267,14 +267,19 @@ ENABLE_RESCHEDULER="${KUBE_ENABLE_RESCHEDULER:-true}"
|
||||
# IP_ALIAS_SUBNETWORK is the subnetwork to allocate from. If empty, a
|
||||
# new subnetwork will be created for the cluster.
|
||||
ENABLE_IP_ALIASES=${KUBE_GCE_ENABLE_IP_ALIASES:-false}
|
||||
NODE_IPAM_MODE=${KUBE_GCE_NODE_IPAM_MODE:-RangeAllocator}
|
||||
if [ ${ENABLE_IP_ALIASES} = true ]; then
|
||||
# Size of ranges allocated to each node. Currently supports only /32 and /24.
|
||||
IP_ALIAS_SIZE=${KUBE_GCE_IP_ALIAS_SIZE:-/24}
|
||||
IP_ALIAS_SUBNETWORK=${KUBE_GCE_IP_ALIAS_SUBNETWORK:-${INSTANCE_PREFIX}-subnet-default}
|
||||
# Reserve the services IP space to avoid being allocated for other GCP resources.
|
||||
SERVICE_CLUSTER_IP_SUBNETWORK=${KUBE_GCE_SERVICE_CLUSTER_IP_SUBNETWORK:-${INSTANCE_PREFIX}-subnet-services}
|
||||
NODE_IPAM_MODE=${KUBE_GCE_NODE_IPAM_MODE:-CloudAllocator}
|
||||
SECONDARY_RANGE_NAME=${SECONDARY_RANGE_NAME:-}
|
||||
# Add to the provider custom variables.
|
||||
PROVIDER_VARS="${PROVIDER_VARS:-} ENABLE_IP_ALIASES"
|
||||
PROVIDER_VARS="${PROVIDER_VARS:-} NODE_IPAM_MODE"
|
||||
PROVIDER_VARS="${PROVIDER_VARS:-} SECONDARY_RANGE_NAME"
|
||||
fi
|
||||
|
||||
# Enable GCE Alpha features.
|
||||
|
||||
@ -1835,7 +1835,7 @@ function start-kube-controller-manager {
|
||||
params+=" --terminated-pod-gc-threshold=${TERMINATED_POD_GC_THRESHOLD}"
|
||||
fi
|
||||
if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then
|
||||
params+=" --cidr-allocator-type=CloudAllocator"
|
||||
params+=" --cidr-allocator-type=${NODE_IPAM_MODE}"
|
||||
params+=" --configure-cloud-routes=false"
|
||||
fi
|
||||
if [[ -n "${FEATURE_GATES:-}" ]]; then
|
||||
|
||||
@ -94,9 +94,16 @@ function create-master-instance-internal() {
|
||||
preemptible_master="--preemptible --maintenance-policy TERMINATE"
|
||||
fi
|
||||
|
||||
local enable_ip_aliases
|
||||
if [[ "${NODE_IPAM_MODE:-}" == "CloudAllocator" ]]; then
|
||||
enable_ip_aliases=true
|
||||
else
|
||||
enable_ip_aliases=false
|
||||
fi
|
||||
|
||||
local network=$(make-gcloud-network-argument \
|
||||
"${NETWORK_PROJECT}" "${REGION}" "${NETWORK}" "${SUBNETWORK:-}" \
|
||||
"${address:-}" "${ENABLE_IP_ALIASES:-}" "${IP_ALIAS_SIZE:-}")
|
||||
"${address:-}" "${enable_ip_aliases:-}" "${IP_ALIAS_SIZE:-}")
|
||||
|
||||
local metadata="kube-env=${KUBE_TEMP}/master-kube-env.yaml"
|
||||
metadata="${metadata},user-data=${KUBE_ROOT}/cluster/gce/gci/master.yaml"
|
||||
|
||||
171
cluster/gce/upgrade-aliases.sh
Executable file
171
cluster/gce/upgrade-aliases.sh
Executable file
@ -0,0 +1,171 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Copyright 2018 The Kubernetes Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# !!!EXPERIMENTAL!!! Upgrade a K8s cluster from routes to IP aliases for
|
||||
# node connectivity on GCE. This is only for migration.
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
if [[ "${KUBERNETES_PROVIDER:-gce}" != "gce" ]]; then
|
||||
echo "ERR: KUBERNETES_PROVIDER must be gce" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
|
||||
source "${KUBE_ROOT}/hack/lib/util.sh"
|
||||
source "${KUBE_ROOT}/cluster/kube-util.sh"
|
||||
|
||||
# Print the number of routes used for K8s cluster node connectivity.
|
||||
#
|
||||
# Assumed vars:
|
||||
# PROJECT
|
||||
function get-k8s-node-routes-count() {
|
||||
local k8s_node_routes_count=$(gcloud compute routes list \
|
||||
--project=${PROJECT} --filter='description=k8s-node-route' \
|
||||
--format='value(name)' | wc -l)
|
||||
echo -n "${k8s_node_routes_count}"
|
||||
}
|
||||
|
||||
# Detect the subnetwork where the K8s cluster resides.
|
||||
#
|
||||
# Assumed vars:
|
||||
# KUBE_MASTER
|
||||
# PROJECT
|
||||
# ZONE
|
||||
# Vars set:
|
||||
# IP_ALIAS_SUBNETWORK
|
||||
function detect-k8s-subnetwork() {
|
||||
local subnetwork_url=$(gcloud compute instances describe \
|
||||
${KUBE_MASTER} --project=${PROJECT} --zone=${ZONE} \
|
||||
--format='value(networkInterfaces[0].subnetwork)')
|
||||
if [ -n ${subnetwork_url} ]; then
|
||||
IP_ALIAS_SUBNETWORK=$(echo ${subnetwork_url##*/})
|
||||
fi
|
||||
}
|
||||
|
||||
# Set IP_ALIAS_SUBNETWORK's allowSubnetCidrRoutesOverlap to a boolean value.
|
||||
# $1: true or false for the desired allowSubnetCidrRoutesOverlap.
|
||||
#
|
||||
# Assumed vars:
|
||||
# IP_ALIAS_SUBNETWORK
|
||||
# GCE_API_ENDPOINT
|
||||
# PROJECT
|
||||
# REGION
|
||||
function set-allow-subnet-cidr-routes-overlap() {
|
||||
local allow_subnet_cidr_routes_overlap
|
||||
allow_subnet_cidr_routes_overlap=$(gcloud beta compute networks subnets \
|
||||
describe ${IP_ALIAS_SUBNETWORK} --project=${PROJECT} --region=${REGION} \
|
||||
--format='value(allowSubnetCidrRoutesOverlap)')
|
||||
local allow_overlap=$1
|
||||
if [ ${allow_subnet_cidr_routes_overlap,,} = ${allow_overlap} ]; then
|
||||
echo "Subnet ${IP_ALIAS_SUBNETWORK}'s allowSubnetCidrRoutesOverlap is already set as $1"
|
||||
return
|
||||
fi
|
||||
|
||||
echo "Setting subnet \"${IP_ALIAS_SUBNETWORK}\" allowSubnetCidrRoutesOverlap to $1"
|
||||
local fingerprint=$(gcloud beta compute networks subnets describe \
|
||||
${IP_ALIAS_SUBNETWORK} --project=${PROJECT} --region=${REGION} \
|
||||
--format='value(fingerprint)')
|
||||
local access_token=$(gcloud auth print-access-token)
|
||||
local request="{\"allowSubnetCidrRoutesOverlap\":$1, \"fingerprint\":\"${fingerprint}\"}"
|
||||
local subnetwork_url="${GCE_API_ENDPOINT}projects/${PROJECT}/regions/${REGION}/subnetworks/${IP_ALIAS_SUBNETWORK}"
|
||||
until curl -s --header "Content-Type: application/json" --header "Authorization: Bearer ${access_token}" \
|
||||
-X PATCH -d "${request}" "${subnetwork_url}" --output /dev/null; do
|
||||
printf "."
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
# Add secondary ranges to K8s subnet.
|
||||
#
|
||||
# Assumed vars:
|
||||
# IP_ALIAS_SUBNETWORK
|
||||
# PROJECT
|
||||
# REGION
|
||||
# CLUSTER_IP_RANGE
|
||||
# SERVICE_CLUSTER_IP_RANGE
|
||||
function add-k8s-subnet-secondary-ranges() {
|
||||
local secondary_ranges=$(gcloud beta compute networks subnets describe "${IP_ALIAS_SUBNETWORK}" \
|
||||
--project="${PROJECT}" --region="${REGION}" \
|
||||
--format='value(secondaryIpRanges)')
|
||||
if [[ "${secondary_ranges}" =~ "pods-default" && "${secondary_ranges}" =~ "services-default" ]]; then
|
||||
echo "${secondary_ranges} already contains both pods-default and services-default secondary ranges"
|
||||
return
|
||||
fi
|
||||
|
||||
echo "Adding secondary ranges: pods-default (${CLUSTER_IP_RANGE}), services-default (${SERVICE_CLUSTER_IP_RANGE})"
|
||||
until gcloud beta compute networks subnets update ${IP_ALIAS_SUBNETWORK} \
|
||||
--project=${PROJECT} --region=${REGION} \
|
||||
--add-secondary-ranges="pods-default=${CLUSTER_IP_RANGE},services-default=${SERVICE_CLUSTER_IP_RANGE}"; do
|
||||
printf "."
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
# Delete all K8s node routes.
|
||||
#
|
||||
# Assumed vars:
|
||||
# PROJECT
|
||||
function delete-k8s-node-routes() {
|
||||
local -a routes
|
||||
local -r batch=200
|
||||
routes=( $(gcloud compute routes list \
|
||||
--project=${PROJECT} --filter='description=k8s-node-route' \
|
||||
--format='value(name)') )
|
||||
while (( "${#routes[@]}" > 0 )); do
|
||||
echo Deleting k8s node routes "${routes[*]::${batch}}"
|
||||
gcloud compute routes delete --project "${PROJECT}" --quiet "${routes[@]::${batch}}"
|
||||
routes=( "${routes[@]:${batch}}" )
|
||||
done
|
||||
}
|
||||
|
||||
detect-project
|
||||
detect-master
|
||||
|
||||
k8s_node_routes_count=$(get-k8s-node-routes-count)
|
||||
if [[ "${k8s_node_routes_count}" -eq 0 ]]; then
|
||||
echo "No k8s node routes found and IP alias should already be enabled. Exiting..."
|
||||
exit 0
|
||||
fi
|
||||
echo "Found ${k8s_node_routes_count} K8s node routes. Proceeding to upgrade them to IP aliases based connectivity..."
|
||||
|
||||
detect-k8s-subnetwork
|
||||
if [ -z ${IP_ALIAS_SUBNETWORK} ]; then
|
||||
echo "No k8s cluster subnetwork found. Exiting..."
|
||||
exit 1
|
||||
fi
|
||||
echo "k8s cluster sits on subnetwork \"${IP_ALIAS_SUBNETWORK}\""
|
||||
|
||||
set-allow-subnet-cidr-routes-overlap true
|
||||
add-k8s-subnet-secondary-ranges
|
||||
|
||||
echo "Changing K8s master envs and restarting..."
|
||||
export KUBE_GCE_IP_ALIAS_SUBNETWORK=${IP_ALIAS_SUBNETWORK}
|
||||
export KUBE_GCE_NODE_IPAM_MODE="IPAMFromCluster"
|
||||
export KUBE_GCE_ENABLE_IP_ALIASES=true
|
||||
export SECONDARY_RANGE_NAME="pods-default"
|
||||
export STORAGE_BACKEND="etcd3"
|
||||
export STORAGE_MEDIA_TYPE="application/vnd.kubernetes.protobuf"
|
||||
export ETCD_IMAGE=3.1.10
|
||||
export ETCD_VERSION=3.1.10
|
||||
|
||||
# Upgrade master with updated kube envs
|
||||
${KUBE_ROOT}/cluster/gce/upgrade.sh -M -l
|
||||
|
||||
delete-k8s-node-routes
|
||||
set-allow-subnet-cidr-routes-overlap false
|
||||
Loading…
Reference in New Issue
Block a user