mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-24 12:15:52 +00:00
Merge pull request #29295 from fgrzadkowski/ha_master
Automatic merge from submit-queue Add load balancer in front of apiserver in HA master setup The first commit is just https://github.com/kubernetes/kubernetes/pull/29201 and has been already LGTMed. Second commit has some small fixes: 1. Precompute REGION variable in config 2. Add timeout for waiting for loadbalancer 3. Fix kube-down so that it doesn't delete some resources if there are still masters/nodes in other zones Second commit also fixes bug in https://github.com/kubernetes/kubernetes/pull/29201 where variable `REGION` was unset in `kube-down` when deleting master IP. The bug caused leaking of IP addresses. https://github.com/kubernetes/kubernetes/issues/21124 @davidopp @jszczepkowski @wojtek-t @mikedanese
This commit is contained in:
k8s-merge-robot
GitHub
commit
20246986df
@ -21,6 +21,7 @@ source "${KUBE_ROOT}/cluster/gce/config-common.sh"
|
||||
|
||||
GCLOUD=gcloud
|
||||
ZONE=${KUBE_GCE_ZONE:-us-central1-b}
|
||||
REGION=${ZONE%-*}
|
||||
RELEASE_REGION_FALLBACK=${RELEASE_REGION_FALLBACK:-false}
|
||||
REGIONAL_KUBE_ADDONS=${REGIONAL_KUBE_ADDONS:-true}
|
||||
NODE_SIZE=${NODE_SIZE:-n1-standard-2}
|
||||
|
||||
@ -21,6 +21,7 @@ source "${KUBE_ROOT}/cluster/gce/config-common.sh"
|
||||
|
||||
GCLOUD=gcloud
|
||||
ZONE=${KUBE_GCE_ZONE:-us-central1-b}
|
||||
REGION=${ZONE%-*}
|
||||
RELEASE_REGION_FALLBACK=${RELEASE_REGION_FALLBACK:-false}
|
||||
REGIONAL_KUBE_ADDONS=${REGIONAL_KUBE_ADDONS:-true}
|
||||
NODE_SIZE=${NODE_SIZE:-n1-standard-2}
|
||||
|
||||
@ -379,6 +379,7 @@ function detect-nodes () {
|
||||
# Assumed vars:
|
||||
# MASTER_NAME
|
||||
# ZONE
|
||||
# REGION
|
||||
# Vars set:
|
||||
# KUBE_MASTER
|
||||
# KUBE_MASTER_IP
|
||||
@ -386,8 +387,8 @@ function detect-master () {
|
||||
detect-project
|
||||
KUBE_MASTER=${MASTER_NAME}
|
||||
if [[ -z "${KUBE_MASTER_IP-}" ]]; then
|
||||
KUBE_MASTER_IP=$(gcloud compute instances describe --project "${PROJECT}" --zone "${ZONE}" \
|
||||
"${MASTER_NAME}" --format='value(networkInterfaces[0].accessConfigs[0].natIP)')
|
||||
KUBE_MASTER_IP=$(gcloud compute addresses describe "${MASTER_NAME}-ip" \
|
||||
--project "${PROJECT}" --region "${REGION}" -q --format='value(address)')
|
||||
fi
|
||||
if [[ -z "${KUBE_MASTER_IP-}" ]]; then
|
||||
echo "Could not detect Kubernetes master node. Make sure you've launched a cluster with 'kube-up.sh'" >&2
|
||||
@ -609,6 +610,9 @@ function kube-up {
|
||||
if [[ ${KUBE_USE_EXISTING_MASTER:-} == "true" ]]; then
|
||||
parse-master-env
|
||||
create-nodes
|
||||
elif [[ ${KUBE_REPLICATE_EXISTING_MASTER:-} == "true" ]]; then
|
||||
create-loadbalancer
|
||||
# TODO: Add logic for copying an existing master.
|
||||
else
|
||||
check-existing
|
||||
create-network
|
||||
@ -723,10 +727,7 @@ function create-master() {
|
||||
KUBE_PROXY_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
|
||||
|
||||
# Reserve the master's IP so that it can later be transferred to another VM
|
||||
# without disrupting the kubelets. IPs are associated with regions, not zones,
|
||||
# so extract the region name, which is the same as the zone but with the final
|
||||
# dash and characters trailing the dash removed.
|
||||
local REGION=${ZONE%-*}
|
||||
# without disrupting the kubelets.
|
||||
create-static-ip "${MASTER_NAME}-ip" "${REGION}"
|
||||
MASTER_RESERVED_IP=$(gcloud compute addresses describe "${MASTER_NAME}-ip" \
|
||||
--project "${PROJECT}" --region "${REGION}" -q --format='value(address)')
|
||||
@ -739,6 +740,83 @@ function create-master() {
|
||||
create-master-instance "${MASTER_RESERVED_IP}" &
|
||||
}
|
||||
|
||||
# Detaches old and ataches new external IP to a VM.
|
||||
#
|
||||
# Arguments:
|
||||
# $1 - VM name
|
||||
# $2 - VM zone
|
||||
# $3 - external static IP; if empty will use an ephemeral IP address.
|
||||
function attach-external-ip() {
|
||||
local NAME=${1}
|
||||
local ZONE=${2}
|
||||
local IP_ADDR=${3:-}
|
||||
local ACCESS_CONFIG_NAME=$(gcloud compute instances describe "${NAME}" \
|
||||
--project "${PROJECT}" --zone "${ZONE}" \
|
||||
--format="value(networkInterfaces[0].accessConfigs[0].name)")
|
||||
gcloud compute instances delete-access-config "${NAME}" \
|
||||
--project "${PROJECT}" --zone "${ZONE}" \
|
||||
--access-config-name "${ACCESS_CONFIG_NAME}"
|
||||
if [[ -z ${IP_ADDR} ]]; then
|
||||
gcloud compute instances add-access-config "${NAME}" \
|
||||
--project "${PROJECT}" --zone "${ZONE}" \
|
||||
--access-config-name "${ACCESS_CONFIG_NAME}"
|
||||
else
|
||||
gcloud compute instances add-access-config "${NAME}" \
|
||||
--project "${PROJECT}" --zone "${ZONE}" \
|
||||
--access-config-name "${ACCESS_CONFIG_NAME}" \
|
||||
--address "${IP_ADDR}"
|
||||
fi
|
||||
}
|
||||
|
||||
# Creates load balancer in front of apiserver if it doesn't exists already. Assumes there's only one
|
||||
# existing master replica.
|
||||
#
|
||||
# Assumes:
|
||||
# PROJECT
|
||||
# MASTER_NAME
|
||||
# ZONE
|
||||
# REGION
|
||||
function create-loadbalancer() {
|
||||
detect-master
|
||||
|
||||
# Step 0: Return early if LB is already configured.
|
||||
if gcloud compute forwarding-rules describe ${MASTER_NAME} \
|
||||
--project "${PROJECT}" --region ${REGION} > /dev/null 2>&1; then
|
||||
echo "Load balancer already exists"
|
||||
return
|
||||
fi
|
||||
local EXISTING_MASTER_ZONE=$(gcloud compute instances list "${MASTER_NAME}" \
|
||||
--project "${PROJECT}" --format="value(zone)")
|
||||
echo "Creating load balancer in front of an already existing master in ${EXISTING_MASTER_ZONE}"
|
||||
|
||||
# Step 1: Detach master IP address and attach ephemeral address to the existing master
|
||||
attach-external-ip ${MASTER_NAME} ${EXISTING_MASTER_ZONE}
|
||||
|
||||
# Step 2: Create target pool.
|
||||
gcloud compute target-pools create "${MASTER_NAME}" --region "${REGION}"
|
||||
# TODO: We should also add master instances with suffixes
|
||||
gcloud compute target-pools add-instances ${MASTER_NAME} --instances ${MASTER_NAME} --zone ${EXISTING_MASTER_ZONE}
|
||||
|
||||
# Step 3: Create forwarding rule.
|
||||
# TODO: This step can take up to 20 min. We need to speed this up...
|
||||
gcloud compute forwarding-rules create ${MASTER_NAME} \
|
||||
--project "${PROJECT}" --region ${REGION} \
|
||||
--target-pool ${MASTER_NAME} --address=${KUBE_MASTER_IP} --ports=443
|
||||
|
||||
echo -n "Waiting for the load balancer configuration to propagate..."
|
||||
local counter=0
|
||||
until $(curl -k -m1 https://${KUBE_MASTER_IP} &> /dev/null); do
|
||||
counter=$((counter+1))
|
||||
echo -n .
|
||||
if [[ ${counter} -ge 1800 ]]; then
|
||||
echo -e "${color_red}TIMEOUT${color_norm}" >&2
|
||||
echo -e "${color_red}Load balancer failed to initialize within ${counter} seconds.${color_norm}" >&2
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
echo "DONE"
|
||||
}
|
||||
|
||||
function create-nodes-firewall() {
|
||||
# Create a single firewall rule for all minions.
|
||||
create-firewall-rule "${NODE_TAG}-all" "${CLUSTER_IP_RANGE}" "${NODE_TAG}" &
|
||||
@ -1029,6 +1107,58 @@ function kube-down {
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check if this are any remaining master replicas.
|
||||
local REMAINING_MASTER_COUNT=$(gcloud compute instances list \
|
||||
--project "${PROJECT}" \
|
||||
--regexp "${MASTER_NAME}(-...)?" \
|
||||
--format "value(zone)" | wc -l)
|
||||
|
||||
# In the replicated scenario, if there's only a single master left, we should also delete load balancer in front of it.
|
||||
if [[ "${REMAINING_MASTER_COUNT}" == "1" ]]; then
|
||||
if gcloud compute forwarding-rules describe "${MASTER_NAME}" --region "${REGION}" --project "${PROJECT}" &>/dev/null; then
|
||||
detect-master
|
||||
local EXISTING_MASTER_ZONE=$(gcloud compute instances list "${MASTER_NAME}" \
|
||||
--project "${PROJECT}" --format="value(zone)")
|
||||
gcloud compute forwarding-rules delete \
|
||||
--project "${PROJECT}" \
|
||||
--region "${REGION}" \
|
||||
--quiet \
|
||||
"${MASTER_NAME}"
|
||||
attach-external-ip "${MASTER_NAME}" "${EXISTING_MASTER_ZONE}" "${KUBE_MASTER_IP}"
|
||||
gcloud compute target-pools delete \
|
||||
--project "${PROJECT}" \
|
||||
--region "${REGION}" \
|
||||
--quiet \
|
||||
"${MASTER_NAME}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# If there are no more remaining master replicas, we should delete all remaining network resources.
|
||||
if [[ "${REMAINING_MASTER_COUNT}" == "0" ]]; then
|
||||
# Delete firewall rule for the master.
|
||||
if gcloud compute firewall-rules describe --project "${PROJECT}" "${MASTER_NAME}-https" &>/dev/null; then
|
||||
gcloud compute firewall-rules delete \
|
||||
--project "${PROJECT}" \
|
||||
--quiet \
|
||||
"${MASTER_NAME}-https"
|
||||
fi
|
||||
# Delete the master's reserved IP
|
||||
if gcloud compute addresses describe "${MASTER_NAME}-ip" --region "${REGION}" --project "${PROJECT}" &>/dev/null; then
|
||||
gcloud compute addresses delete \
|
||||
--project "${PROJECT}" \
|
||||
--region "${REGION}" \
|
||||
--quiet \
|
||||
"${MASTER_NAME}-ip"
|
||||
fi
|
||||
# Delete firewall rule for minions.
|
||||
if gcloud compute firewall-rules describe --project "${PROJECT}" "${NODE_TAG}-all" &>/dev/null; then
|
||||
gcloud compute firewall-rules delete \
|
||||
--project "${PROJECT}" \
|
||||
--quiet \
|
||||
"${NODE_TAG}-all"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Find out what minions are running.
|
||||
local -a minions
|
||||
minions=( $(gcloud compute instances list \
|
||||
@ -1047,22 +1177,6 @@ function kube-down {
|
||||
minions=( "${minions[@]:${batch}}" )
|
||||
done
|
||||
|
||||
# Delete firewall rule for the master.
|
||||
if gcloud compute firewall-rules describe --project "${PROJECT}" "${MASTER_NAME}-https" &>/dev/null; then
|
||||
gcloud compute firewall-rules delete \
|
||||
--project "${PROJECT}" \
|
||||
--quiet \
|
||||
"${MASTER_NAME}-https"
|
||||
fi
|
||||
|
||||
# Delete firewall rule for minions.
|
||||
if gcloud compute firewall-rules describe --project "${PROJECT}" "${NODE_TAG}-all" &>/dev/null; then
|
||||
gcloud compute firewall-rules delete \
|
||||
--project "${PROJECT}" \
|
||||
--quiet \
|
||||
"${NODE_TAG}-all"
|
||||
fi
|
||||
|
||||
# Delete routes.
|
||||
local -a routes
|
||||
# Clean up all routes w/ names like "<cluster-name>-<node-GUID>"
|
||||
@ -1083,16 +1197,6 @@ function kube-down {
|
||||
routes=( "${routes[@]:${batch}}" )
|
||||
done
|
||||
|
||||
# Delete the master's reserved IP
|
||||
local REGION=${ZONE%-*}
|
||||
if gcloud compute addresses describe "${MASTER_NAME}-ip" --region "${REGION}" --project "${PROJECT}" &>/dev/null; then
|
||||
gcloud compute addresses delete \
|
||||
--project "${PROJECT}" \
|
||||
--region "${REGION}" \
|
||||
--quiet \
|
||||
"${MASTER_NAME}-ip"
|
||||
fi
|
||||
|
||||
# Delete persistent disk for influx-db.
|
||||
if gcloud compute disks describe "${INSTANCE_PREFIX}"-influxdb-pd --zone "${ZONE}" --project "${PROJECT}" &>/dev/null; then
|
||||
gcloud compute disks delete \
|
||||
@ -1102,8 +1206,11 @@ function kube-down {
|
||||
"${INSTANCE_PREFIX}"-influxdb-pd
|
||||
fi
|
||||
|
||||
export CONTEXT="${PROJECT}_${INSTANCE_PREFIX}"
|
||||
clear-kubeconfig
|
||||
# If there are no more remaining master replicas, we should update kubeconfig.
|
||||
if [[ "${REMAINING_MASTER_COUNT}" == "0" ]]; then
|
||||
export CONTEXT="${PROJECT}_${INSTANCE_PREFIX}"
|
||||
clear-kubeconfig
|
||||
fi
|
||||
set -e
|
||||
}
|
||||
|
||||
@ -1124,6 +1231,7 @@ function get-template {
|
||||
# MASTER_NAME
|
||||
# NODE_INSTANCE_PREFIX
|
||||
# ZONE
|
||||
# REGION
|
||||
# Vars set:
|
||||
# KUBE_RESOURCE_FOUND
|
||||
function check-resources {
|
||||
@ -1192,7 +1300,6 @@ function check-resources {
|
||||
return 1
|
||||
fi
|
||||
|
||||
local REGION=${ZONE%-*}
|
||||
if gcloud compute addresses describe --project "${PROJECT}" "${MASTER_NAME}-ip" --region "${REGION}" &>/dev/null; then
|
||||
KUBE_RESOURCE_FOUND="Master's reserved IP"
|
||||
return 1
|
||||
|
||||
Loading…
Reference in New Issue
Block a user