mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-23 03:41:45 +00:00
Merge pull request #34009 from wojtek-t/upgrade_etcd_image
Automatic merge from submit-queue Support upgrade/downgrade in etcd image. Ref #22448 #20504
This commit is contained in:
commit
42392fea3c
@ -1,4 +1,4 @@
|
||||
#!/bin/sh
|
||||
#!/bin/bash
|
||||
|
||||
# Copyright 2016 The Kubernetes Authors.
|
||||
#
|
||||
@ -14,49 +14,93 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This script performs data migration between etcd2 and etcd3 versions
|
||||
# if needed.
|
||||
# Expected usage of it is:
|
||||
# ./migrate_if_needed <target-storage> <data-dir>
|
||||
# It will look into the contents of file <data-dir>/version.txt to
|
||||
# determine the current storage version (no file means etcd2).
|
||||
# NOTES
|
||||
# This script performs etcd upgrade based on the following environmental
|
||||
# variables:
|
||||
# TARGET_STORAGE - API of etcd to be used (supported: 'etcd2', 'etcd3')
|
||||
# TARGET_VERSION - etcd release to be used (supported: '2.2.1', '2.3.7', '3.0.10')
|
||||
# DATA_DIRECTORY - directory with etcd data
|
||||
#
|
||||
# The current etcd version and storage format is detected based on the
|
||||
# contents of "${DATA_DIRECTORY}/version.txt" file (if the file doesn't
|
||||
# exist, we default it to "2.2.1/etcd2".
|
||||
#
|
||||
# The update workflow support the following upgrade steps:
|
||||
# - 2.2.1/etcd2 -> 2.3.7/etcd2
|
||||
# - 2.3.7/etcd2 -> 3.0.10/etcd2
|
||||
# - 3.0.10/etcd2 -> 3.0.10/etcd3
|
||||
#
|
||||
# NOTE: The releases supported in this script has to match release binaries
|
||||
# present in the etcd image (to make this script work correctly).
|
||||
#
|
||||
# Based on the current etcd version and storage format we detect what
|
||||
# upgrade step from this should be done to get reach target configuration
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
|
||||
if [ -z "${TARGET_STORAGE:-}" ]; then
|
||||
echo "TARGET_USAGE variable unset - skipping migration"
|
||||
echo "TARGET_STORAGE variable unset - skipping migration"
|
||||
exit 0
|
||||
fi
|
||||
if [ -z "${TARGET_VERSION:-}" ]; then
|
||||
echo "TARGET_VERSION variable unset - skipping migration"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ -z "${DATA_DIRECTORY:-}" ]; then
|
||||
echo "DATA_DIRECTORY variable unset - skipping migration"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
ETCD="${ETCD:-/usr/local/bin/etcd}"
|
||||
ETCDCTL="${ETCDCTL:-/usr/local/bin/etcdctl}"
|
||||
ATTACHLEASE="${ATTACHLEASE:-/usr/local/bin/attachlease}"
|
||||
VERSION_FILE="version.txt"
|
||||
CURRENT_STORAGE='etcd2'
|
||||
if [ -e "${DATA_DIRECTORY}/${VERSION_FILE}" ]; then
|
||||
CURRENT_STORAGE="$(cat ${DATA_DIRECTORY}/${VERSION_FILE})"
|
||||
if [ "${TARGET_STORAGE}" != "etcd2" -a "${TARGET_STORAGE}" != "etcd3" ]; then
|
||||
echo "Not supported version of storage: ${TARGET_STORAGE}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# NOTE: SUPPORTED_VERSION has to match release binaries present in the
|
||||
# etcd image (to make this script work correctly).
|
||||
SUPPORTED_VERSIONS=("2.2.1" "2.3.7" "3.0.10")
|
||||
|
||||
VERSION_FILE="version.txt"
|
||||
CURRENT_STORAGE="etcd2"
|
||||
CURRENT_VERSION="2.2.1"
|
||||
if [ -e "${DATA_DIRECTORY}/${VERSION_FILE}" ]; then
|
||||
VERSION_CONTENTS="$(cat ${DATA_DIRECTORY}/${VERSION_FILE})"
|
||||
# Example usage: if contents of VERSION_FILE is 2.3.7/etcd2, then
|
||||
# - CURRENT_VERSION would be '2.3.7'
|
||||
# - CURRENT_STORAGE would be 'etcd2'
|
||||
CURRENT_VERSION="$(echo $VERSION_CONTENTS | cut -d '/' -f 1)"
|
||||
CURRENT_STORAGE="$(echo $VERSION_CONTENTS | cut -d '/' -f 2)"
|
||||
fi
|
||||
|
||||
# Starts 'etcd' version ${START_VERSION} and writes to it:
|
||||
# 'etcd_version' -> "${START_VERSION}"
|
||||
# Successful write confirms that etcd is up and running.
|
||||
# Sets ETCD_PID at the end.
|
||||
# Returns 0 if etcd was successfully started, non-0 otherwise.
|
||||
start_etcd() {
|
||||
# Use random ports, so that apiserver cannot connect to etcd.
|
||||
ETCD_PORT=18629
|
||||
ETCD_PEER_PORT=18630
|
||||
${ETCD} --data-dir=${DATA_DIRECTORY} \
|
||||
local ETCD_CMD="${ETCD:-/usr/local/bin/etcd-${START_VERSION}}"
|
||||
local ETCDCTL_CMD="${ETCDCTL:-/usr/local/bin/etcdctl-${START_VERSION}}"
|
||||
if [ "${START_VERSION:0:2}" == "2." ]; then
|
||||
ETCDCTL_CMD="${ETCDCTL_CMD} --endpoint=http://127.0.0.1:${ETCD_PORT} set"
|
||||
else
|
||||
ETCDCTL_CMD="${ETCDCTL_CMD} --endpoints=http://127.0.0.1:${ETCD_PORT} put"
|
||||
fi
|
||||
${ETCD_CMD} --data-dir=${DATA_DIRECTORY} \
|
||||
--listen-client-urls http://127.0.0.1:${ETCD_PORT} \
|
||||
--advertise-client-urls http://127.0.0.1:${ETCD_PORT} \
|
||||
--listen-peer-urls http://127.0.0.1:${ETCD_PEER_PORT} \
|
||||
--initial-advertise-peer-urls http://127.0.0.1:${ETCD_PEER_PORT} \
|
||||
1>>/dev/null 2>&1 &
|
||||
ETCD_PID=$!
|
||||
# Wait until etcd is up.
|
||||
for i in $(seq 30); do
|
||||
local out
|
||||
if out=$(wget -q --timeout=1 http://127.0.0.1:${ETCD_PORT}/v2/machines 2> /dev/null); then
|
||||
local API_VERSION="${START_STORAGE:4:4}"
|
||||
# Wait until we can write to etcd.
|
||||
for i in $(seq 240); do
|
||||
ETCDCTL_API="${API_VERSION}" ${ETCDCTL_CMD} 'etcd_version' ${START_VERSION}
|
||||
if [ "$?" -eq "0" ]; then
|
||||
echo "Etcd on port ${ETCD_PORT} is up."
|
||||
return 0
|
||||
fi
|
||||
@ -66,43 +110,85 @@ start_etcd() {
|
||||
return 1
|
||||
}
|
||||
|
||||
# Stops etcd with ${ETCD_PID} pid.
|
||||
stop_etcd() {
|
||||
kill "${ETCD_PID-}" >/dev/null 2>&1 || :
|
||||
wait "${ETCD_PID-}" >/dev/null 2>&1 || :
|
||||
}
|
||||
|
||||
if [ "${CURRENT_STORAGE}" = "etcd2" -a "${TARGET_STORAGE}" = "etcd3" ]; then
|
||||
# If directory doesn't exist or is empty, this means that there aren't any
|
||||
# data for migration, which means we can skip this step.
|
||||
ATTACHLEASE="${ATTACHLEASE:-/usr/local/bin/attachlease}"
|
||||
ROLLBACK="${ROLLBACK:-/usr/local/bin/rollback}"
|
||||
|
||||
# Do the roll-forward migration if needed.
|
||||
for step in "${SUPPORTED_VERSIONS[@]}"; do
|
||||
if [ "${step}" == "${CURRENT_VERSION}" -a "${CURRENT_VERSION}" != "${TARGET_VERSION}" ]; then
|
||||
# Do the migration step, by just starting etcd in this version.
|
||||
START_VERSION="${step}"
|
||||
START_STORAGE="${CURRENT_STORAGE}"
|
||||
if ! start_etcd; then
|
||||
# Migration failed.
|
||||
echo "Starting etcd ${step} failed"
|
||||
exit 1
|
||||
fi
|
||||
# Kill etcd and wait until this is down.
|
||||
stop_etcd
|
||||
fi
|
||||
CURRENT_VERSION=${step}
|
||||
echo "${CURRENT_VERSION}/${CURRENT_STORAGE}" > "${DATA_DIRECTORY}/${VERSION_FILE}"
|
||||
if [ "${CURRENT_VERSION:0:2}" == "3." -a "${CURRENT_STORAGE}" == "etcd2" -a "${TARGET_STORAGE}" == "etcd3" ]; then
|
||||
# If it is the first 3.x release in the list and we are migrating
|
||||
# also from 'etcd2' to 'etcd3', do the migration now.
|
||||
if [ -d "${DATA_DIRECTORY}" ]; then
|
||||
if [ "$(ls -A ${DATA_DIRECTORY})" ]; then
|
||||
echo "Performing etcd2 -> etcd3 migration"
|
||||
START_VERSION="${step}"
|
||||
START_STORAGE="etcd3"
|
||||
ETCDCTL_CMD="${ETCDCTL:-/usr/local/bin/etcdctl-${START_VERSION}}"
|
||||
ETCDCTL_API=3 ${ETCDCTL_CMD} migrate --data-dir=${DATA_DIRECTORY}
|
||||
echo "Attaching leases to TTL entries"
|
||||
# Now attach lease to all keys.
|
||||
# To do it, we temporarily start etcd on a random port (so that
|
||||
# apiserver actually cannot access it).
|
||||
if ! start_etcd; then
|
||||
echo "Starting etcd ${step} in v3 mode failed"
|
||||
exit 1
|
||||
fi
|
||||
# Create a lease and attach all keys to it.
|
||||
${ATTACHLEASE} \
|
||||
--etcd-address http://127.0.0.1:${ETCD_PORT} \
|
||||
--ttl-keys-prefix "${TTL_KEYS_DIRECTORY:-/registry/events}" \
|
||||
--lease-duration 1h
|
||||
# Kill etcd and wait until this is down.
|
||||
stop_etcd
|
||||
fi
|
||||
fi
|
||||
CURRENT_STORAGE="etcd3"
|
||||
echo "${CURRENT_VERSION}/${CURRENT_STORAGE}" > "${DATA_DIRECTORY}/${VERSION_FILE}"
|
||||
fi
|
||||
if [ "${CURRENT_VERSION}" == "${TARGET_VERSION}" -a "${CURRENT_STORAGE}" == "${TARGET_STORAGE}" ]; then
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
# Do the rollback of needed.
|
||||
# NOTE: Rollback is only supported from "3.0.x" version in 'etcd3' mode to
|
||||
# "2.3.7" version in 'etcd2' mode.
|
||||
if [ "${CURRENT_STORAGE}" == "etcd3" -a "${TARGET_STORAGE}" == "etcd2" ]; then
|
||||
if [ "${CURRENT_VERSION:0:4}" != "3.0." -o "${TARGET_VERSION}" != "2.3.7" ]; then
|
||||
echo "etcd3 -> etcd2 downgrade is supported only between 3.0.x and 2.3.7"
|
||||
return 0
|
||||
fi
|
||||
if [ -d "${DATA_DIRECTORY}" ]; then
|
||||
if [ "$(ls -A ${DATA_DIRECTORY})" ]; then
|
||||
echo "Performing etcd2 -> etcd3 migration"
|
||||
ETCDCTL_API=3 ${ETCDCTL} migrate --data-dir=${DATA_DIRECTORY}
|
||||
echo "Attaching leases to TTL entries"
|
||||
# Now attach lease to all keys.
|
||||
# To do it, we temporarily start etcd on a random port (so that
|
||||
# apiserver actually cannot access it).
|
||||
start_etcd
|
||||
# Create a lease and attach all keys to it.
|
||||
${ATTACHLEASE} \
|
||||
--etcd-address http://127.0.0.1:${ETCD_PORT} \
|
||||
--ttl-keys-prefix "${TTL_KEYS_DIRECTORY:-/registry/events}" \
|
||||
--lease-duration 1h
|
||||
# Kill etcd and wait until this is down.
|
||||
stop_etcd
|
||||
echo "Performing etcd3 -> etcd2 rollback"
|
||||
${ROLLBACK} --data-dir "${DATA_DIRECTORY}"
|
||||
if [ "$?" -ne "0" ]; then
|
||||
echo "Rollback to etcd2 failed"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
CURRENT_STORAGE="etcd2"
|
||||
CURRENT_VERSION="2.3.7"
|
||||
echo "${CURRENT_VERSION}/${CURRENT_STORAGE}" > "${DATA_DIRECTORY}/${VERSION_FILE}"
|
||||
fi
|
||||
|
||||
if [ "${CURRENT_STORAGE}" = "etcd3" -a "${TARGET_STORAGE}" = "etcd2" ]; then
|
||||
echo "Performing etcd3 -> etcd2 migration"
|
||||
# TODO: Implement rollback once this will be supported.
|
||||
echo "etcd3 -> etcd2 downgrade is NOT supported."
|
||||
# FIXME: On rollback, we will not support TTLs - we will simply clear
|
||||
# all events.
|
||||
fi
|
||||
|
||||
# Write current storage version to avoid future migrations.
|
||||
# If directory doesn't exist, we need to create it first.
|
||||
mkdir -p "${DATA_DIRECTORY}"
|
||||
echo "${TARGET_STORAGE}" > "${DATA_DIRECTORY}/${VERSION_FILE}"
|
||||
|
@ -103,6 +103,7 @@ make -C "${KUBE_ROOT}" WHAT=cmd/kube-apiserver
|
||||
make -C "${KUBE_ROOT}" WHAT=cluster/images/etcd/attachlease
|
||||
|
||||
kube::etcd::start
|
||||
echo "${ETCD_VERSION}/${STORAGE_BACKEND_ETCD2}" > "${ETCD_DIR}/version.txt"
|
||||
|
||||
### BEGIN TEST DEFINITION CUSTOMIZATION ###
|
||||
|
||||
@ -159,6 +160,7 @@ killApiServer
|
||||
|
||||
kube::etcd::stop
|
||||
TARGET_STORAGE="etcd3" \
|
||||
TARGET_VERSION="3.0.10" \
|
||||
DATA_DIRECTORY="${ETCD_DIR}" \
|
||||
ETCD=$(which etcd) \
|
||||
ETCDCTL=$(which etcdctl) \
|
||||
|
Loading…
Reference in New Issue
Block a user