From e54bc86ab313d688506418c2edf3730956a9fa5b Mon Sep 17 00:00:00 2001 From: Wojciech Tyczynski Date: Tue, 4 Oct 2016 10:15:12 +0200 Subject: [PATCH] Support upgrade/downgrade in etcd image. --- cluster/images/etcd/migrate-if-needed.sh | 188 +++++++++++++++++------ hack/test-update-storage-objects.sh | 2 + 2 files changed, 139 insertions(+), 51 deletions(-) diff --git a/cluster/images/etcd/migrate-if-needed.sh b/cluster/images/etcd/migrate-if-needed.sh index 05087684092..2dab1a3f141 100755 --- a/cluster/images/etcd/migrate-if-needed.sh +++ b/cluster/images/etcd/migrate-if-needed.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Copyright 2016 The Kubernetes Authors. # @@ -14,49 +14,93 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This script performs data migration between etcd2 and etcd3 versions -# if needed. -# Expected usage of it is: -# ./migrate_if_needed -# It will look into the contents of file /version.txt to -# determine the current storage version (no file means etcd2). +# NOTES +# This script performs etcd upgrade based on the following environmental +# variables: +# TARGET_STORAGE - API of etcd to be used (supported: 'etcd2', 'etcd3') +# TARGET_VERSION - etcd release to be used (supported: '2.2.1', '2.3.7', '3.0.10') +# DATA_DIRECTORY - directory with etcd data +# +# The current etcd version and storage format is detected based on the +# contents of "${DATA_DIRECTORY}/version.txt" file (if the file doesn't +# exist, we default it to "2.2.1/etcd2". +# +# The update workflow support the following upgrade steps: +# - 2.2.1/etcd2 -> 2.3.7/etcd2 +# - 2.3.7/etcd2 -> 3.0.10/etcd2 +# - 3.0.10/etcd2 -> 3.0.10/etcd3 +# +# NOTE: The releases supported in this script has to match release binaries +# present in the etcd image (to make this script work correctly). +# +# Based on the current etcd version and storage format we detect what +# upgrade step from this should be done to get reach target configuration set -o errexit set -o nounset if [ -z "${TARGET_STORAGE:-}" ]; then - echo "TARGET_USAGE variable unset - skipping migration" + echo "TARGET_STORAGE variable unset - skipping migration" + exit 0 +fi +if [ -z "${TARGET_VERSION:-}" ]; then + echo "TARGET_VERSION variable unset - skipping migration" exit 0 fi - if [ -z "${DATA_DIRECTORY:-}" ]; then echo "DATA_DIRECTORY variable unset - skipping migration" exit 0 fi -ETCD="${ETCD:-/usr/local/bin/etcd}" -ETCDCTL="${ETCDCTL:-/usr/local/bin/etcdctl}" -ATTACHLEASE="${ATTACHLEASE:-/usr/local/bin/attachlease}" -VERSION_FILE="version.txt" -CURRENT_STORAGE='etcd2' -if [ -e "${DATA_DIRECTORY}/${VERSION_FILE}" ]; then - CURRENT_STORAGE="$(cat ${DATA_DIRECTORY}/${VERSION_FILE})" +if [ "${TARGET_STORAGE}" != "etcd2" -a "${TARGET_STORAGE}" != "etcd3" ]; then + echo "Not supported version of storage: ${TARGET_STORAGE}" + exit 1 fi +# NOTE: SUPPORTED_VERSION has to match release binaries present in the +# etcd image (to make this script work correctly). +SUPPORTED_VERSIONS=("2.2.1" "2.3.7" "3.0.10") + +VERSION_FILE="version.txt" +CURRENT_STORAGE="etcd2" +CURRENT_VERSION="2.2.1" +if [ -e "${DATA_DIRECTORY}/${VERSION_FILE}" ]; then + VERSION_CONTENTS="$(cat ${DATA_DIRECTORY}/${VERSION_FILE})" + # Example usage: if contents of VERSION_FILE is 2.3.7/etcd2, then + # - CURRENT_VERSION would be '2.3.7' + # - CURRENT_STORAGE would be 'etcd2' + CURRENT_VERSION="$(echo $VERSION_CONTENTS | cut -d '/' -f 1)" + CURRENT_STORAGE="$(echo $VERSION_CONTENTS | cut -d '/' -f 2)" +fi + +# Starts 'etcd' version ${START_VERSION} and writes to it: +# 'etcd_version' -> "${START_VERSION}" +# Successful write confirms that etcd is up and running. +# Sets ETCD_PID at the end. +# Returns 0 if etcd was successfully started, non-0 otherwise. start_etcd() { + # Use random ports, so that apiserver cannot connect to etcd. ETCD_PORT=18629 ETCD_PEER_PORT=18630 - ${ETCD} --data-dir=${DATA_DIRECTORY} \ + local ETCD_CMD="${ETCD:-/usr/local/bin/etcd-${START_VERSION}}" + local ETCDCTL_CMD="${ETCDCTL:-/usr/local/bin/etcdctl-${START_VERSION}}" + if [ "${START_VERSION:0:2}" == "2." ]; then + ETCDCTL_CMD="${ETCDCTL_CMD} --endpoint=http://127.0.0.1:${ETCD_PORT} set" + else + ETCDCTL_CMD="${ETCDCTL_CMD} --endpoints=http://127.0.0.1:${ETCD_PORT} put" + fi + ${ETCD_CMD} --data-dir=${DATA_DIRECTORY} \ --listen-client-urls http://127.0.0.1:${ETCD_PORT} \ --advertise-client-urls http://127.0.0.1:${ETCD_PORT} \ --listen-peer-urls http://127.0.0.1:${ETCD_PEER_PORT} \ --initial-advertise-peer-urls http://127.0.0.1:${ETCD_PEER_PORT} \ 1>>/dev/null 2>&1 & ETCD_PID=$! - # Wait until etcd is up. - for i in $(seq 30); do - local out - if out=$(wget -q --timeout=1 http://127.0.0.1:${ETCD_PORT}/v2/machines 2> /dev/null); then + local API_VERSION="${START_STORAGE:4:4}" + # Wait until we can write to etcd. + for i in $(seq 240); do + ETCDCTL_API="${API_VERSION}" ${ETCDCTL_CMD} 'etcd_version' ${START_VERSION} + if [ "$?" -eq "0" ]; then echo "Etcd on port ${ETCD_PORT} is up." return 0 fi @@ -66,43 +110,85 @@ start_etcd() { return 1 } +# Stops etcd with ${ETCD_PID} pid. stop_etcd() { kill "${ETCD_PID-}" >/dev/null 2>&1 || : wait "${ETCD_PID-}" >/dev/null 2>&1 || : } -if [ "${CURRENT_STORAGE}" = "etcd2" -a "${TARGET_STORAGE}" = "etcd3" ]; then - # If directory doesn't exist or is empty, this means that there aren't any - # data for migration, which means we can skip this step. +ATTACHLEASE="${ATTACHLEASE:-/usr/local/bin/attachlease}" +ROLLBACK="${ROLLBACK:-/usr/local/bin/rollback}" + +# Do the roll-forward migration if needed. +for step in "${SUPPORTED_VERSIONS[@]}"; do + if [ "${step}" == "${CURRENT_VERSION}" -a "${CURRENT_VERSION}" != "${TARGET_VERSION}" ]; then + # Do the migration step, by just starting etcd in this version. + START_VERSION="${step}" + START_STORAGE="${CURRENT_STORAGE}" + if ! start_etcd; then + # Migration failed. + echo "Starting etcd ${step} failed" + exit 1 + fi + # Kill etcd and wait until this is down. + stop_etcd + fi + CURRENT_VERSION=${step} + echo "${CURRENT_VERSION}/${CURRENT_STORAGE}" > "${DATA_DIRECTORY}/${VERSION_FILE}" + if [ "${CURRENT_VERSION:0:2}" == "3." -a "${CURRENT_STORAGE}" == "etcd2" -a "${TARGET_STORAGE}" == "etcd3" ]; then + # If it is the first 3.x release in the list and we are migrating + # also from 'etcd2' to 'etcd3', do the migration now. + if [ -d "${DATA_DIRECTORY}" ]; then + if [ "$(ls -A ${DATA_DIRECTORY})" ]; then + echo "Performing etcd2 -> etcd3 migration" + START_VERSION="${step}" + START_STORAGE="etcd3" + ETCDCTL_CMD="${ETCDCTL:-/usr/local/bin/etcdctl-${START_VERSION}}" + ETCDCTL_API=3 ${ETCDCTL_CMD} migrate --data-dir=${DATA_DIRECTORY} + echo "Attaching leases to TTL entries" + # Now attach lease to all keys. + # To do it, we temporarily start etcd on a random port (so that + # apiserver actually cannot access it). + if ! start_etcd; then + echo "Starting etcd ${step} in v3 mode failed" + exit 1 + fi + # Create a lease and attach all keys to it. + ${ATTACHLEASE} \ + --etcd-address http://127.0.0.1:${ETCD_PORT} \ + --ttl-keys-prefix "${TTL_KEYS_DIRECTORY:-/registry/events}" \ + --lease-duration 1h + # Kill etcd and wait until this is down. + stop_etcd + fi + fi + CURRENT_STORAGE="etcd3" + echo "${CURRENT_VERSION}/${CURRENT_STORAGE}" > "${DATA_DIRECTORY}/${VERSION_FILE}" + fi + if [ "${CURRENT_VERSION}" == "${TARGET_VERSION}" -a "${CURRENT_STORAGE}" == "${TARGET_STORAGE}" ]; then + break + fi +done + +# Do the rollback of needed. +# NOTE: Rollback is only supported from "3.0.x" version in 'etcd3' mode to +# "2.3.7" version in 'etcd2' mode. +if [ "${CURRENT_STORAGE}" == "etcd3" -a "${TARGET_STORAGE}" == "etcd2" ]; then + if [ "${CURRENT_VERSION:0:4}" != "3.0." -o "${TARGET_VERSION}" != "2.3.7" ]; then + echo "etcd3 -> etcd2 downgrade is supported only between 3.0.x and 2.3.7" + return 0 + fi if [ -d "${DATA_DIRECTORY}" ]; then if [ "$(ls -A ${DATA_DIRECTORY})" ]; then - echo "Performing etcd2 -> etcd3 migration" - ETCDCTL_API=3 ${ETCDCTL} migrate --data-dir=${DATA_DIRECTORY} - echo "Attaching leases to TTL entries" - # Now attach lease to all keys. - # To do it, we temporarily start etcd on a random port (so that - # apiserver actually cannot access it). - start_etcd - # Create a lease and attach all keys to it. - ${ATTACHLEASE} \ - --etcd-address http://127.0.0.1:${ETCD_PORT} \ - --ttl-keys-prefix "${TTL_KEYS_DIRECTORY:-/registry/events}" \ - --lease-duration 1h - # Kill etcd and wait until this is down. - stop_etcd + echo "Performing etcd3 -> etcd2 rollback" + ${ROLLBACK} --data-dir "${DATA_DIRECTORY}" + if [ "$?" -ne "0" ]; then + echo "Rollback to etcd2 failed" + exit 1 + fi fi fi + CURRENT_STORAGE="etcd2" + CURRENT_VERSION="2.3.7" + echo "${CURRENT_VERSION}/${CURRENT_STORAGE}" > "${DATA_DIRECTORY}/${VERSION_FILE}" fi - -if [ "${CURRENT_STORAGE}" = "etcd3" -a "${TARGET_STORAGE}" = "etcd2" ]; then - echo "Performing etcd3 -> etcd2 migration" - # TODO: Implement rollback once this will be supported. - echo "etcd3 -> etcd2 downgrade is NOT supported." - # FIXME: On rollback, we will not support TTLs - we will simply clear - # all events. -fi - -# Write current storage version to avoid future migrations. -# If directory doesn't exist, we need to create it first. -mkdir -p "${DATA_DIRECTORY}" -echo "${TARGET_STORAGE}" > "${DATA_DIRECTORY}/${VERSION_FILE}" diff --git a/hack/test-update-storage-objects.sh b/hack/test-update-storage-objects.sh index bcb98f271d7..97edb7838be 100755 --- a/hack/test-update-storage-objects.sh +++ b/hack/test-update-storage-objects.sh @@ -103,6 +103,7 @@ make -C "${KUBE_ROOT}" WHAT=cmd/kube-apiserver make -C "${KUBE_ROOT}" WHAT=cluster/images/etcd/attachlease kube::etcd::start +echo "${ETCD_VERSION}/${STORAGE_BACKEND_ETCD2}" > "${ETCD_DIR}/version.txt" ### BEGIN TEST DEFINITION CUSTOMIZATION ### @@ -159,6 +160,7 @@ killApiServer kube::etcd::stop TARGET_STORAGE="etcd3" \ + TARGET_VERSION="3.0.10" \ DATA_DIRECTORY="${ETCD_DIR}" \ ETCD=$(which etcd) \ ETCDCTL=$(which etcdctl) \