mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-23 03:41:45 +00:00
Merge pull request #59298 from jpbetz/etcd3-minor-version-rollback
Automatic merge from submit-queue (batch tested with PRs 59298, 59773, 59772). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.
Add etcd 3.x minor version rollback support to migrate-if-needed.sh
Provide automatic etcd 3.x minor version downgrade when using the gcr.io/google_containers/etcd docker images to operate etcd.
Uses `etcdctl snapshot save` and `etcdctl snapshot restore` to safely downgrade etcd from 3.2->3.1 or 3.1->3.0. This is safe because the data storage file formats used by etcd have not changed between these versions.
Intended as a stop-gap until we can introduce more comprehensive downgrade support in etcd. The main limitation of this approach is that it is not able to perform zero downtime downgrades for HA clusters. For HA clusters, all members must be stopped and downgraded before the cluster may be restarted at the downgraded version.
Example usage:
- Initially the [etcd.manifest](58547ebd72/cluster/gce/manifests/etcd.manifest (L43)) is set to gcr.io/google_containers/etcd:3.0.17, TARGET_VERSION=3.0.17
- A upgrade to 3.1.11 is initiated.
- etcd.manifest is updated to gcr.io/google_containers/etcd:3.1.11, TARGET_VERSION=3.1.11
- etcd restarts and establishes 3.1 as it's "cluster version"
- For whatever reason, a downgrade is initiated
- etcd.manifest is updated gcr.io/google_containers/etcd:3.1.11, TARGET_VERSION=3.0.17
- migrate-if-needed.sh detects that the current version (3.1.11) is newer than the target version, so it:
- creates a snapshot using etcd & etcdctl 3.1.11
- backs up the data dir
- restores the snapshot using etcdctl 3.0.17 to create a replacement data dir
- starts etcd 3.0.17
Note that while this will rollback to an earlier etcd version, the newer etcd gcr.io image version must continue to be used throughout the downgrade. Only TARGET_VERSION is downgraded.
Test coverage was lacking for `migrate-if-needed.sh` so this adds some container level testing to the `Makefile` for migrating and rolling back. This surfaced a couple bugs that are fixed by this PR as well.
cc @mml @lavalamp @wenjiaswe
```release-note
Add automatic etcd 3.2->3.1 and 3.1->3.0 minor version rollback support to gcr.io/google_container/etcd images. For HA clusters, all members must be stopped before performing a rollback.
```
This commit is contained in:
Kubernetes Submit Queue
GitHub
commit
c1216dfdc4
@ -33,6 +33,9 @@
|
||||
},
|
||||
{ "name": "DATA_DIRECTORY",
|
||||
"value": "/var/etcd/data{{ suffix }}"
|
||||
},
|
||||
{ "name": "INITIAL_CLUSTER",
|
||||
"value": "{{ etcd_cluster }}"
|
||||
}
|
||||
],
|
||||
"livenessProbe": {
|
||||
|
||||
@ -16,4 +16,4 @@ FROM BASEIMAGE
|
||||
|
||||
EXPOSE 2379 2380 4001 7001
|
||||
COPY etcd* etcdctl* /usr/local/bin/
|
||||
COPY migrate-if-needed.sh attachlease rollback /usr/local/bin/
|
||||
COPY migrate-if-needed.sh start-stop-etcd.sh attachlease rollback /usr/local/bin/
|
||||
|
||||
@ -28,6 +28,8 @@
|
||||
# That binary will be set to the last tag from $(TAGS).
|
||||
TAGS?=2.2.1 2.3.7 3.0.17 3.1.11 3.2.14
|
||||
REGISTRY_TAG?=3.2.14
|
||||
# ROLLBACK_REGISTRY_TAG specified the tag that REGISTRY_TAG may be rolled back to.
|
||||
ROLLBACK_REGISTRY_TAG?=3.1.11
|
||||
ARCH?=amd64
|
||||
REGISTRY?=k8s.gcr.io
|
||||
# golang version should match the golang version from https://github.com/coreos/etcd/releases for REGISTRY_TAG version of etcd.
|
||||
@ -57,10 +59,10 @@ build:
|
||||
find ./ -maxdepth 1 -type f | xargs -I {} cp {} $(TEMP_DIR)
|
||||
|
||||
# Compile attachlease
|
||||
docker run -i -v $(shell pwd)/../../../:/go/src/k8s.io/kubernetes -v $(TEMP_DIR):/build -e GOARCH=$(ARCH) golang:$(GOLANG_VERSION) \
|
||||
docker run --interactive -v $(shell pwd)/../../../:/go/src/k8s.io/kubernetes -v $(TEMP_DIR):/build -e GOARCH=$(ARCH) golang:$(GOLANG_VERSION) \
|
||||
/bin/bash -c "CGO_ENABLED=0 go build -o /build/attachlease k8s.io/kubernetes/cluster/images/etcd/attachlease"
|
||||
# Compile rollback
|
||||
docker run -i -v $(shell pwd)/../../../:/go/src/k8s.io/kubernetes -v $(TEMP_DIR):/build -e GOARCH=$(ARCH) golang:$(GOLANG_VERSION) \
|
||||
docker run --interactive -v $(shell pwd)/../../../:/go/src/k8s.io/kubernetes -v $(TEMP_DIR):/build -e GOARCH=$(ARCH) golang:$(GOLANG_VERSION) \
|
||||
/bin/bash -c "CGO_ENABLED=0 go build -o /build/rollback k8s.io/kubernetes/cluster/images/etcd/rollback"
|
||||
|
||||
|
||||
@ -81,7 +83,7 @@ else
|
||||
# For each release create a tmp dir 'etcd_release_tmp_dir' and unpack the release tar there.
|
||||
for tag in $(TAGS); do \
|
||||
etcd_release_tmp_dir=$(shell mktemp -d); \
|
||||
docker run -i -v $$etcd_release_tmp_dir:/etcdbin golang:$(GOLANG_VERSION) /bin/bash -c \
|
||||
docker run --interactive -v $$etcd_release_tmp_dir:/etcdbin golang:$(GOLANG_VERSION) /bin/bash -c \
|
||||
"git clone https://github.com/coreos/etcd /go/src/github.com/coreos/etcd \
|
||||
&& cd /go/src/github.com/coreos/etcd \
|
||||
&& git checkout v$$tag \
|
||||
@ -114,5 +116,127 @@ ifeq ($(ARCH),amd64)
|
||||
gcloud docker -- push $(REGISTRY)/etcd:$(REGISTRY_TAG)
|
||||
endif
|
||||
|
||||
all: build
|
||||
.PHONY: build push
|
||||
ETCD2_ROLLBACK_NEW_TAG=3.0.17
|
||||
ETCD2_ROLLBACK_OLD_TAG=2.2.1
|
||||
|
||||
# Test a rollback to etcd2 from the earliest etcd3 version.
|
||||
test-rollback-etcd2:
|
||||
mkdir -p $(TEMP_DIR)/rollback-etcd2
|
||||
cd $(TEMP_DIR)/rollback-etcd2
|
||||
|
||||
@echo "Starting $(ETCD2_ROLLBACK_NEW_TAG) etcd and writing some sample data."
|
||||
docker run --tty --interactive -v $(TEMP_DIR)/rollback-etcd2:/var/etcd \
|
||||
-e "TARGET_STORAGE=etcd3" \
|
||||
-e "TARGET_VERSION=$(ETCD2_ROLLBACK_NEW_TAG)" \
|
||||
-e "DATA_DIRECTORY=/var/etcd/data" \
|
||||
gcr.io/google_containers/etcd-$(ARCH):$(REGISTRY_TAG) /bin/sh -c \
|
||||
'INITIAL_CLUSTER=etcd-$$(hostname)=http://localhost:2380 \
|
||||
/usr/local/bin/migrate-if-needed.sh && \
|
||||
source /usr/local/bin/start-stop-etcd.sh && \
|
||||
START_STORAGE=etcd3 START_VERSION=$(ETCD2_ROLLBACK_NEW_TAG) start_etcd && \
|
||||
ETCDCTL_API=3 /usr/local/bin/etcdctl-$(ETCD2_ROLLBACK_NEW_TAG) --endpoints http://127.0.0.1:$${ETCD_PORT} put /registry/k1 value1 && \
|
||||
stop_etcd && \
|
||||
[ $$(cat /var/etcd/data/version.txt) = $(ETCD2_ROLLBACK_NEW_TAG)/etcd3 ]'
|
||||
|
||||
@echo "Rolling back to the previous version of etcd and recording keyspace to a flat file."
|
||||
docker run --tty --interactive -v $(TEMP_DIR)/rollback-etcd2:/var/etcd \
|
||||
-e "TARGET_STORAGE=etcd2" \
|
||||
-e "TARGET_VERSION=$(ETCD2_ROLLBACK_OLD_TAG)" \
|
||||
-e "DATA_DIRECTORY=/var/etcd/data" \
|
||||
gcr.io/google_containers/etcd-$(ARCH):$(REGISTRY_TAG) /bin/sh -c \
|
||||
'INITIAL_CLUSTER=etcd-$$(hostname)=http://localhost:2380 \
|
||||
/usr/local/bin/migrate-if-needed.sh && \
|
||||
source /usr/local/bin/start-stop-etcd.sh && \
|
||||
START_STORAGE=etcd2 START_VERSION=$(ETCD2_ROLLBACK_OLD_TAG) start_etcd && \
|
||||
/usr/local/bin/etcdctl-$(ETCD2_ROLLBACK_OLD_TAG) --endpoint 127.0.0.1:$${ETCD_PORT} get /registry/k1 > /var/etcd/keyspace.txt && \
|
||||
stop_etcd'
|
||||
|
||||
@echo "Checking if rollback successfully downgraded etcd to $(ETCD2_ROLLBACK_OLD_TAG)"
|
||||
docker run --tty --interactive -v $(TEMP_DIR)/rollback-etcd2:/var/etcd \
|
||||
gcr.io/google_containers/etcd-$(ARCH):$(REGISTRY_TAG) /bin/sh -c \
|
||||
'[ $$(cat /var/etcd/data/version.txt) = $(ETCD2_ROLLBACK_OLD_TAG)/etcd2 ] && \
|
||||
grep -q value1 /var/etcd/keyspace.txt'
|
||||
|
||||
# Test a rollback from the latest version to the previous version.
|
||||
test-rollback:
|
||||
mkdir -p $(TEMP_DIR)/rollback-test
|
||||
cd $(TEMP_DIR)/rollback-test
|
||||
|
||||
@echo "Starting $(REGISTRY_TAG) etcd and writing some sample data."
|
||||
docker run --tty --interactive -v $(TEMP_DIR)/rollback-test:/var/etcd \
|
||||
-e "TARGET_STORAGE=etcd3" \
|
||||
-e "TARGET_VERSION=$(REGISTRY_TAG)" \
|
||||
-e "DATA_DIRECTORY=/var/etcd/data" \
|
||||
gcr.io/google_containers/etcd-$(ARCH):$(REGISTRY_TAG) /bin/sh -c \
|
||||
'INITIAL_CLUSTER=etcd-$$(hostname)=http://localhost:2380 \
|
||||
/usr/local/bin/migrate-if-needed.sh && \
|
||||
source /usr/local/bin/start-stop-etcd.sh && \
|
||||
START_STORAGE=etcd3 START_VERSION=$(REGISTRY_TAG) start_etcd && \
|
||||
ETCDCTL_API=3 /usr/local/bin/etcdctl --endpoints http://127.0.0.1:$${ETCD_PORT} put /registry/k1 value1 && \
|
||||
stop_etcd'
|
||||
|
||||
@echo "Rolling back to the previous version of etcd and recording keyspace to a flat file."
|
||||
docker run --tty --interactive -v $(TEMP_DIR)/rollback-test:/var/etcd \
|
||||
-e "TARGET_STORAGE=etcd3" \
|
||||
-e "TARGET_VERSION=$(ROLLBACK_REGISTRY_TAG)" \
|
||||
-e "DATA_DIRECTORY=/var/etcd/data" \
|
||||
gcr.io/google_containers/etcd-$(ARCH):$(REGISTRY_TAG) /bin/sh -c \
|
||||
'INITIAL_CLUSTER=etcd-$$(hostname)=http://localhost:2380 \
|
||||
/usr/local/bin/migrate-if-needed.sh && \
|
||||
source /usr/local/bin/start-stop-etcd.sh && \
|
||||
START_STORAGE=etcd3 START_VERSION=$(ROLLBACK_REGISTRY_TAG) start_etcd && \
|
||||
ETCDCTL_API=3 /usr/local/bin/etcdctl --endpoints http://127.0.0.1:$${ETCD_PORT} get --prefix / > /var/etcd/keyspace.txt && \
|
||||
stop_etcd'
|
||||
|
||||
@echo "Checking if rollback successfully downgraded etcd to $(ROLLBACK_REGISTRY_TAG)"
|
||||
docker run --tty --interactive -v $(TEMP_DIR)/rollback-test:/var/etcd \
|
||||
gcr.io/google_containers/etcd-$(ARCH):$(REGISTRY_TAG) /bin/sh -c \
|
||||
'[ $$(cat /var/etcd/data/version.txt) = $(ROLLBACK_REGISTRY_TAG)/etcd3 ] && \
|
||||
grep -q value1 /var/etcd/keyspace.txt'
|
||||
|
||||
# Test migrating from each supported versions to the latest version.
|
||||
test-migrate:
|
||||
for tag in $(TAGS); do \
|
||||
echo "Testing migration from $${tag} to $(REGISTRY_TAG)" && \
|
||||
mkdir -p $(TEMP_DIR)/migrate-$${tag} && \
|
||||
cd $(TEMP_DIR)/migrate-$${tag} && \
|
||||
MAJOR_VERSION=$$(echo $${tag} | cut -c 1) && \
|
||||
echo "Starting etcd $${tag} and writing sample data to keyspace" && \
|
||||
docker run --tty --interactive -v $(TEMP_DIR)/migrate-$${tag}:/var/etcd \
|
||||
-e "TARGET_STORAGE=etcd$${MAJOR_VERSION}" \
|
||||
-e "TARGET_VERSION=$${tag}" \
|
||||
-e "DATA_DIRECTORY=/var/etcd/data" \
|
||||
gcr.io/google_containers/etcd-$(ARCH):$(REGISTRY_TAG) /bin/sh -c \
|
||||
"INITIAL_CLUSTER=etcd-\$$(hostname)=http://localhost:2380 \
|
||||
/usr/local/bin/migrate-if-needed.sh && \
|
||||
source /usr/local/bin/start-stop-etcd.sh && \
|
||||
START_STORAGE=etcd$${MAJOR_VERSION} START_VERSION=$${tag} start_etcd && \
|
||||
if [ $${MAJOR_VERSION} == 2 ]; then \
|
||||
/usr/local/bin/etcdctl --endpoint http://127.0.0.1:\$${ETCD_PORT} set /registry/k1 value1; \
|
||||
else \
|
||||
ETCDCTL_API=3 /usr/local/bin/etcdctl --endpoints http://127.0.0.1:\$${ETCD_PORT} put /registry/k1 value1; \
|
||||
fi && \
|
||||
stop_etcd" && \
|
||||
echo " Migrating from $${tag} to $(REGISTRY_TAG) and capturing keyspace" && \
|
||||
docker run --tty --interactive -v $(TEMP_DIR)/migrate-$${tag}:/var/etcd \
|
||||
-e "TARGET_STORAGE=etcd3" \
|
||||
-e "TARGET_VERSION=$(REGISTRY_TAG)" \
|
||||
-e "DATA_DIRECTORY=/var/etcd/data" \
|
||||
gcr.io/google_containers/etcd-$(ARCH):$(REGISTRY_TAG) /bin/sh -c \
|
||||
'INITIAL_CLUSTER=etcd-$$(hostname)=http://localhost:2380 \
|
||||
/usr/local/bin/migrate-if-needed.sh && \
|
||||
source /usr/local/bin/start-stop-etcd.sh && \
|
||||
START_STORAGE=etcd3 START_VERSION=$(REGISTRY_TAG) start_etcd && \
|
||||
ETCDCTL_API=3 /usr/local/bin/etcdctl --endpoints http://127.0.0.1:$${ETCD_PORT} get --prefix / > /var/etcd/keyspace.txt && \
|
||||
stop_etcd' && \
|
||||
echo "Checking if migrate from $${tag} successfully upgraded etcd to $(REGISTRY_TAG)" && \
|
||||
docker run --tty --interactive -v $(TEMP_DIR)/migrate-$${tag}:/var/etcd \
|
||||
gcr.io/google_containers/etcd-$(ARCH):$(REGISTRY_TAG) /bin/sh -c \
|
||||
'[ $$(cat /var/etcd/data/version.txt) = $(REGISTRY_TAG)/etcd3 ] && \
|
||||
grep -q value1 /var/etcd/keyspace.txt'; \
|
||||
done
|
||||
|
||||
test: test-rollback test-rollback-etcd2 test-migrate
|
||||
|
||||
all: build test
|
||||
.PHONY: build push test-rollback test-rollback-etcd2 test-migrate test
|
||||
|
||||
@ -7,6 +7,14 @@ For other architectures, `etcd` is cross-compiled from source. Arch-specific `bu
|
||||
|
||||
#### How to release
|
||||
|
||||
First, run the migration and rollback tests.
|
||||
|
||||
```console
|
||||
$ make build test
|
||||
```
|
||||
|
||||
Next, build and push the docker images for all supported architectures.
|
||||
|
||||
```console
|
||||
# Build for linux/amd64 (default)
|
||||
$ make push ARCH=amd64
|
||||
|
||||
@ -18,7 +18,7 @@
|
||||
# This script performs etcd upgrade based on the following environmental
|
||||
# variables:
|
||||
# TARGET_STORAGE - API of etcd to be used (supported: 'etcd2', 'etcd3')
|
||||
# TARGET_VERSION - etcd release to be used (supported: '2.2.1', '2.3.7', '3.0.17')
|
||||
# TARGET_VERSION - etcd release to be used (supported: '2.2.1', '2.3.7', '3.0.17', '3.1.11', '3.2.14')
|
||||
# DATA_DIRECTORY - directory with etcd data
|
||||
#
|
||||
# The current etcd version and storage format is detected based on the
|
||||
@ -28,7 +28,8 @@
|
||||
# The update workflow support the following upgrade steps:
|
||||
# - 2.2.1/etcd2 -> 2.3.7/etcd2
|
||||
# - 2.3.7/etcd2 -> 3.0.17/etcd2
|
||||
# - 3.0.17/etcd2 -> 3.0.17/etcd3
|
||||
# - 3.0.17/etcd3 -> 3.1.11/etcd3
|
||||
# - 3.1.11/etcd3 -> 3.2.14/etcd3
|
||||
#
|
||||
# NOTE: The releases supported in this script has to match release binaries
|
||||
# present in the etcd image (to make this script work correctly).
|
||||
@ -39,6 +40,72 @@
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
|
||||
source $(dirname "$0")/start-stop-etcd.sh
|
||||
|
||||
# Rollback to previous minor version of etcd 3.x, if needed.
|
||||
#
|
||||
# Warning: For HA etcd clusters (any cluster with more than one member), all members must be stopped before rolling back, zero
|
||||
# downtime rollbacks are not supported.
|
||||
rollback_etcd3_minor_version() {
|
||||
if [ ${TARGET_MINOR_VERSION} != $((${CURRENT_MINOR_VERSION}-1)) ]; then
|
||||
echo "Rollback from ${CURRENT_VERSION} to ${TARGET_VERSION} not supported, only rollbacks to the previous minor version are supported."
|
||||
exit 1
|
||||
fi
|
||||
echo "Performing etcd ${CURRENT_VERSION} -> ${TARGET_VERSION} rollback"
|
||||
ROLLBACK_BACKUP_DIR="${DATA_DIRECTORY}.bak"
|
||||
rm -rf "${ROLLBACK_BACKUP_DIR}"
|
||||
SNAPSHOT_FILE="${DATA_DIRECTORY}.snapshot.db"
|
||||
rm -rf "${SNAPSHOT_FILE}"
|
||||
ETCD_CMD="/usr/local/bin/etcd-${CURRENT_VERSION}"
|
||||
ETCDCTL_CMD="/usr/local/bin/etcdctl-${CURRENT_VERSION}"
|
||||
|
||||
# Start CURRENT_VERSION of etcd.
|
||||
START_VERSION="${CURRENT_VERSION}"
|
||||
START_STORAGE="${CURRENT_STORAGE}"
|
||||
echo "Starting etcd version ${START_VERSION} to capture rollback snapshot."
|
||||
if ! start_etcd; then
|
||||
echo "Unable to automatically downgrade etcd: starting etcd version ${START_VERSION} to capture rollback snapshot failed."
|
||||
echo "See https://coreos.com/etcd/docs/3.2.13/op-guide/recovery.html for manual downgrade options."
|
||||
exit 1
|
||||
else
|
||||
ETCDCTL_API=3 ${ETCDCTL_CMD} snapshot --endpoints "http://127.0.0.1:${ETCD_PORT}" save "${SNAPSHOT_FILE}"
|
||||
fi
|
||||
stop_etcd
|
||||
|
||||
# Backup the data before rolling back.
|
||||
mv "${DATA_DIRECTORY}" "${ROLLBACK_BACKUP_DIR}"
|
||||
ETCDCTL_CMD="/usr/local/bin/etcdctl-${TARGET_VERSION}"
|
||||
NAME="etcd-$(hostname)"
|
||||
ETCDCTL_API=3 ${ETCDCTL_CMD} snapshot restore "${SNAPSHOT_FILE}" \
|
||||
--data-dir "${DATA_DIRECTORY}" --name "${NAME}" --initial-cluster "${INITIAL_CLUSTER}"
|
||||
|
||||
CURRENT_VERSION="${TARGET_VERSION}"
|
||||
echo "${CURRENT_VERSION}/${CURRENT_STORAGE}" > "${DATA_DIRECTORY}/${VERSION_FILE}"
|
||||
}
|
||||
|
||||
# Rollback from "3.0.x" version in 'etcd3' mode to "2.2.1" version in 'etcd2' mode, if needed.
|
||||
rollback_to_etcd2() {
|
||||
if [ "$(echo ${CURRENT_VERSION} | cut -c1-4)" != "3.0." -o "${TARGET_VERSION}" != "2.2.1" ]; then
|
||||
echo "etcd3 -> etcd2 downgrade is supported only between 3.0.x and 2.2.1"
|
||||
return 0
|
||||
fi
|
||||
echo "Backup and remove all existing v2 data"
|
||||
ROLLBACK_BACKUP_DIR="${DATA_DIRECTORY}.bak"
|
||||
rm -rf "${ROLLBACK_BACKUP_DIR}"
|
||||
mkdir -p "${ROLLBACK_BACKUP_DIR}"
|
||||
cp -r "${DATA_DIRECTORY}" "${ROLLBACK_BACKUP_DIR}"
|
||||
echo "Performing etcd3 -> etcd2 rollback"
|
||||
${ROLLBACK} --data-dir "${DATA_DIRECTORY}"
|
||||
if [ "$?" -ne "0" ]; then
|
||||
echo "Rollback to etcd2 failed"
|
||||
exit 1
|
||||
fi
|
||||
CURRENT_STORAGE="etcd2"
|
||||
CURRENT_VERSION="2.2.1"
|
||||
echo "${CURRENT_VERSION}/${CURRENT_STORAGE}" > "${DATA_DIRECTORY}/${VERSION_FILE}"
|
||||
}
|
||||
|
||||
|
||||
if [ -z "${TARGET_STORAGE:-}" ]; then
|
||||
echo "TARGET_STORAGE variable unset - unexpected failure"
|
||||
exit 1
|
||||
@ -51,6 +118,10 @@ if [ -z "${DATA_DIRECTORY:-}" ]; then
|
||||
echo "DATA_DIRECTORY variable unset - unexpected failure"
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "${INITIAL_CLUSTER:-}" ]; then
|
||||
echo "Warn: INITIAL_CLUSTER variable unset - defaulting to etcd-$(hostname)=http://localhost:2380"
|
||||
INITIAL_CLUSTER="etcd-$(hostname)=http://localhost:2380"
|
||||
fi
|
||||
|
||||
echo "$(date +'%Y-%m-%d %H:%M:%S') Detecting if migration is needed"
|
||||
|
||||
@ -68,7 +139,7 @@ fi
|
||||
# NOTE: SUPPORTED_VERSION has to match release binaries present in the
|
||||
# etcd image (to make this script work correctly).
|
||||
# We cannot use array since sh doesn't support it.
|
||||
SUPPORTED_VERSIONS_STRING="2.2.1 2.3.7 3.0.17"
|
||||
SUPPORTED_VERSIONS_STRING="2.2.1 2.3.7 3.0.17 3.1.11 3.2.14"
|
||||
SUPPORTED_VERSIONS=$(echo "${SUPPORTED_VERSIONS_STRING}" | tr " " "\n")
|
||||
|
||||
VERSION_FILE="version.txt"
|
||||
@ -96,58 +167,6 @@ if [ -z "$(ls -A ${DATA_DIRECTORY})" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Starts 'etcd' version ${START_VERSION} and writes to it:
|
||||
# 'etcd_version' -> "${START_VERSION}"
|
||||
# Successful write confirms that etcd is up and running.
|
||||
# Sets ETCD_PID at the end.
|
||||
# Returns 0 if etcd was successfully started, non-0 otherwise.
|
||||
start_etcd() {
|
||||
# Use random ports, so that apiserver cannot connect to etcd.
|
||||
ETCD_PORT=18629
|
||||
ETCD_PEER_PORT=2380
|
||||
# Avoid collisions between etcd and event-etcd.
|
||||
case "${DATA_DIRECTORY}" in
|
||||
*event*)
|
||||
ETCD_PORT=18631
|
||||
ETCD_PEER_PORT=2381
|
||||
;;
|
||||
esac
|
||||
local ETCD_CMD="${ETCD:-/usr/local/bin/etcd-${START_VERSION}}"
|
||||
local ETCDCTL_CMD="${ETCDCTL:-/usr/local/bin/etcdctl-${START_VERSION}}"
|
||||
local API_VERSION="$(echo ${START_STORAGE} | cut -c5-5)"
|
||||
if [ "${API_VERSION}" = "2" ]; then
|
||||
ETCDCTL_CMD="${ETCDCTL_CMD} --debug --endpoint=http://127.0.0.1:${ETCD_PORT} set"
|
||||
else
|
||||
ETCDCTL_CMD="${ETCDCTL_CMD} --endpoints=http://127.0.0.1:${ETCD_PORT} put"
|
||||
fi
|
||||
${ETCD_CMD} \
|
||||
--name="etcd-$(hostname)" \
|
||||
--debug \
|
||||
--data-dir=${DATA_DIRECTORY} \
|
||||
--listen-client-urls http://127.0.0.1:${ETCD_PORT} \
|
||||
--advertise-client-urls http://127.0.0.1:${ETCD_PORT} \
|
||||
--listen-peer-urls http://127.0.0.1:${ETCD_PEER_PORT} \
|
||||
--initial-advertise-peer-urls http://127.0.0.1:${ETCD_PEER_PORT} &
|
||||
ETCD_PID=$!
|
||||
# Wait until we can write to etcd.
|
||||
for i in $(seq 240); do
|
||||
sleep 0.5
|
||||
ETCDCTL_API="${API_VERSION}" ${ETCDCTL_CMD} 'etcd_version' ${START_VERSION}
|
||||
if [ "$?" -eq "0" ]; then
|
||||
echo "Etcd on port ${ETCD_PORT} is up."
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
echo "Timeout while waiting for etcd on port ${ETCD_PORT}"
|
||||
return 1
|
||||
}
|
||||
|
||||
# Stops etcd with ${ETCD_PID} pid.
|
||||
stop_etcd() {
|
||||
kill "${ETCD_PID-}" >/dev/null 2>&1 || :
|
||||
wait "${ETCD_PID-}" >/dev/null 2>&1 || :
|
||||
}
|
||||
|
||||
ATTACHLEASE="${ATTACHLEASE:-/usr/local/bin/attachlease}"
|
||||
ROLLBACK="${ROLLBACK:-/usr/local/bin/rollback}"
|
||||
|
||||
@ -163,6 +182,24 @@ if [ "${CURRENT_VERSION}" = "2.2.1" -a "${CURRENT_VERSION}" != "${TARGET_VERSION
|
||||
echo "Backup done in ${BACKUP_DIR}"
|
||||
fi
|
||||
|
||||
CURRENT_MINOR_VERSION="$(echo ${CURRENT_VERSION} | awk -F'.' '{print $2}')"
|
||||
TARGET_MINOR_VERSION="$(echo ${TARGET_VERSION} | awk -F'.' '{print $2}')"
|
||||
|
||||
# "rollback-if-needed"
|
||||
case "${CURRENT_STORAGE}-${TARGET_STORAGE}" in
|
||||
"etcd3-etcd3")
|
||||
[ ${TARGET_MINOR_VERSION} -lt ${CURRENT_MINOR_VERSION} ] && rollback_etcd3_minor_version
|
||||
break
|
||||
;;
|
||||
"etcd3-etcd2")
|
||||
rollback_to_etcd2
|
||||
break
|
||||
;;
|
||||
*)
|
||||
break
|
||||
;;
|
||||
esac
|
||||
|
||||
# Do the roll-forward migration if needed.
|
||||
# The migration goes as following:
|
||||
# 1. for all versions starting one after the current version of etcd
|
||||
@ -227,7 +264,7 @@ for step in ${SUPPORTED_VERSIONS}; do
|
||||
echo "Starting etcd ${step} in v3 mode failed"
|
||||
exit 1
|
||||
fi
|
||||
${ETCDCTL_CMD} rm --recursive "${ETCD_DATA_PREFIX}"
|
||||
${ETCDCTL_CMD} --endpoints "http://127.0.0.1:${ETCD_PORT}" rm --recursive "${ETCD_DATA_PREFIX}"
|
||||
# Kill etcd and wait until this is down.
|
||||
stop_etcd
|
||||
echo "Successfully remove v2 data"
|
||||
@ -239,28 +276,4 @@ for step in ${SUPPORTED_VERSIONS}; do
|
||||
fi
|
||||
done
|
||||
|
||||
# Do the rollback of needed.
|
||||
# NOTE: Rollback is only supported from "3.0.x" version in 'etcd3' mode to
|
||||
# "2.2.1" version in 'etcd2' mode.
|
||||
if [ "${CURRENT_STORAGE}" = "etcd3" -a "${TARGET_STORAGE}" = "etcd2" ]; then
|
||||
if [ "$(echo ${CURRENT_VERSION} | cut -c1-4)" != "3.0." -o "${TARGET_VERSION}" != "2.2.1" ]; then
|
||||
echo "etcd3 -> etcd2 downgrade is supported only between 3.0.x and 2.2.1"
|
||||
return 0
|
||||
fi
|
||||
echo "Backup and remove all existing v2 data"
|
||||
ROLLBACK_BACKUP_DIR="${DATA_DIRECTORY}.bak"
|
||||
rm -rf "${ROLLBACK_BACKUP_DIR}"
|
||||
mkdir -p "${ROLLBACK_BACKUP_DIR}"
|
||||
cp -r "${DATA_DIRECTORY}" "${ROLLBACK_BACKUP_DIR}"
|
||||
echo "Performing etcd3 -> etcd2 rollback"
|
||||
${ROLLBACK} --data-dir "${DATA_DIRECTORY}"
|
||||
if [ "$?" -ne "0" ]; then
|
||||
echo "Rollback to etcd2 failed"
|
||||
exit 1
|
||||
fi
|
||||
CURRENT_STORAGE="etcd2"
|
||||
CURRENT_VERSION="2.2.1"
|
||||
echo "${CURRENT_VERSION}/${CURRENT_STORAGE}" > "${DATA_DIRECTORY}/${VERSION_FILE}"
|
||||
fi
|
||||
|
||||
echo "$(date +'%Y-%m-%d %H:%M:%S') Migration finished"
|
||||
|
||||
68
cluster/images/etcd/start-stop-etcd.sh
Executable file
68
cluster/images/etcd/start-stop-etcd.sh
Executable file
@ -0,0 +1,68 @@
|
||||
#!/bin/sh
|
||||
|
||||
# Copyright 2016 The Kubernetes Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Starts 'etcd' version ${START_VERSION} and writes to it:
|
||||
# 'etcd_version' -> "${START_VERSION}"
|
||||
# Successful write confirms that etcd is up and running.
|
||||
# Sets ETCD_PID at the end.
|
||||
# Returns 0 if etcd was successfully started, non-0 otherwise.
|
||||
start_etcd() {
|
||||
# Use random ports, so that apiserver cannot connect to etcd.
|
||||
ETCD_PORT=18629
|
||||
ETCD_PEER_PORT=2380
|
||||
# Avoid collisions between etcd and event-etcd.
|
||||
case "${DATA_DIRECTORY}" in
|
||||
*event*)
|
||||
ETCD_PORT=18631
|
||||
ETCD_PEER_PORT=2381
|
||||
;;
|
||||
esac
|
||||
local ETCD_CMD="${ETCD:-/usr/local/bin/etcd-${START_VERSION}}"
|
||||
local ETCDCTL_CMD="${ETCDCTL:-/usr/local/bin/etcdctl-${START_VERSION}}"
|
||||
local API_VERSION="$(echo ${START_STORAGE} | cut -c5-5)"
|
||||
if [ "${API_VERSION}" = "2" ]; then
|
||||
ETCDCTL_CMD="${ETCDCTL_CMD} --debug --endpoint=http://127.0.0.1:${ETCD_PORT} set"
|
||||
else
|
||||
ETCDCTL_CMD="${ETCDCTL_CMD} --endpoints=http://127.0.0.1:${ETCD_PORT} put"
|
||||
fi
|
||||
${ETCD_CMD} \
|
||||
--name="etcd-$(hostname)" \
|
||||
--initial-cluster="etcd-$(hostname)=http://127.0.0.1:${ETCD_PEER_PORT}" \
|
||||
--debug \
|
||||
--data-dir=${DATA_DIRECTORY} \
|
||||
--listen-client-urls http://127.0.0.1:${ETCD_PORT} \
|
||||
--advertise-client-urls http://127.0.0.1:${ETCD_PORT} \
|
||||
--listen-peer-urls http://127.0.0.1:${ETCD_PEER_PORT} \
|
||||
--initial-advertise-peer-urls http://127.0.0.1:${ETCD_PEER_PORT} &
|
||||
ETCD_PID=$!
|
||||
# Wait until we can write to etcd.
|
||||
for i in $(seq 240); do
|
||||
sleep 0.5
|
||||
ETCDCTL_API="${API_VERSION}" ${ETCDCTL_CMD} 'etcd_version' ${START_VERSION}
|
||||
if [ "$?" -eq "0" ]; then
|
||||
echo "Etcd on port ${ETCD_PORT} is up."
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
echo "Timeout while waiting for etcd on port ${ETCD_PORT}"
|
||||
return 1
|
||||
}
|
||||
|
||||
# Stops etcd with ${ETCD_PID} pid.
|
||||
stop_etcd() {
|
||||
kill "${ETCD_PID-}" >/dev/null 2>&1 || :
|
||||
wait "${ETCD_PID-}" >/dev/null 2>&1 || :
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user