From 746e247e8750acb5f635132c74e0e28f2ac9dc73 Mon Sep 17 00:00:00 2001 From: Joe Betz Date: Thu, 1 Feb 2018 16:55:58 -0800 Subject: [PATCH] Add etcd 3.x minor version rollback support to migrate-if-needed.sh --- cluster/gce/manifests/etcd.manifest | 3 + cluster/images/etcd/Dockerfile | 2 +- cluster/images/etcd/Makefile | 134 +++++++++++++++++- cluster/images/etcd/README.md | 8 ++ cluster/images/etcd/migrate-if-needed.sh | 173 ++++++++++++----------- cluster/images/etcd/start-stop-etcd.sh | 68 +++++++++ 6 files changed, 302 insertions(+), 86 deletions(-) create mode 100755 cluster/images/etcd/start-stop-etcd.sh diff --git a/cluster/gce/manifests/etcd.manifest b/cluster/gce/manifests/etcd.manifest index 8fbc0afcc11..0983145ca19 100644 --- a/cluster/gce/manifests/etcd.manifest +++ b/cluster/gce/manifests/etcd.manifest @@ -33,6 +33,9 @@ }, { "name": "DATA_DIRECTORY", "value": "/var/etcd/data{{ suffix }}" + }, + { "name": "INITIAL_CLUSTER", + "value": "{{ etcd_cluster }}" } ], "livenessProbe": { diff --git a/cluster/images/etcd/Dockerfile b/cluster/images/etcd/Dockerfile index 1d75d1787f4..5e7eaf9f51b 100644 --- a/cluster/images/etcd/Dockerfile +++ b/cluster/images/etcd/Dockerfile @@ -16,4 +16,4 @@ FROM BASEIMAGE EXPOSE 2379 2380 4001 7001 COPY etcd* etcdctl* /usr/local/bin/ -COPY migrate-if-needed.sh attachlease rollback /usr/local/bin/ +COPY migrate-if-needed.sh start-stop-etcd.sh attachlease rollback /usr/local/bin/ diff --git a/cluster/images/etcd/Makefile b/cluster/images/etcd/Makefile index 401811cad10..f0477d4555e 100644 --- a/cluster/images/etcd/Makefile +++ b/cluster/images/etcd/Makefile @@ -28,6 +28,8 @@ # That binary will be set to the last tag from $(TAGS). TAGS?=2.2.1 2.3.7 3.0.17 3.1.11 3.2.14 REGISTRY_TAG?=3.2.14 +# ROLLBACK_REGISTRY_TAG specified the tag that REGISTRY_TAG may be rolled back to. +ROLLBACK_REGISTRY_TAG?=3.1.11 ARCH?=amd64 REGISTRY?=k8s.gcr.io # golang version should match the golang version from https://github.com/coreos/etcd/releases for REGISTRY_TAG version of etcd. @@ -57,10 +59,10 @@ build: find ./ -maxdepth 1 -type f | xargs -I {} cp {} $(TEMP_DIR) # Compile attachlease - docker run -i -v $(shell pwd)/../../../:/go/src/k8s.io/kubernetes -v $(TEMP_DIR):/build -e GOARCH=$(ARCH) golang:$(GOLANG_VERSION) \ + docker run --interactive -v $(shell pwd)/../../../:/go/src/k8s.io/kubernetes -v $(TEMP_DIR):/build -e GOARCH=$(ARCH) golang:$(GOLANG_VERSION) \ /bin/bash -c "CGO_ENABLED=0 go build -o /build/attachlease k8s.io/kubernetes/cluster/images/etcd/attachlease" # Compile rollback - docker run -i -v $(shell pwd)/../../../:/go/src/k8s.io/kubernetes -v $(TEMP_DIR):/build -e GOARCH=$(ARCH) golang:$(GOLANG_VERSION) \ + docker run --interactive -v $(shell pwd)/../../../:/go/src/k8s.io/kubernetes -v $(TEMP_DIR):/build -e GOARCH=$(ARCH) golang:$(GOLANG_VERSION) \ /bin/bash -c "CGO_ENABLED=0 go build -o /build/rollback k8s.io/kubernetes/cluster/images/etcd/rollback" @@ -81,7 +83,7 @@ else # For each release create a tmp dir 'etcd_release_tmp_dir' and unpack the release tar there. for tag in $(TAGS); do \ etcd_release_tmp_dir=$(shell mktemp -d); \ - docker run -i -v $$etcd_release_tmp_dir:/etcdbin golang:$(GOLANG_VERSION) /bin/bash -c \ + docker run --interactive -v $$etcd_release_tmp_dir:/etcdbin golang:$(GOLANG_VERSION) /bin/bash -c \ "git clone https://github.com/coreos/etcd /go/src/github.com/coreos/etcd \ && cd /go/src/github.com/coreos/etcd \ && git checkout v$$tag \ @@ -114,5 +116,127 @@ ifeq ($(ARCH),amd64) gcloud docker -- push $(REGISTRY)/etcd:$(REGISTRY_TAG) endif -all: build -.PHONY: build push +ETCD2_ROLLBACK_NEW_TAG=3.0.17 +ETCD2_ROLLBACK_OLD_TAG=2.2.1 + +# Test a rollback to etcd2 from the earliest etcd3 version. +test-rollback-etcd2: + mkdir -p $(TEMP_DIR)/rollback-etcd2 + cd $(TEMP_DIR)/rollback-etcd2 + + @echo "Starting $(ETCD2_ROLLBACK_NEW_TAG) etcd and writing some sample data." + docker run --tty --interactive -v $(TEMP_DIR)/rollback-etcd2:/var/etcd \ + -e "TARGET_STORAGE=etcd3" \ + -e "TARGET_VERSION=$(ETCD2_ROLLBACK_NEW_TAG)" \ + -e "DATA_DIRECTORY=/var/etcd/data" \ + gcr.io/google_containers/etcd-$(ARCH):$(REGISTRY_TAG) /bin/sh -c \ + 'INITIAL_CLUSTER=etcd-$$(hostname)=http://localhost:2380 \ + /usr/local/bin/migrate-if-needed.sh && \ + source /usr/local/bin/start-stop-etcd.sh && \ + START_STORAGE=etcd3 START_VERSION=$(ETCD2_ROLLBACK_NEW_TAG) start_etcd && \ + ETCDCTL_API=3 /usr/local/bin/etcdctl-$(ETCD2_ROLLBACK_NEW_TAG) --endpoints http://127.0.0.1:$${ETCD_PORT} put /registry/k1 value1 && \ + stop_etcd && \ + [ $$(cat /var/etcd/data/version.txt) = $(ETCD2_ROLLBACK_NEW_TAG)/etcd3 ]' + + @echo "Rolling back to the previous version of etcd and recording keyspace to a flat file." + docker run --tty --interactive -v $(TEMP_DIR)/rollback-etcd2:/var/etcd \ + -e "TARGET_STORAGE=etcd2" \ + -e "TARGET_VERSION=$(ETCD2_ROLLBACK_OLD_TAG)" \ + -e "DATA_DIRECTORY=/var/etcd/data" \ + gcr.io/google_containers/etcd-$(ARCH):$(REGISTRY_TAG) /bin/sh -c \ + 'INITIAL_CLUSTER=etcd-$$(hostname)=http://localhost:2380 \ + /usr/local/bin/migrate-if-needed.sh && \ + source /usr/local/bin/start-stop-etcd.sh && \ + START_STORAGE=etcd2 START_VERSION=$(ETCD2_ROLLBACK_OLD_TAG) start_etcd && \ + /usr/local/bin/etcdctl-$(ETCD2_ROLLBACK_OLD_TAG) --endpoint 127.0.0.1:$${ETCD_PORT} get /registry/k1 > /var/etcd/keyspace.txt && \ + stop_etcd' + + @echo "Checking if rollback successfully downgraded etcd to $(ETCD2_ROLLBACK_OLD_TAG)" + docker run --tty --interactive -v $(TEMP_DIR)/rollback-etcd2:/var/etcd \ + gcr.io/google_containers/etcd-$(ARCH):$(REGISTRY_TAG) /bin/sh -c \ + '[ $$(cat /var/etcd/data/version.txt) = $(ETCD2_ROLLBACK_OLD_TAG)/etcd2 ] && \ + grep -q value1 /var/etcd/keyspace.txt' + +# Test a rollback from the latest version to the previous version. +test-rollback: + mkdir -p $(TEMP_DIR)/rollback-test + cd $(TEMP_DIR)/rollback-test + + @echo "Starting $(REGISTRY_TAG) etcd and writing some sample data." + docker run --tty --interactive -v $(TEMP_DIR)/rollback-test:/var/etcd \ + -e "TARGET_STORAGE=etcd3" \ + -e "TARGET_VERSION=$(REGISTRY_TAG)" \ + -e "DATA_DIRECTORY=/var/etcd/data" \ + gcr.io/google_containers/etcd-$(ARCH):$(REGISTRY_TAG) /bin/sh -c \ + 'INITIAL_CLUSTER=etcd-$$(hostname)=http://localhost:2380 \ + /usr/local/bin/migrate-if-needed.sh && \ + source /usr/local/bin/start-stop-etcd.sh && \ + START_STORAGE=etcd3 START_VERSION=$(REGISTRY_TAG) start_etcd && \ + ETCDCTL_API=3 /usr/local/bin/etcdctl --endpoints http://127.0.0.1:$${ETCD_PORT} put /registry/k1 value1 && \ + stop_etcd' + + @echo "Rolling back to the previous version of etcd and recording keyspace to a flat file." + docker run --tty --interactive -v $(TEMP_DIR)/rollback-test:/var/etcd \ + -e "TARGET_STORAGE=etcd3" \ + -e "TARGET_VERSION=$(ROLLBACK_REGISTRY_TAG)" \ + -e "DATA_DIRECTORY=/var/etcd/data" \ + gcr.io/google_containers/etcd-$(ARCH):$(REGISTRY_TAG) /bin/sh -c \ + 'INITIAL_CLUSTER=etcd-$$(hostname)=http://localhost:2380 \ + /usr/local/bin/migrate-if-needed.sh && \ + source /usr/local/bin/start-stop-etcd.sh && \ + START_STORAGE=etcd3 START_VERSION=$(ROLLBACK_REGISTRY_TAG) start_etcd && \ + ETCDCTL_API=3 /usr/local/bin/etcdctl --endpoints http://127.0.0.1:$${ETCD_PORT} get --prefix / > /var/etcd/keyspace.txt && \ + stop_etcd' + + @echo "Checking if rollback successfully downgraded etcd to $(ROLLBACK_REGISTRY_TAG)" + docker run --tty --interactive -v $(TEMP_DIR)/rollback-test:/var/etcd \ + gcr.io/google_containers/etcd-$(ARCH):$(REGISTRY_TAG) /bin/sh -c \ + '[ $$(cat /var/etcd/data/version.txt) = $(ROLLBACK_REGISTRY_TAG)/etcd3 ] && \ + grep -q value1 /var/etcd/keyspace.txt' + +# Test migrating from each supported versions to the latest version. +test-migrate: + for tag in $(TAGS); do \ + echo "Testing migration from $${tag} to $(REGISTRY_TAG)" && \ + mkdir -p $(TEMP_DIR)/migrate-$${tag} && \ + cd $(TEMP_DIR)/migrate-$${tag} && \ + MAJOR_VERSION=$$(echo $${tag} | cut -c 1) && \ + echo "Starting etcd $${tag} and writing sample data to keyspace" && \ + docker run --tty --interactive -v $(TEMP_DIR)/migrate-$${tag}:/var/etcd \ + -e "TARGET_STORAGE=etcd$${MAJOR_VERSION}" \ + -e "TARGET_VERSION=$${tag}" \ + -e "DATA_DIRECTORY=/var/etcd/data" \ + gcr.io/google_containers/etcd-$(ARCH):$(REGISTRY_TAG) /bin/sh -c \ + "INITIAL_CLUSTER=etcd-\$$(hostname)=http://localhost:2380 \ + /usr/local/bin/migrate-if-needed.sh && \ + source /usr/local/bin/start-stop-etcd.sh && \ + START_STORAGE=etcd$${MAJOR_VERSION} START_VERSION=$${tag} start_etcd && \ + if [ $${MAJOR_VERSION} == 2 ]; then \ + /usr/local/bin/etcdctl --endpoint http://127.0.0.1:\$${ETCD_PORT} set /registry/k1 value1; \ + else \ + ETCDCTL_API=3 /usr/local/bin/etcdctl --endpoints http://127.0.0.1:\$${ETCD_PORT} put /registry/k1 value1; \ + fi && \ + stop_etcd" && \ + echo " Migrating from $${tag} to $(REGISTRY_TAG) and capturing keyspace" && \ + docker run --tty --interactive -v $(TEMP_DIR)/migrate-$${tag}:/var/etcd \ + -e "TARGET_STORAGE=etcd3" \ + -e "TARGET_VERSION=$(REGISTRY_TAG)" \ + -e "DATA_DIRECTORY=/var/etcd/data" \ + gcr.io/google_containers/etcd-$(ARCH):$(REGISTRY_TAG) /bin/sh -c \ + 'INITIAL_CLUSTER=etcd-$$(hostname)=http://localhost:2380 \ + /usr/local/bin/migrate-if-needed.sh && \ + source /usr/local/bin/start-stop-etcd.sh && \ + START_STORAGE=etcd3 START_VERSION=$(REGISTRY_TAG) start_etcd && \ + ETCDCTL_API=3 /usr/local/bin/etcdctl --endpoints http://127.0.0.1:$${ETCD_PORT} get --prefix / > /var/etcd/keyspace.txt && \ + stop_etcd' && \ + echo "Checking if migrate from $${tag} successfully upgraded etcd to $(REGISTRY_TAG)" && \ + docker run --tty --interactive -v $(TEMP_DIR)/migrate-$${tag}:/var/etcd \ + gcr.io/google_containers/etcd-$(ARCH):$(REGISTRY_TAG) /bin/sh -c \ + '[ $$(cat /var/etcd/data/version.txt) = $(REGISTRY_TAG)/etcd3 ] && \ + grep -q value1 /var/etcd/keyspace.txt'; \ + done + +test: test-rollback test-rollback-etcd2 test-migrate + +all: build test +.PHONY: build push test-rollback test-rollback-etcd2 test-migrate test diff --git a/cluster/images/etcd/README.md b/cluster/images/etcd/README.md index df5cfe5b3d7..8e9c9cb5de4 100644 --- a/cluster/images/etcd/README.md +++ b/cluster/images/etcd/README.md @@ -7,6 +7,14 @@ For other architectures, `etcd` is cross-compiled from source. Arch-specific `bu #### How to release +First, run the migration and rollback tests. + +```console +$ make build test +``` + +Next, build and push the docker images for all supported architectures. + ```console # Build for linux/amd64 (default) $ make push ARCH=amd64 diff --git a/cluster/images/etcd/migrate-if-needed.sh b/cluster/images/etcd/migrate-if-needed.sh index 9e65f926d8c..5ef718fae42 100755 --- a/cluster/images/etcd/migrate-if-needed.sh +++ b/cluster/images/etcd/migrate-if-needed.sh @@ -18,7 +18,7 @@ # This script performs etcd upgrade based on the following environmental # variables: # TARGET_STORAGE - API of etcd to be used (supported: 'etcd2', 'etcd3') -# TARGET_VERSION - etcd release to be used (supported: '2.2.1', '2.3.7', '3.0.17') +# TARGET_VERSION - etcd release to be used (supported: '2.2.1', '2.3.7', '3.0.17', '3.1.11', '3.2.14') # DATA_DIRECTORY - directory with etcd data # # The current etcd version and storage format is detected based on the @@ -28,7 +28,8 @@ # The update workflow support the following upgrade steps: # - 2.2.1/etcd2 -> 2.3.7/etcd2 # - 2.3.7/etcd2 -> 3.0.17/etcd2 -# - 3.0.17/etcd2 -> 3.0.17/etcd3 +# - 3.0.17/etcd3 -> 3.1.11/etcd3 +# - 3.1.11/etcd3 -> 3.2.14/etcd3 # # NOTE: The releases supported in this script has to match release binaries # present in the etcd image (to make this script work correctly). @@ -39,6 +40,72 @@ set -o errexit set -o nounset +source $(dirname "$0")/start-stop-etcd.sh + +# Rollback to previous minor version of etcd 3.x, if needed. +# +# Warning: For HA etcd clusters (any cluster with more than one member), all members must be stopped before rolling back, zero +# downtime rollbacks are not supported. +rollback_etcd3_minor_version() { + if [ ${TARGET_MINOR_VERSION} != $((${CURRENT_MINOR_VERSION}-1)) ]; then + echo "Rollback from ${CURRENT_VERSION} to ${TARGET_VERSION} not supported, only rollbacks to the previous minor version are supported." + exit 1 + fi + echo "Performing etcd ${CURRENT_VERSION} -> ${TARGET_VERSION} rollback" + ROLLBACK_BACKUP_DIR="${DATA_DIRECTORY}.bak" + rm -rf "${ROLLBACK_BACKUP_DIR}" + SNAPSHOT_FILE="${DATA_DIRECTORY}.snapshot.db" + rm -rf "${SNAPSHOT_FILE}" + ETCD_CMD="/usr/local/bin/etcd-${CURRENT_VERSION}" + ETCDCTL_CMD="/usr/local/bin/etcdctl-${CURRENT_VERSION}" + + # Start CURRENT_VERSION of etcd. + START_VERSION="${CURRENT_VERSION}" + START_STORAGE="${CURRENT_STORAGE}" + echo "Starting etcd version ${START_VERSION} to capture rollback snapshot." + if ! start_etcd; then + echo "Unable to automatically downgrade etcd: starting etcd version ${START_VERSION} to capture rollback snapshot failed." + echo "See https://coreos.com/etcd/docs/3.2.13/op-guide/recovery.html for manual downgrade options." + exit 1 + else + ETCDCTL_API=3 ${ETCDCTL_CMD} snapshot --endpoints "http://127.0.0.1:${ETCD_PORT}" save "${SNAPSHOT_FILE}" + fi + stop_etcd + + # Backup the data before rolling back. + mv "${DATA_DIRECTORY}" "${ROLLBACK_BACKUP_DIR}" + ETCDCTL_CMD="/usr/local/bin/etcdctl-${TARGET_VERSION}" + NAME="etcd-$(hostname)" + ETCDCTL_API=3 ${ETCDCTL_CMD} snapshot restore "${SNAPSHOT_FILE}" \ + --data-dir "${DATA_DIRECTORY}" --name "${NAME}" --initial-cluster "${INITIAL_CLUSTER}" + + CURRENT_VERSION="${TARGET_VERSION}" + echo "${CURRENT_VERSION}/${CURRENT_STORAGE}" > "${DATA_DIRECTORY}/${VERSION_FILE}" +} + +# Rollback from "3.0.x" version in 'etcd3' mode to "2.2.1" version in 'etcd2' mode, if needed. +rollback_to_etcd2() { + if [ "$(echo ${CURRENT_VERSION} | cut -c1-4)" != "3.0." -o "${TARGET_VERSION}" != "2.2.1" ]; then + echo "etcd3 -> etcd2 downgrade is supported only between 3.0.x and 2.2.1" + return 0 + fi + echo "Backup and remove all existing v2 data" + ROLLBACK_BACKUP_DIR="${DATA_DIRECTORY}.bak" + rm -rf "${ROLLBACK_BACKUP_DIR}" + mkdir -p "${ROLLBACK_BACKUP_DIR}" + cp -r "${DATA_DIRECTORY}" "${ROLLBACK_BACKUP_DIR}" + echo "Performing etcd3 -> etcd2 rollback" + ${ROLLBACK} --data-dir "${DATA_DIRECTORY}" + if [ "$?" -ne "0" ]; then + echo "Rollback to etcd2 failed" + exit 1 + fi + CURRENT_STORAGE="etcd2" + CURRENT_VERSION="2.2.1" + echo "${CURRENT_VERSION}/${CURRENT_STORAGE}" > "${DATA_DIRECTORY}/${VERSION_FILE}" +} + + if [ -z "${TARGET_STORAGE:-}" ]; then echo "TARGET_STORAGE variable unset - unexpected failure" exit 1 @@ -51,6 +118,10 @@ if [ -z "${DATA_DIRECTORY:-}" ]; then echo "DATA_DIRECTORY variable unset - unexpected failure" exit 1 fi +if [ -z "${INITIAL_CLUSTER:-}" ]; then + echo "Warn: INITIAL_CLUSTER variable unset - defaulting to etcd-$(hostname)=http://localhost:2380" + INITIAL_CLUSTER="etcd-$(hostname)=http://localhost:2380" +fi echo "$(date +'%Y-%m-%d %H:%M:%S') Detecting if migration is needed" @@ -68,7 +139,7 @@ fi # NOTE: SUPPORTED_VERSION has to match release binaries present in the # etcd image (to make this script work correctly). # We cannot use array since sh doesn't support it. -SUPPORTED_VERSIONS_STRING="2.2.1 2.3.7 3.0.17" +SUPPORTED_VERSIONS_STRING="2.2.1 2.3.7 3.0.17 3.1.11 3.2.14" SUPPORTED_VERSIONS=$(echo "${SUPPORTED_VERSIONS_STRING}" | tr " " "\n") VERSION_FILE="version.txt" @@ -96,58 +167,6 @@ if [ -z "$(ls -A ${DATA_DIRECTORY})" ]; then exit 0 fi -# Starts 'etcd' version ${START_VERSION} and writes to it: -# 'etcd_version' -> "${START_VERSION}" -# Successful write confirms that etcd is up and running. -# Sets ETCD_PID at the end. -# Returns 0 if etcd was successfully started, non-0 otherwise. -start_etcd() { - # Use random ports, so that apiserver cannot connect to etcd. - ETCD_PORT=18629 - ETCD_PEER_PORT=2380 - # Avoid collisions between etcd and event-etcd. - case "${DATA_DIRECTORY}" in - *event*) - ETCD_PORT=18631 - ETCD_PEER_PORT=2381 - ;; - esac - local ETCD_CMD="${ETCD:-/usr/local/bin/etcd-${START_VERSION}}" - local ETCDCTL_CMD="${ETCDCTL:-/usr/local/bin/etcdctl-${START_VERSION}}" - local API_VERSION="$(echo ${START_STORAGE} | cut -c5-5)" - if [ "${API_VERSION}" = "2" ]; then - ETCDCTL_CMD="${ETCDCTL_CMD} --debug --endpoint=http://127.0.0.1:${ETCD_PORT} set" - else - ETCDCTL_CMD="${ETCDCTL_CMD} --endpoints=http://127.0.0.1:${ETCD_PORT} put" - fi - ${ETCD_CMD} \ - --name="etcd-$(hostname)" \ - --debug \ - --data-dir=${DATA_DIRECTORY} \ - --listen-client-urls http://127.0.0.1:${ETCD_PORT} \ - --advertise-client-urls http://127.0.0.1:${ETCD_PORT} \ - --listen-peer-urls http://127.0.0.1:${ETCD_PEER_PORT} \ - --initial-advertise-peer-urls http://127.0.0.1:${ETCD_PEER_PORT} & - ETCD_PID=$! - # Wait until we can write to etcd. - for i in $(seq 240); do - sleep 0.5 - ETCDCTL_API="${API_VERSION}" ${ETCDCTL_CMD} 'etcd_version' ${START_VERSION} - if [ "$?" -eq "0" ]; then - echo "Etcd on port ${ETCD_PORT} is up." - return 0 - fi - done - echo "Timeout while waiting for etcd on port ${ETCD_PORT}" - return 1 -} - -# Stops etcd with ${ETCD_PID} pid. -stop_etcd() { - kill "${ETCD_PID-}" >/dev/null 2>&1 || : - wait "${ETCD_PID-}" >/dev/null 2>&1 || : -} - ATTACHLEASE="${ATTACHLEASE:-/usr/local/bin/attachlease}" ROLLBACK="${ROLLBACK:-/usr/local/bin/rollback}" @@ -163,6 +182,24 @@ if [ "${CURRENT_VERSION}" = "2.2.1" -a "${CURRENT_VERSION}" != "${TARGET_VERSION echo "Backup done in ${BACKUP_DIR}" fi +CURRENT_MINOR_VERSION="$(echo ${CURRENT_VERSION} | awk -F'.' '{print $2}')" +TARGET_MINOR_VERSION="$(echo ${TARGET_VERSION} | awk -F'.' '{print $2}')" + +# "rollback-if-needed" +case "${CURRENT_STORAGE}-${TARGET_STORAGE}" in + "etcd3-etcd3") + [ ${TARGET_MINOR_VERSION} -lt ${CURRENT_MINOR_VERSION} ] && rollback_etcd3_minor_version + break + ;; + "etcd3-etcd2") + rollback_to_etcd2 + break + ;; + *) + break + ;; +esac + # Do the roll-forward migration if needed. # The migration goes as following: # 1. for all versions starting one after the current version of etcd @@ -227,7 +264,7 @@ for step in ${SUPPORTED_VERSIONS}; do echo "Starting etcd ${step} in v3 mode failed" exit 1 fi - ${ETCDCTL_CMD} rm --recursive "${ETCD_DATA_PREFIX}" + ${ETCDCTL_CMD} --endpoints "http://127.0.0.1:${ETCD_PORT}" rm --recursive "${ETCD_DATA_PREFIX}" # Kill etcd and wait until this is down. stop_etcd echo "Successfully remove v2 data" @@ -239,28 +276,4 @@ for step in ${SUPPORTED_VERSIONS}; do fi done -# Do the rollback of needed. -# NOTE: Rollback is only supported from "3.0.x" version in 'etcd3' mode to -# "2.2.1" version in 'etcd2' mode. -if [ "${CURRENT_STORAGE}" = "etcd3" -a "${TARGET_STORAGE}" = "etcd2" ]; then - if [ "$(echo ${CURRENT_VERSION} | cut -c1-4)" != "3.0." -o "${TARGET_VERSION}" != "2.2.1" ]; then - echo "etcd3 -> etcd2 downgrade is supported only between 3.0.x and 2.2.1" - return 0 - fi - echo "Backup and remove all existing v2 data" - ROLLBACK_BACKUP_DIR="${DATA_DIRECTORY}.bak" - rm -rf "${ROLLBACK_BACKUP_DIR}" - mkdir -p "${ROLLBACK_BACKUP_DIR}" - cp -r "${DATA_DIRECTORY}" "${ROLLBACK_BACKUP_DIR}" - echo "Performing etcd3 -> etcd2 rollback" - ${ROLLBACK} --data-dir "${DATA_DIRECTORY}" - if [ "$?" -ne "0" ]; then - echo "Rollback to etcd2 failed" - exit 1 - fi - CURRENT_STORAGE="etcd2" - CURRENT_VERSION="2.2.1" - echo "${CURRENT_VERSION}/${CURRENT_STORAGE}" > "${DATA_DIRECTORY}/${VERSION_FILE}" -fi - echo "$(date +'%Y-%m-%d %H:%M:%S') Migration finished" diff --git a/cluster/images/etcd/start-stop-etcd.sh b/cluster/images/etcd/start-stop-etcd.sh new file mode 100755 index 00000000000..5a8ba8b7f18 --- /dev/null +++ b/cluster/images/etcd/start-stop-etcd.sh @@ -0,0 +1,68 @@ +#!/bin/sh + +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Starts 'etcd' version ${START_VERSION} and writes to it: +# 'etcd_version' -> "${START_VERSION}" +# Successful write confirms that etcd is up and running. +# Sets ETCD_PID at the end. +# Returns 0 if etcd was successfully started, non-0 otherwise. +start_etcd() { + # Use random ports, so that apiserver cannot connect to etcd. + ETCD_PORT=18629 + ETCD_PEER_PORT=2380 + # Avoid collisions between etcd and event-etcd. + case "${DATA_DIRECTORY}" in + *event*) + ETCD_PORT=18631 + ETCD_PEER_PORT=2381 + ;; + esac + local ETCD_CMD="${ETCD:-/usr/local/bin/etcd-${START_VERSION}}" + local ETCDCTL_CMD="${ETCDCTL:-/usr/local/bin/etcdctl-${START_VERSION}}" + local API_VERSION="$(echo ${START_STORAGE} | cut -c5-5)" + if [ "${API_VERSION}" = "2" ]; then + ETCDCTL_CMD="${ETCDCTL_CMD} --debug --endpoint=http://127.0.0.1:${ETCD_PORT} set" + else + ETCDCTL_CMD="${ETCDCTL_CMD} --endpoints=http://127.0.0.1:${ETCD_PORT} put" + fi + ${ETCD_CMD} \ + --name="etcd-$(hostname)" \ + --initial-cluster="etcd-$(hostname)=http://127.0.0.1:${ETCD_PEER_PORT}" \ + --debug \ + --data-dir=${DATA_DIRECTORY} \ + --listen-client-urls http://127.0.0.1:${ETCD_PORT} \ + --advertise-client-urls http://127.0.0.1:${ETCD_PORT} \ + --listen-peer-urls http://127.0.0.1:${ETCD_PEER_PORT} \ + --initial-advertise-peer-urls http://127.0.0.1:${ETCD_PEER_PORT} & + ETCD_PID=$! + # Wait until we can write to etcd. + for i in $(seq 240); do + sleep 0.5 + ETCDCTL_API="${API_VERSION}" ${ETCDCTL_CMD} 'etcd_version' ${START_VERSION} + if [ "$?" -eq "0" ]; then + echo "Etcd on port ${ETCD_PORT} is up." + return 0 + fi + done + echo "Timeout while waiting for etcd on port ${ETCD_PORT}" + return 1 +} + +# Stops etcd with ${ETCD_PID} pid. +stop_etcd() { + kill "${ETCD_PID-}" >/dev/null 2>&1 || : + wait "${ETCD_PID-}" >/dev/null 2>&1 || : +}