diff --git a/cluster/images/etcd/migrate-if-needed.sh b/cluster/images/etcd/migrate-if-needed.sh index 83cdca9436b..36ad4190af1 100755 --- a/cluster/images/etcd/migrate-if-needed.sh +++ b/cluster/images/etcd/migrate-if-needed.sh @@ -23,7 +23,6 @@ set -o errexit set -o nounset -set -o pipefail if [ -z "${TARGET_STORAGE:-}" ]; then echo "TARGET_USAGE variable unset - skipping migration" @@ -35,6 +34,7 @@ if [ -z "${DATA_DIRECTORY:-}" ]; then exit 0 fi +ETCDCTL="${ETCDCTL:-/usr/local/bin/etcdctl}" VERSION_FILE="version.txt" CURRENT_STORAGE='etcd2' if [ -e "${DATA_DIRECTORY}/${VERSION_FILE}" ]; then @@ -48,7 +48,8 @@ if [ "${CURRENT_STORAGE}" = "etcd2" -a "${TARGET_STORAGE}" = "etcd3" ]; then if [ "$(ls -A ${DATA_DIRECTORY})" ]; then echo "Performing etcd2 -> etcd3 migration" # TODO: Pass a correct transformer to handle TTLs. - ETCDCTL_API=3 /usr/local/bin/etcdctl migrate --data-dir=${DATA_DIRECTORY} + echo "ETCDCTL_API=3 ${ETCDCTL} migrate --data-dir=${DATA_DIRECTORY}" + ETCDCTL_API=3 ${ETCDCTL} migrate --data-dir=${DATA_DIRECTORY} fi fi fi diff --git a/hack/lib/etcd.sh b/hack/lib/etcd.sh index 13b4b709568..2c3ebebabdf 100644 --- a/hack/lib/etcd.sh +++ b/hack/lib/etcd.sh @@ -40,7 +40,7 @@ kube::etcd::start() { fi # Start etcd - ETCD_DIR=$(mktemp -d 2>/dev/null || mktemp -d -t test-etcd.XXXXXX) + ETCD_DIR=${ETCD_DIR:-$(mktemp -d 2>/dev/null || mktemp -d -t test-etcd.XXXXXX)} if [[ -d "${ARTIFACTS_DIR:-}" ]]; then ETCD_LOGFILE="${ARTIFACTS_DIR}/etcd.$(uname -n).$(id -un).log.DEBUG.$(date +%Y%m%d-%H%M%S).$$" else diff --git a/hack/test-update-storage-objects.sh b/hack/test-update-storage-objects.sh index e282c6439d2..25cb74d32c9 100755 --- a/hack/test-update-storage-objects.sh +++ b/hack/test-update-storage-objects.sh @@ -32,6 +32,9 @@ KUBE_NEW_API_VERSION=${KUBE_NEW_API_VERSION:-"v1"} KUBE_OLD_STORAGE_VERSIONS=${KUBE_OLD_STORAGE_VERSIONs:-""} KUBE_NEW_STORAGE_VERSIONS=${KUBE_NEW_STORAGE_VERSIONs:-""} +STORAGE_BACKEND_ETCD2="etcd2" +STORAGE_BACKEND_ETCD3="etcd3" + KUBE_STORAGE_MEDIA_TYPE_JSON="application/json" KUBE_STORAGE_MEDIA_TYPE_PROTOBUF="application/vnd.kubernetes.protobuf" @@ -43,13 +46,16 @@ API_HOST=${API_HOST:-127.0.0.1} KUBE_API_VERSIONS="" RUNTIME_CONFIG="" +ETCDCTL=$(which etcdctl) KUBECTL="${KUBE_OUTPUT_HOSTBIN}/kubectl" UPDATE_ETCD_OBJECTS_SCRIPT="${KUBE_ROOT}/cluster/update-storage-objects.sh" function startApiServer() { - local storage_versions=${1:-""} - local storage_media_type=${2:-""} + local storage_backend=${1:-"${STORAGE_BACKEND_ETCD2}"} + local storage_versions=${2:-""} + local storage_media_type=${3:-""} kube::log::status "Starting kube-apiserver with KUBE_API_VERSIONS: ${KUBE_API_VERSIONS}" + kube::log::status " and storage-backend: ${storage_backend}" kube::log::status " and storage-media-type: ${storage_media_type}" kube::log::status " and runtime-config: ${RUNTIME_CONFIG}" kube::log::status " and storage-version overrides: ${storage_versions}" @@ -59,8 +65,9 @@ function startApiServer() { --insecure-bind-address="${API_HOST}" \ --bind-address="${API_HOST}" \ --insecure-port="${API_PORT}" \ + --storage-backend="${storage_backend}" \ --etcd-servers="http://${ETCD_HOST}:${ETCD_PORT}" \ - --etcd-prefix="${ETCD_PREFIX}" \ + --etcd-prefix="/${ETCD_PREFIX}" \ --runtime-config="${RUNTIME_CONFIG}" \ --cert-dir="${TMPDIR:-/tmp/}" \ --service-cluster-ip-range="10.0.0.0/24" \ @@ -120,7 +127,7 @@ KUBE_NEW_STORAGE_VERSIONS="batch/v1,autoscaling/v1" ####################################################### KUBE_API_VERSIONS="${KUBE_OLD_API_VERSION},${KUBE_NEW_API_VERSION}" RUNTIME_CONFIG="api/all=false,api/${KUBE_OLD_API_VERSION}=true,api/${KUBE_NEW_API_VERSION}=true" -startApiServer ${KUBE_OLD_STORAGE_VERSIONS} ${KUBE_STORAGE_MEDIA_TYPE_JSON} +startApiServer ${STORAGE_BACKEND_ETCD2} ${KUBE_OLD_STORAGE_VERSIONS} ${KUBE_STORAGE_MEDIA_TYPE_JSON} # Create object(s) @@ -145,14 +152,27 @@ killApiServer ####################################################### -# Step 2: Start a server which supports both the old and new api versions, +# Step 2: Perform etcd2 -> etcd migration. +# We always perform offline migration, so we need to stop etcd. +####################################################### + +kube::etcd::stop +TARGET_STORAGE="etcd3" \ + DATA_DIRECTORY="${ETCD_DIR}" \ + ETCDCTL=$(which etcdctl) \ + ${KUBE_ROOT}/cluster/images/etcd/migrate-if-needed.sh +kube::etcd::start + + +####################################################### +# Step 3: Start a server which supports both the old and new api versions, # but KUBE_NEW_API_VERSION is the latest (storage) version. # Still use KUBE_STORAGE_MEDIA_TYPE_JSON for storage encoding. ####################################################### KUBE_API_VERSIONS="${KUBE_NEW_API_VERSION},${KUBE_OLD_API_VERSION}" RUNTIME_CONFIG="api/all=false,api/${KUBE_OLD_API_VERSION}=true,api/${KUBE_NEW_API_VERSION}=true" -startApiServer ${KUBE_NEW_STORAGE_VERSIONS} ${KUBE_STORAGE_MEDIA_TYPE_JSON} +startApiServer ${STORAGE_BACKEND_ETCD3} ${KUBE_NEW_STORAGE_VERSIONS} ${KUBE_STORAGE_MEDIA_TYPE_JSON} # Update etcd objects, so that will now be stored in the new api version. kube::log::status "Updating storage versions in etcd" @@ -167,14 +187,14 @@ for test in ${tests[@]}; do new_storage_version=${test_data[5]} kube::log::status "Verifying ${resource}/${namespace}/${name} has updated storage version ${new_storage_version} in etcd" - curl -s http://${ETCD_HOST}:${ETCD_PORT}/v2/keys/${ETCD_PREFIX}/${resource}/${namespace}/${name} | grep ${new_storage_version} + ETCDCTL_API=3 ${ETCDCTL} --endpoints="${ETCD_HOST}:${ETCD_PORT}" get "/${ETCD_PREFIX}/${resource}/${namespace}/${name}" | grep ${new_storage_version} done killApiServer ####################################################### -# Step 3 : Start a server which supports only the new api version. +# Step 4 : Start a server which supports only the new api version. # However, change storage encoding to KUBE_STORAGE_MEDIA_TYPE_PROTOBUF. ####################################################### @@ -183,7 +203,7 @@ RUNTIME_CONFIG="api/all=false,api/${KUBE_NEW_API_VERSION}=true" # This seems to reduce flakiness. sleep 1 -startApiServer ${KUBE_NEW_STORAGE_VERSIONS} ${KUBE_STORAGE_MEDIA_TYPE_PROTOBUF} +startApiServer ${STORAGE_BACKEND_ETCD3} ${KUBE_NEW_STORAGE_VERSIONS} ${KUBE_STORAGE_MEDIA_TYPE_PROTOBUF} for test in ${tests[@]}; do IFS=',' read -ra test_data <<<"$test"