Add migration test

cluster/images/etcd/migrate-if-needed.sh

@@ -23,7 +23,6 @@
 
 set -o errexit
 set -o nounset
-set -o pipefail
 
 if [ -z "${TARGET_STORAGE:-}" ]; then
   echo "TARGET_USAGE variable unset - skipping migration"
@@ -35,6 +34,7 @@ if [ -z "${DATA_DIRECTORY:-}" ]; then
   exit 0
 fi
 
+ETCDCTL="${ETCDCTL:-/usr/local/bin/etcdctl}"
 VERSION_FILE="version.txt"
 CURRENT_STORAGE='etcd2'
 if [ -e "${DATA_DIRECTORY}/${VERSION_FILE}" ]; then
@@ -48,7 +48,8 @@ if [ "${CURRENT_STORAGE}" = "etcd2" -a "${TARGET_STORAGE}" = "etcd3" ]; then
     if [ "$(ls -A ${DATA_DIRECTORY})" ]; then
       echo "Performing etcd2 -> etcd3 migration"
       # TODO: Pass a correct transformer to handle TTLs.
-      ETCDCTL_API=3 /usr/local/bin/etcdctl migrate --data-dir=${DATA_DIRECTORY}
+      echo "ETCDCTL_API=3 ${ETCDCTL} migrate --data-dir=${DATA_DIRECTORY}"
+      ETCDCTL_API=3 ${ETCDCTL} migrate --data-dir=${DATA_DIRECTORY}
     fi
   fi
 fi

hack/lib/etcd.sh

@@ -40,7 +40,7 @@ kube::etcd::start() {
   fi
 
   # Start etcd
-  ETCD_DIR=$(mktemp -d 2>/dev/null || mktemp -d -t test-etcd.XXXXXX)
+  ETCD_DIR=${ETCD_DIR:-$(mktemp -d 2>/dev/null || mktemp -d -t test-etcd.XXXXXX)}
   if [[ -d "${ARTIFACTS_DIR:-}" ]]; then
     ETCD_LOGFILE="${ARTIFACTS_DIR}/etcd.$(uname -n).$(id -un).log.DEBUG.$(date +%Y%m%d-%H%M%S).$$"
   else

hack/test-update-storage-objects.sh

@@ -32,6 +32,9 @@ KUBE_NEW_API_VERSION=${KUBE_NEW_API_VERSION:-"v1"}
 KUBE_OLD_STORAGE_VERSIONS=${KUBE_OLD_STORAGE_VERSIONs:-""}
 KUBE_NEW_STORAGE_VERSIONS=${KUBE_NEW_STORAGE_VERSIONs:-""}
 
+STORAGE_BACKEND_ETCD2="etcd2"
+STORAGE_BACKEND_ETCD3="etcd3"
+
 KUBE_STORAGE_MEDIA_TYPE_JSON="application/json"
 KUBE_STORAGE_MEDIA_TYPE_PROTOBUF="application/vnd.kubernetes.protobuf"
 
@@ -43,13 +46,16 @@ API_HOST=${API_HOST:-127.0.0.1}
 KUBE_API_VERSIONS=""
 RUNTIME_CONFIG=""
 
+ETCDCTL=$(which etcdctl)
 KUBECTL="${KUBE_OUTPUT_HOSTBIN}/kubectl"
 UPDATE_ETCD_OBJECTS_SCRIPT="${KUBE_ROOT}/cluster/update-storage-objects.sh"
 
 function startApiServer() {
-  local storage_versions=${1:-""}
+  local storage_backend=${1:-"${STORAGE_BACKEND_ETCD2}"}
-  local storage_media_type=${2:-""}
+  local storage_versions=${2:-""}
+  local storage_media_type=${3:-""}
   kube::log::status "Starting kube-apiserver with KUBE_API_VERSIONS: ${KUBE_API_VERSIONS}"
+  kube::log::status "                           and storage-backend: ${storage_backend}"
   kube::log::status "                        and storage-media-type: ${storage_media_type}"
   kube::log::status "                            and runtime-config: ${RUNTIME_CONFIG}"
   kube::log::status "                 and storage-version overrides: ${storage_versions}"
@@ -59,8 +65,9 @@ function startApiServer() {
     --insecure-bind-address="${API_HOST}" \
     --bind-address="${API_HOST}" \
     --insecure-port="${API_PORT}" \
+    --storage-backend="${storage_backend}" \
     --etcd-servers="http://${ETCD_HOST}:${ETCD_PORT}" \
-    --etcd-prefix="${ETCD_PREFIX}" \
+    --etcd-prefix="/${ETCD_PREFIX}" \
     --runtime-config="${RUNTIME_CONFIG}" \
     --cert-dir="${TMPDIR:-/tmp/}" \
     --service-cluster-ip-range="10.0.0.0/24" \
@@ -120,7 +127,7 @@ KUBE_NEW_STORAGE_VERSIONS="batch/v1,autoscaling/v1"
 #######################################################
 KUBE_API_VERSIONS="${KUBE_OLD_API_VERSION},${KUBE_NEW_API_VERSION}"
 RUNTIME_CONFIG="api/all=false,api/${KUBE_OLD_API_VERSION}=true,api/${KUBE_NEW_API_VERSION}=true"
-startApiServer ${KUBE_OLD_STORAGE_VERSIONS} ${KUBE_STORAGE_MEDIA_TYPE_JSON}
+startApiServer ${STORAGE_BACKEND_ETCD2} ${KUBE_OLD_STORAGE_VERSIONS} ${KUBE_STORAGE_MEDIA_TYPE_JSON}
 
 
 # Create object(s)
@@ -145,14 +152,27 @@ killApiServer
 
 
 #######################################################
-# Step 2: Start a server which supports both the old and new api versions,
+# Step 2: Perform etcd2 -> etcd migration.
+# We always perform offline migration, so we need to stop etcd.
+#######################################################
+
+kube::etcd::stop
+TARGET_STORAGE="etcd3" \
+  DATA_DIRECTORY="${ETCD_DIR}" \
+  ETCDCTL=$(which etcdctl) \
+  ${KUBE_ROOT}/cluster/images/etcd/migrate-if-needed.sh
+kube::etcd::start
+
+
+#######################################################
+# Step 3: Start a server which supports both the old and new api versions,
 # but KUBE_NEW_API_VERSION is the latest (storage) version.
 # Still use KUBE_STORAGE_MEDIA_TYPE_JSON for storage encoding.
 #######################################################
 
 KUBE_API_VERSIONS="${KUBE_NEW_API_VERSION},${KUBE_OLD_API_VERSION}"
 RUNTIME_CONFIG="api/all=false,api/${KUBE_OLD_API_VERSION}=true,api/${KUBE_NEW_API_VERSION}=true"
-startApiServer ${KUBE_NEW_STORAGE_VERSIONS} ${KUBE_STORAGE_MEDIA_TYPE_JSON}
+startApiServer ${STORAGE_BACKEND_ETCD3} ${KUBE_NEW_STORAGE_VERSIONS} ${KUBE_STORAGE_MEDIA_TYPE_JSON}
 
 # Update etcd objects, so that will now be stored in the new api version.
 kube::log::status "Updating storage versions in etcd"
@@ -167,14 +187,14 @@ for test in ${tests[@]}; do
   new_storage_version=${test_data[5]}
 
   kube::log::status "Verifying ${resource}/${namespace}/${name} has updated storage version ${new_storage_version} in etcd"
-  curl -s http://${ETCD_HOST}:${ETCD_PORT}/v2/keys/${ETCD_PREFIX}/${resource}/${namespace}/${name} | grep ${new_storage_version}
+  ETCDCTL_API=3 ${ETCDCTL} --endpoints="${ETCD_HOST}:${ETCD_PORT}" get "/${ETCD_PREFIX}/${resource}/${namespace}/${name}" | grep ${new_storage_version}
 done
 
 killApiServer
 
 
 #######################################################
-# Step 3 : Start a server which supports only the new api version.
+# Step 4 : Start a server which supports only the new api version.
 # However, change storage encoding to KUBE_STORAGE_MEDIA_TYPE_PROTOBUF.
 #######################################################
 
@@ -183,7 +203,7 @@ RUNTIME_CONFIG="api/all=false,api/${KUBE_NEW_API_VERSION}=true"
 
 # This seems to reduce flakiness.
 sleep 1
-startApiServer ${KUBE_NEW_STORAGE_VERSIONS} ${KUBE_STORAGE_MEDIA_TYPE_PROTOBUF}
+startApiServer ${STORAGE_BACKEND_ETCD3} ${KUBE_NEW_STORAGE_VERSIONS} ${KUBE_STORAGE_MEDIA_TYPE_PROTOBUF}
 
 for test in ${tests[@]}; do
   IFS=',' read -ra test_data <<<"$test"