From 81e91db69776d7e9bacdf212a2e1ea643e98ff66 Mon Sep 17 00:00:00 2001 From: Joe Beda Date: Wed, 24 Sep 2014 10:55:58 -0700 Subject: [PATCH] Grab images as part of update e2e test --- cluster/gce/util.sh | 38 ++++++++++++--------- hack/e2e-suite/update.sh | 74 +++++++++++++++++++++++----------------- hack/e2e-test.sh | 3 ++ 3 files changed, 67 insertions(+), 48 deletions(-) diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 96ed50e7bfa..3d12fe53d6c 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -131,26 +131,23 @@ function kube-up { # Detect the project into $PROJECT if it isn't set detect-project + # This will take us up to the git repo root + local base_dir=$(dirname "${BASH_SOURCE}")/../.. + # Build up start up script for master KUBE_TEMP=$(mktemp -d -t kubernetes.XXXXXX) - trap "rm -rf ${KUBE_TEMP}" EXIT + trap 'rm -rf "${KUBE_TEMP}"' EXIT get-password echo "Using password: $user:$passwd" - python $(dirname $0)/../third_party/htpasswd/htpasswd.py -b -c ${KUBE_TEMP}/htpasswd $user $passwd - HTPASSWD=$(cat ${KUBE_TEMP}/htpasswd) - - ( - echo "#! /bin/bash" - echo "MASTER_NAME=${MASTER_NAME}" - echo "MASTER_RELEASE_TAR=${RELEASE_NORMALIZED}/master-release.tgz" - echo "MASTER_HTPASSWD='${HTPASSWD}'" - grep -v "^#" $(dirname $0)/templates/download-release.sh - grep -v "^#" $(dirname $0)/templates/salt-master.sh - ) > ${KUBE_TEMP}/master-start.sh + python "${base_dir}/third_party/htpasswd/htpasswd.py" -b \ + -c "${KUBE_TEMP}/htpasswd" $user $passwd + HTPASSWD=$(cat "${KUBE_TEMP}/htpasswd") if ! gcutil getnetwork "${NETWORK}"; then echo "Creating new network for: ${NETWORK}" + # The network needs to be created synchronously or we have a race. The + # firewalls can be added concurrent with instance creation. gcutil addnetwork "${NETWORK}" --range "10.240.0.0/16" gcutil addfirewall "${NETWORK}-default-internal" \ --norespect_terminal_width \ @@ -174,6 +171,15 @@ function kube-up { --target_tags ${MASTER_TAG} \ --allowed tcp:443 & + ( + echo "#! /bin/bash" + echo "MASTER_NAME='${MASTER_NAME}'" + echo "MASTER_RELEASE_TAR=${RELEASE_NORMALIZED}/master-release.tgz" + echo "MASTER_HTPASSWD='${HTPASSWD}'" + grep -v "^#" "${base_dir}/cluster/templates/download-release.sh" + grep -v "^#" "${base_dir}/cluster/templates/salt-master.sh" + ) > "${KUBE_TEMP}/master-start.sh" + gcutil addinstance ${MASTER_NAME}\ --norespect_terminal_width \ --project ${PROJECT} \ @@ -184,14 +190,14 @@ function kube-up { --network ${NETWORK} \ --service_account_scopes="storage-ro,compute-rw" \ --automatic_restart \ - --metadata_from_file startup-script:${KUBE_TEMP}/master-start.sh & + --metadata_from_file "startup-script:${KUBE_TEMP}/master-start.sh" & for (( i=0; i<${#MINION_NAMES[@]}; i++)); do ( echo "#! /bin/bash" - echo "MASTER_NAME=${MASTER_NAME}" + echo "MASTER_NAME='${MASTER_NAME}'" echo "MINION_IP_RANGE=${MINION_IP_RANGES[$i]}" - grep -v "^#" $(dirname $0)/templates/salt-minion.sh + grep -v "^#" "${base_dir}/cluster/templates/salt-minion.sh" ) > ${KUBE_TEMP}/minion-start-${i}.sh gcutil addfirewall ${MINION_NAMES[$i]}-all \ @@ -212,7 +218,7 @@ function kube-up { --service_account_scopes=${MINION_SCOPES} \ --automatic_restart \ --can_ip_forward \ - --metadata_from_file startup-script:${KUBE_TEMP}/minion-start-${i}.sh & + --metadata_from_file "startup-script:${KUBE_TEMP}/minion-start-${i}.sh" & gcutil addroute ${MINION_NAMES[$i]} ${MINION_IP_RANGES[$i]} \ --norespect_terminal_width \ diff --git a/hack/e2e-suite/update.sh b/hack/e2e-suite/update.sh index 81c33b6c6be..471646497e1 100755 --- a/hack/e2e-suite/update.sh +++ b/hack/e2e-suite/update.sh @@ -19,7 +19,6 @@ set -o errexit set -o nounset set -o pipefail -set -x source "${KUBE_REPO_ROOT}/cluster/kube-env.sh" source "${KUBE_REPO_ROOT}/cluster/$KUBERNETES_PROVIDER/util.sh" @@ -28,42 +27,53 @@ source "${KUBE_REPO_ROOT}/cluster/$KUBERNETES_PROVIDER/util.sh" CONTROLLER_NAME=update-demo function validate() { - NUM_REPLICAS=$1 - CONTAINER_IMAGE_VERSION=$2 - POD_ID_LIST=$($KUBECFG '-template={{range.Items}}{{.ID}} {{end}}' -l simpleService=${CONTROLLER_NAME} list pods) - POD_ARR=($POD_ID_LIST) - while [ ${#POD_ARR[@]} -ne $NUM_REPLICAS ]; do - echo "Waiting for the right number of containers" - sleep 5 - POD_ID_LIST=$($KUBECFG '-template={{range.Items}}{{.ID}} {{end}}' -l simpleService=${CONTROLLER_NAME} list pods) - POD_ARR=($POD_ID_LIST) - done + local num_replicas=$1 + local container_image_version=$2 # Container turn up on a clean cluster can take a while for the docker image pull. - ALL_RUNNING=0 - while [ $ALL_RUNNING -ne 1 ]; do - echo "Waiting for all containers in pod to come up." - sleep 5 - ALL_RUNNING=1 - for id in $POD_ID_LIST; do - TEMPLATE_STRING="{{and ((index .CurrentState.Info \"${CONTROLLER_NAME}\").State.Running) .CurrentState.Info.net.State.Running}}" - CURRENT_STATUS=$($KUBECFG -template "${TEMPLATE_STRING}" get pods/$id) - if [ "$CURRENT_STATUS" != "{}" ]; then - ALL_RUNNING=0 - else - CURRENT_IMAGE=$($KUBECFG -template "{{(index .CurrentState.Info \"${CONTROLLER_NAME}\").DetailInfo.Config.Image}}" get pods/$id) - if [ "$CURRENT_IMAGE" != "${DOCKER_HUB_USER}/update-demo:${CONTAINER_IMAGE_VERSION}" ]; then - ALL_RUNNING=0 - fi + local num_running=0 + while [[ $num_running -ne $num_replicas ]]; do + echo "Waiting for all containers in pod to come up. Currently: ${num_running}/${num_replicas}" + sleep 2 + + local pod_id_list + pod_id_list=($($KUBECFG -template='{{range.Items}}{{.ID}} {{end}}' -l simpleService="${CONTROLLER_NAME}" list pods)) + + echo " ${#pod_id_list[@]} out of ${num_replicas} created" + + local id + num_running=0 + for id in "${pod_id_list[@]}"; do + local template_string current_status current_image host_ip + template_string="{{and ((index .CurrentState.Info \"${CONTROLLER_NAME}\").State.Running) .CurrentState.Info.net.State.Running}}" + current_status=$($KUBECFG -template="${template_string}" get "pods/$id") + if [[ "$current_status" != "{}" ]]; then + echo " $id is created but not running" + continue fi + + template_string="{{(index .CurrentState.Info \"${CONTROLLER_NAME}\").DetailInfo.Config.Image}}" + current_image=$($KUBECFG -template="${template_string}" get "pods/$id") + if [[ "$current_image" != "${DOCKER_HUB_USER}/update-demo:${container_image_version}" ]]; then + echo " ${id} is created but running wrong image" + continue + fi + + + host_ip=$($KUBECFG -template='{{.CurrentState.HostIP}}' get pods/$id) + curl -s --max-time 5 --fail http://${host_ip}:8080/data.json \ + | grep -q ${container_image_version} || { + echo " ${id} is running the right image but curl to contents failed or returned wrong info" + continue + + } + + echo " ${id} is verified up and running" + + ((num_running++)) || true done done - - ids=($POD_ID_LIST) - if [ ${#ids[@]} -ne $NUM_REPLICAS ]; then - echo "Unexpected number of pods: ${#ids[@]}. Expected $NUM_REPLICAS" - exit 1 - fi + return 0 } export DOCKER_HUB_USER=jbeda diff --git a/hack/e2e-test.sh b/hack/e2e-test.sh index 28fb5e97ff8..19805d4d587 100755 --- a/hack/e2e-test.sh +++ b/hack/e2e-test.sh @@ -20,6 +20,9 @@ # Use testing config export KUBE_CONFIG_FILE="config-test.sh" export KUBE_REPO_ROOT="$(dirname $0)/.." + +# TODO(jbeda): This will break on usage if there is a space in +# ${KUBE_REPO_ROOT}. Covert to an array? Or an exported function? export KUBECFG="${KUBE_REPO_ROOT}/cluster/kubecfg.sh -expect_version_match" source $(dirname $0)/../cluster/kube-env.sh