Merge pull request #8953 from thockin/services-e2e

Make services.sh e2e more reliable(?) and debuggable

hack/e2e-suite/services.sh

@@ -22,11 +22,12 @@ set -o pipefail
 
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
 
+: ${NAMESPACE:="services-${RANDOM}"}
 : ${KUBE_VERSION_ROOT:=${KUBE_ROOT}}
-: ${KUBECTL:="${KUBE_VERSION_ROOT}/cluster/kubectl.sh"}
+: ${KUBECTL:="${KUBE_VERSION_ROOT}/cluster/kubectl.sh --namespace=${NAMESPACE}"}
 : ${KUBE_CONFIG_FILE:="config-test.sh"}
 
-export KUBECTL KUBE_CONFIG_FILE
+export NAMESPACE KUBECTL KUBE_CONFIG_FILE
 
 source "${KUBE_ROOT}/cluster/kube-env.sh"
 source "${KUBE_VERSION_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh"
@@ -39,7 +40,9 @@ function error() {
 }
 
 function sort_args() {
-  printf "%s\n" "$@" | sort -n | tr '\n\r' ' ' | sed 's/  */ /g'
+  [ $# == 0 ] && return
+  a=($(printf "%s\n" "$@" | sort -n))
+  echo "${a[*]}"
 }
 
 # Join args $2... with $1 between them.
@@ -58,10 +61,7 @@ function join() {
 
 svcs_to_clean=()
 function do_teardown() {
-  local svc
-  for svc in "${svcs_to_clean[@]:+${svcs_to_clean[@]}}"; do
-    stop_service "${svc}"
-  done
+  ${KUBECTL} delete namespace "${NAMESPACE}"
 }
 
 # Args:
@@ -70,7 +70,7 @@ function do_teardown() {
 #   $3: service replica count
 #   $4: public IPs (optional, string e.g. "1.2.3.4 5.6.7.8")
 function start_service() {
-  echo "Starting service '$1' on port $2 with $3 replicas"
+  echo "Starting service '${NAMESPACE}/$1' on port $2 with $3 replicas"
   svcs_to_clean+=("$1")
   ${KUBECTL} create -f - << __EOF__
 {
@@ -78,7 +78,7 @@ function start_service() {
   "apiVersion": "v1beta3",
   "metadata": {
     "name": "$1",
-    "namespace": "default",
+    "namespace": "${NAMESPACE}",
     "labels": {
       "name": "$1"
     }
@@ -124,7 +124,7 @@ __EOF__
   "apiVersion": "v1beta3",
   "metadata": {
     "name": "$1",
-    "namespace": "default",
+    "namespace": "${NAMESPACE}",
     "labels": {
       "name": "$1"
     }
@@ -218,30 +218,28 @@ function wait_for_pods() {
 #   $2: service IP
 #   $3: service port
 #   $4: pod count
-#   $5: pod IDs
+#   $5: pod IDs (sorted)
 function wait_for_service_up() {
   local i
   local found_pods
-  for i in $(seq 1 20); do
+  echo "waiting for $1 at $2:$3"
+  for i in $(seq 1 5); do
     results=($(ssh-to-node "${test_node}" "
         set -e;
-        for i in $(seq -s' ' 1 $4); do
+        for i in $(seq -s' ' 1 $(($4*3))); do
           curl -s --connect-timeout 1 http://$2:$3;
           echo;
-        done | sort | uniq
+        done | sort -n | uniq
         "))
 
     found_pods=$(sort_args "${results[@]:+${results[@]}}")
-    echo "Checking if ${found_pods} == ${5}"
     if [[ "${found_pods}" == "$5" ]]; then
-      break
+      return
     fi
-    echo "Waiting for endpoints to propagate"
-    sleep 3
+    echo "expected '$5', got '${found_pods}': will try again"
+    sleep 3  # wait for endpoints to propagate
   done
-  if [[ "${found_pods}" != "$5" ]]; then
-    error "Endpoints did not propagate in time"
-  fi
+  error "$1: failed to verify portal from host"
 }
 
 # Args:
@@ -264,35 +262,26 @@ function wait_for_service_down() {
 #   $2: service IP
 #   $3: service port
 #   $4: pod count
-#   $5: pod IDs
+#   $5: pod IDs (sorted)
 function verify_from_container() {
+  echo "waiting for $1 at $2:$3"
   results=($(ssh-to-node "${test_node}" "
       set -e;
       sudo docker pull busybox >/dev/null;
       sudo docker run busybox sh -c '
-          for i in $(seq -s' ' 1 $4); do
-            ok=false
-            for j in $(seq -s' ' 1 10); do
-              if wget -q -T 5 -O - http://$2:$3; then
-                echo
-                ok=true
-                break
-              fi
-              sleep 1
-            done
-            if [[ \${ok} == false ]]; then
+          for i in $(seq -s' ' 1 $(($4*3))); do
+            if wget -q -T 3 -O - http://$2:$3; then
+              echo
+            else
               exit 1
             fi
           done
-      '")) \
+      '" | sort -r -n | uniq)) \
       || error "testing $1 VIP from container failed"
   found_pods=$(sort_args "${results[@]}")
   if [[ "${found_pods}" != "$5" ]]; then
-    error -e "$1 VIP failed from container, expected:\n
-        $(printf '\t%s\n' $5)\n
-        got:\n
-        $(printf '\t%s\n' ${found_pods})
-        "
+    echo "expected '$5', got '${found_pods}'"
+    error "$1: failed to verify VIP from container"
   fi
 }
 
@@ -304,13 +293,13 @@ test_node="${MINION_NAMES[0]}"
 master="${MASTER_NAME}"
 
 # Launch some pods and services.
-svc1_name="service-${RANDOM}"
+svc1_name="service1"
 svc1_port=80
 svc1_count=3
 svc1_publics="192.168.1.1 192.168.1.2"
 start_service "${svc1_name}" "${svc1_port}" "${svc1_count}" "${svc1_publics}"
 
-svc2_name="service-${RANDOM}"
+svc2_name="service2"
 svc2_port=80
 svc2_count=3
 start_service "${svc2_name}" "${svc2_port}" "${svc2_count}"
@@ -414,7 +403,6 @@ echo "Manually removing iptables rules"
 # Remove both the new and old style chains, in case we're testing on an old kubelet
 ssh-to-node "${test_node}" "sudo iptables -t nat -F KUBE-PORTALS-HOST || true"
 ssh-to-node "${test_node}" "sudo iptables -t nat -F KUBE-PORTALS-CONTAINER || true"
-ssh-to-node "${test_node}" "sudo iptables -t nat -F KUBE-PROXY || true"
 echo "Verifying the VIPs from the host"
 wait_for_service_up "${svc3_name}" "${svc3_ip}" "${svc3_port}" \
     "${svc3_count}" "${svc3_pods}"