Verify cluster size before e2e

cluster/vagrant/util.sh

@@ -21,12 +21,12 @@ source "${KUBE_ROOT}/cluster/vagrant/${KUBE_CONFIG_FILE-"config-default.sh"}"
 
 function detect-master () {
   KUBE_MASTER_IP=$MASTER_IP
-  echo "KUBE_MASTER_IP: ${KUBE_MASTER_IP}"
+  echo "KUBE_MASTER_IP: ${KUBE_MASTER_IP}" 1>&2
 }
 
 # Get minion IP addresses and store in KUBE_MINION_IP_ADDRESSES[]
 function detect-minions {
-  echo "Minions already detected"
+  echo "Minions already detected" 1>&2
   KUBE_MINION_IP_ADDRESSES=("${MINION_IPS[@]}")
 }
 
@@ -155,13 +155,15 @@ function verify-cluster {
     done
   done
 
-  echo
+  (
-  echo "Kubernetes cluster is running.  The master is running at:"
+    echo
-  echo
+    echo "Kubernetes cluster is running.  The master is running at:"
-  echo "  https://${MASTER_IP}"
+    echo
-  echo
+    echo "  https://${MASTER_IP}"
-  echo "The user name and password to use is located in ~/.kubernetes_vagrant_auth."
+    echo
-  echo
+    echo "The user name and password to use is located in ~/.kubernetes_vagrant_auth."
+    echo
+    )
 }
 
 
@@ -218,19 +220,19 @@ function test-build-release {
 
 # Execute prior to running tests to initialize required structure
 function test-setup {
-  echo "Vagrant test setup complete"
+  echo "Vagrant test setup complete" 1>&2
 }
 
 # Execute after running tests to perform any required clean-up
 function test-teardown {
-  echo "Vagrant ignores tear-down"
+  echo "Vagrant ignores tear-down" 1>&2
 }
 
 # Set the {user} and {password} environment values required to interact with provider
 function get-password {
   export KUBE_USER=vagrant
   export KUBE_PASSWORD=vagrant
-  echo "Using credentials: $KUBE_USER:$KUBE_PASSWORD"
+  echo "Using credentials: $KUBE_USER:$KUBE_PASSWORD" 1>&2
 }
 
 # Find the minion name based on the IP address
@@ -262,16 +264,16 @@ function restart-kube-proxy {
 }
 
 function setup-monitoring {
-    echo "TODO"
+  echo "TODO" 1>&2
 }
 
 function teardown-monitoring {
-  echo "TODO"
+  echo "TODO" 1>&2
 }
 
 # Perform preparations required to run e2e tests
 function prepare-e2e() {
-  echo "Vagrant doesn't need special preparations for e2e tests"
+  echo "Vagrant doesn't need special preparations for e2e tests" 1>&2
 }
 
 function setup-logging {

hack/e2e.go

@@ -67,6 +67,7 @@ const (
 	downloadDirName = "_output/downloads"
 	tarDirName      = "server"
 	tempDirName     = "upgrade-e2e-temp-dir"
+	minMinionCount  = 3
 )
 
 var (
@@ -196,6 +197,26 @@ func Up() bool {
 	return runBash("up", path.Join(versionRoot, "/cluster/kube-up.sh; test-setup;"))
 }
 
+// Ensure that the cluster is large engough to run the e2e tests.
+func ValidateClusterSize() {
+	// Check that there are at least 3 minions running
+	res, stdout, _ := runBashWithOutputs(
+		"validate cluster size",
+		"cluster/kubectl.sh get minions --no-headers | wc -l")
+	if !res {
+		log.Fatal("Could not get nodes to validate cluster size")
+	}
+
+	numNodes, err := strconv.Atoi(strings.TrimSpace(stdout))
+	if err != nil {
+		log.Fatalf("Could not count number of nodes to validate cluster size (%s)", err)
+	}
+
+	if numNodes < minMinionCount {
+		log.Fatalf("Cluster size (%d) is too small to run e2e tests.  %d Minions are required.", numNodes, minMinionCount)
+	}
+}
+
 // Is the e2e cluster up?
 func IsUp() bool {
 	return runBash("get status", `$KUBECTL version`)
@@ -264,6 +285,8 @@ func Test() (results ResultsByTest) {
 		log.Fatal("Testing requested, but e2e cluster not up!")
 	}
 
+	ValidateClusterSize()
+
 	// run tests!
 	dir, err := os.Open(filepath.Join(*root, "hack", "e2e-suite"))
 	if err != nil {
@@ -440,8 +463,8 @@ func finishRunning(stepName string, cmd *exec.Cmd) (bool, string, string) {
 	log.Printf("Running: %v", stepName)
 	stdout, stderr := bytes.NewBuffer(nil), bytes.NewBuffer(nil)
 	if *verbose {
-		cmd.Stdout = os.Stdout
+		cmd.Stdout = io.MultiWriter(os.Stdout, stdout)
-		cmd.Stderr = os.Stderr
+		cmd.Stderr = io.MultiWriter(os.Stderr, stderr)
 	} else {
 		cmd.Stdout = stdout
 		cmd.Stderr = stderr
@@ -462,13 +485,9 @@ func finishRunning(stepName string, cmd *exec.Cmd) (bool, string, string) {
 
 	if err := cmd.Run(); err != nil {
 		log.Printf("Error running %v: %v", stepName, err)
-		if !*verbose {
+		return false, string(stdout.Bytes()), string(stderr.Bytes())
-			return false, string(stdout.Bytes()), string(stderr.Bytes())
-		} else {
-			return false, "", ""
-		}
 	}
-	return true, "", ""
+	return true, string(stdout.Bytes()), string(stderr.Bytes())
 }
 
 func printBashOutputs(headerprefix, lineprefix, stdout, stderr string, escape bool) {