Convert liveness e2e test to Go

This commit reimplements hack/e2e-suite/liveness.sh in Go as part of cmd/e2e.

Tested by running it on a live cluster:
  $ cmd/e2e --host=https://w.x.y.z --provider=gce -t TestLivenessHttp -t TestLivenessExec
  I0122 08:12:53.183298    6502 liveness.go:72] Restart count of pod liveness-exec-6f917474-a251-11e4-8cc2-d4ae52bb3eea increased from 0 to 1 during the test
  I0122 08:13:23.605471    6502 liveness.go:72] Restart count of pod liveness-http-84d28569-a251-11e4-8cc2-d4ae52bb3eea increased from 0 to 1 during the test

Also ran the full e2e suite including kube-up/kube-down to confirm it works.

test/e2e/driver.go

@@ -88,6 +88,8 @@ func RunE2ETests(authConfig, certDir, host, repoRoot, provider string, orderseed
 		{TestPodHasServiceEnvVars, "TestPodHasServiceEnvVars"},
 		{TestBasic, "TestBasic"},
 		{TestPrivate, "TestPrivate"},
+		{TestLivenessHttp, "TestLivenessHttp"},
+		{TestLivenessExec, "TestLivenessExec"},
 	}
 
 	// Check testList for non-existent tests and populate a StringSet with tests to run.

test/e2e/liveness.go

@@ -0,0 +1,94 @@
+/*
+Copyright 2015 Google Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Tests for liveness probes, both with http and with docker exec.
+// These tests use the descriptions in examples/liveness to create test pods.
+
+package e2e
+
+import (
+	"time"
+
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
+	"github.com/golang/glog"
+)
+
+func runLivenessTest(c *client.Client, yamlFileName string) bool {
+	// Read the pod description from the YAML file.
+	podDescr := loadPodOrDie(assetPath("examples", "liveness", yamlFileName))
+	// Randomize the pod name to prevent the test to fail due to problems in clean up from
+	// previous tests or parallel executions of this same test.
+	podName := podDescr.Name + "-" + string(util.NewUUID())
+	podDescr.Name = podName
+	// Create the pod.
+	glog.Infof("Creating pod %s", podName)
+	_, err := c.Pods(api.NamespaceDefault).Create(podDescr)
+	if err != nil {
+		glog.Infof("Failed to create pod %s: %v", podName, err)
+		return false
+	}
+	// At the end of the test, clean up by removing the pod.
+	defer c.Pods(api.NamespaceDefault).Delete(podName)
+	// Wait until the pod is not pending. (Here we need to check for something other than
+	// 'Pending' other than checking for 'Running', since when failures occur, we go to
+	// 'Terminated' which can cause indefinite blocking.)
+	if !waitForPodNotPending(c, podName) {
+		glog.Infof("Failed to start pod %s", podName)
+		return false
+	}
+	glog.Infof("Started pod %s", podName)
+
+	// Check the pod's current state and verify that restartCount is present.
+	pod, err := c.Pods(api.NamespaceDefault).Get(podName)
+	if err != nil {
+		glog.Errorf("Get pod %s failed: %v", podName, err)
+		return false
+	}
+	initialRestartCount := pod.Status.Info["liveness"].RestartCount
+	glog.Infof("Initial restart count of pod %s is %d", podName, initialRestartCount)
+
+	// Wait for at most 48 * 5 = 240s = 4 minutes until restartCount is incremented
+	for i := 0; i < 48; i++ {
+		// Wait until restartCount is incremented.
+		time.Sleep(5 * time.Second)
+		pod, err = c.Pods(api.NamespaceDefault).Get(podName)
+		if err != nil {
+			glog.Errorf("Get pod %s failed: %v", podName, err)
+			return false
+		}
+		restartCount := pod.Status.Info["liveness"].RestartCount
+		glog.Infof("Restart count of pod %s is now %d", podName, restartCount)
+		if restartCount > initialRestartCount {
+			glog.Infof("Restart count of pod %s increased from %d to %d during the test", podName, initialRestartCount, restartCount)
+			return true
+		}
+	}
+
+	glog.Errorf("Did not see the restart count of pod %s increase from %d during the test", podName, initialRestartCount)
+	return false
+}
+
+// TestLivenessHttp tests restarts with a /healthz http liveness probe.
+func TestLivenessHttp(c *client.Client) bool {
+	return runLivenessTest(c, "http-liveness.yaml")
+}
+
+// TestLivenessExec tests restarts with a docker exec "cat /tmp/health" liveness probe.
+func TestLivenessExec(c *client.Client) bool {
+	return runLivenessTest(c, "exec-liveness.yaml")
+}

test/e2e/util.go

@@ -54,6 +54,27 @@ func waitForPodRunning(c *client.Client, id string) {
 	}
 }
 
+// waitForPodNotPending returns false if it took too long for the pod to go out of pending state.
+func waitForPodNotPending(c *client.Client, podName string) bool {
+	for i := 0; i < 10; i++ {
+		if i > 0 {
+			time.Sleep(5 * time.Second)
+		}
+		pod, err := c.Pods(api.NamespaceDefault).Get(podName)
+		if err != nil {
+			glog.Warningf("Get pod %s failed: %v", podName, err)
+			continue
+		}
+		if pod.Status.Phase != api.PodPending {
+			glog.Infof("Saw pod %s out of pending state (found %q)", podName, pod.Status.Phase)
+			return true
+		}
+		glog.Infof("Waiting for pod %s status to be !%q (found %q)", podName, api.PodPending, pod.Status.Phase)
+	}
+	glog.Warningf("Gave up waiting for pod %s status to go out of pending", podName)
+	return false
+}
+
 // waitForPodSuccess returns true if the pod reached state success, or false if it reached failure or ran too long.
 func waitForPodSuccess(c *client.Client, podName string, contName string) bool {
 	for i := 0; i < 10; i++ {