diff --git a/test/e2e/driver.go b/test/e2e/driver.go index cbee3c7aae7..697cf3f1aeb 100644 --- a/test/e2e/driver.go +++ b/test/e2e/driver.go @@ -88,6 +88,8 @@ func RunE2ETests(authConfig, certDir, host, repoRoot, provider string, orderseed {TestPodHasServiceEnvVars, "TestPodHasServiceEnvVars"}, {TestBasic, "TestBasic"}, {TestPrivate, "TestPrivate"}, + {TestLivenessHttp, "TestLivenessHttp"}, + {TestLivenessExec, "TestLivenessExec"}, } // Check testList for non-existent tests and populate a StringSet with tests to run. diff --git a/test/e2e/liveness.go b/test/e2e/liveness.go new file mode 100644 index 00000000000..0468426a3db --- /dev/null +++ b/test/e2e/liveness.go @@ -0,0 +1,94 @@ +/* +Copyright 2015 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Tests for liveness probes, both with http and with docker exec. +// These tests use the descriptions in examples/liveness to create test pods. + +package e2e + +import ( + "time" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/api" + "github.com/GoogleCloudPlatform/kubernetes/pkg/client" + "github.com/GoogleCloudPlatform/kubernetes/pkg/util" + "github.com/golang/glog" +) + +func runLivenessTest(c *client.Client, yamlFileName string) bool { + // Read the pod description from the YAML file. + podDescr := loadPodOrDie(assetPath("examples", "liveness", yamlFileName)) + // Randomize the pod name to prevent the test to fail due to problems in clean up from + // previous tests or parallel executions of this same test. + podName := podDescr.Name + "-" + string(util.NewUUID()) + podDescr.Name = podName + // Create the pod. + glog.Infof("Creating pod %s", podName) + _, err := c.Pods(api.NamespaceDefault).Create(podDescr) + if err != nil { + glog.Infof("Failed to create pod %s: %v", podName, err) + return false + } + // At the end of the test, clean up by removing the pod. + defer c.Pods(api.NamespaceDefault).Delete(podName) + // Wait until the pod is not pending. (Here we need to check for something other than + // 'Pending' other than checking for 'Running', since when failures occur, we go to + // 'Terminated' which can cause indefinite blocking.) + if !waitForPodNotPending(c, podName) { + glog.Infof("Failed to start pod %s", podName) + return false + } + glog.Infof("Started pod %s", podName) + + // Check the pod's current state and verify that restartCount is present. + pod, err := c.Pods(api.NamespaceDefault).Get(podName) + if err != nil { + glog.Errorf("Get pod %s failed: %v", podName, err) + return false + } + initialRestartCount := pod.Status.Info["liveness"].RestartCount + glog.Infof("Initial restart count of pod %s is %d", podName, initialRestartCount) + + // Wait for at most 48 * 5 = 240s = 4 minutes until restartCount is incremented + for i := 0; i < 48; i++ { + // Wait until restartCount is incremented. + time.Sleep(5 * time.Second) + pod, err = c.Pods(api.NamespaceDefault).Get(podName) + if err != nil { + glog.Errorf("Get pod %s failed: %v", podName, err) + return false + } + restartCount := pod.Status.Info["liveness"].RestartCount + glog.Infof("Restart count of pod %s is now %d", podName, restartCount) + if restartCount > initialRestartCount { + glog.Infof("Restart count of pod %s increased from %d to %d during the test", podName, initialRestartCount, restartCount) + return true + } + } + + glog.Errorf("Did not see the restart count of pod %s increase from %d during the test", podName, initialRestartCount) + return false +} + +// TestLivenessHttp tests restarts with a /healthz http liveness probe. +func TestLivenessHttp(c *client.Client) bool { + return runLivenessTest(c, "http-liveness.yaml") +} + +// TestLivenessExec tests restarts with a docker exec "cat /tmp/health" liveness probe. +func TestLivenessExec(c *client.Client) bool { + return runLivenessTest(c, "exec-liveness.yaml") +} diff --git a/test/e2e/util.go b/test/e2e/util.go index 8e8eacb6f0a..03baac6ba44 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -54,6 +54,27 @@ func waitForPodRunning(c *client.Client, id string) { } } +// waitForPodNotPending returns false if it took too long for the pod to go out of pending state. +func waitForPodNotPending(c *client.Client, podName string) bool { + for i := 0; i < 10; i++ { + if i > 0 { + time.Sleep(5 * time.Second) + } + pod, err := c.Pods(api.NamespaceDefault).Get(podName) + if err != nil { + glog.Warningf("Get pod %s failed: %v", podName, err) + continue + } + if pod.Status.Phase != api.PodPending { + glog.Infof("Saw pod %s out of pending state (found %q)", podName, pod.Status.Phase) + return true + } + glog.Infof("Waiting for pod %s status to be !%q (found %q)", podName, api.PodPending, pod.Status.Phase) + } + glog.Warningf("Gave up waiting for pod %s status to go out of pending", podName) + return false +} + // waitForPodSuccess returns true if the pod reached state success, or false if it reached failure or ran too long. func waitForPodSuccess(c *client.Client, podName string, contName string) bool { for i := 0; i < 10; i++ {