mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-27 21:47:07 +00:00
Merge pull request #4280 from rrati/e2e-liveness-4253
E2e liveness 4253
This commit is contained in:
Zach Loafman
commit
ac8e8cca78
@ -1,153 +0,0 @@
|
|||||||
/*
|
|
||||||
Copyright 2015 Google Inc. All rights reserved.
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
you may not use this file except in compliance with the License.
|
|
||||||
You may obtain a copy of the License at
|
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
// Tests for liveness probes, both with http and with docker exec.
|
|
||||||
// These tests use the descriptions in examples/liveness to create test pods.
|
|
||||||
|
|
||||||
package e2e
|
|
||||||
|
|
||||||
import (
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
|
|
||||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
|
|
||||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
|
|
||||||
"github.com/golang/glog"
|
|
||||||
|
|
||||||
. "github.com/onsi/ginkgo"
|
|
||||||
. "github.com/onsi/gomega"
|
|
||||||
)
|
|
||||||
|
|
||||||
func runLivenessTest(c *client.Client, podDescr *api.Pod) bool {
|
|
||||||
ns := "e2e-test-" + string(util.NewUUID())
|
|
||||||
glog.Infof("Creating pod %s in namespace %s", podDescr.Name, ns)
|
|
||||||
_, err := c.Pods(ns).Create(podDescr)
|
|
||||||
if err != nil {
|
|
||||||
glog.Infof("Failed to create pod %s: %v", podDescr.Name, err)
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
// At the end of the test, clean up by removing the pod.
|
|
||||||
defer c.Pods(ns).Delete(podDescr.Name)
|
|
||||||
// Wait until the pod is not pending. (Here we need to check for something other than
|
|
||||||
// 'Pending' other than checking for 'Running', since when failures occur, we go to
|
|
||||||
// 'Terminated' which can cause indefinite blocking.)
|
|
||||||
err = waitForPodNotPending(c, ns, podDescr.Name, 60*time.Second)
|
|
||||||
if err != nil {
|
|
||||||
glog.Infof("Failed to start pod %s in namespace %s: %v", podDescr.Name, ns, err)
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
glog.Infof("Started pod %s in namespace %s", podDescr.Name, ns)
|
|
||||||
|
|
||||||
// Check the pod's current state and verify that restartCount is present.
|
|
||||||
pod, err := c.Pods(ns).Get(podDescr.Name)
|
|
||||||
if err != nil {
|
|
||||||
glog.Errorf("Get pod %s in namespace %s failed: %v", podDescr.Name, ns, err)
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
initialRestartCount := pod.Status.Info["liveness"].RestartCount
|
|
||||||
glog.Infof("Initial restart count of pod %s is %d", podDescr.Name, initialRestartCount)
|
|
||||||
|
|
||||||
// Wait for at most 48 * 5 = 240s = 4 minutes until restartCount is incremented
|
|
||||||
for i := 0; i < 48; i++ {
|
|
||||||
// Wait until restartCount is incremented.
|
|
||||||
time.Sleep(5 * time.Second)
|
|
||||||
pod, err = c.Pods(ns).Get(podDescr.Name)
|
|
||||||
if err != nil {
|
|
||||||
glog.Errorf("Get pod %s failed: %v", podDescr.Name, err)
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
restartCount := pod.Status.Info["liveness"].RestartCount
|
|
||||||
glog.Infof("Restart count of pod %s in namespace %s is now %d", podDescr.Name, ns, restartCount)
|
|
||||||
if restartCount > initialRestartCount {
|
|
||||||
glog.Infof("Restart count of pod %s in namespace %s increased from %d to %d during the test", podDescr.Name, ns, initialRestartCount, restartCount)
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
glog.Errorf("Did not see the restart count of pod %s in namespace %s increase from %d during the test", podDescr.Name, ns, initialRestartCount)
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLivenessHttp tests restarts with a /healthz http liveness probe.
|
|
||||||
func TestLivenessHttp(c *client.Client) bool {
|
|
||||||
return runLivenessTest(c, &api.Pod{
|
|
||||||
ObjectMeta: api.ObjectMeta{
|
|
||||||
Name: "liveness-http",
|
|
||||||
Labels: map[string]string{"test": "liveness"},
|
|
||||||
},
|
|
||||||
Spec: api.PodSpec{
|
|
||||||
Containers: []api.Container{
|
|
||||||
{
|
|
||||||
Name: "liveness",
|
|
||||||
Image: "kubernetes/liveness",
|
|
||||||
Command: []string{"/server"},
|
|
||||||
LivenessProbe: &api.Probe{
|
|
||||||
Handler: api.Handler{
|
|
||||||
HTTPGet: &api.HTTPGetAction{
|
|
||||||
Path: "/healthz",
|
|
||||||
Port: util.NewIntOrStringFromInt(8080),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
InitialDelaySeconds: 15,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLivenessExec tests restarts with a docker exec "cat /tmp/health" liveness probe.
|
|
||||||
func TestLivenessExec(c *client.Client) bool {
|
|
||||||
return runLivenessTest(c, &api.Pod{
|
|
||||||
ObjectMeta: api.ObjectMeta{
|
|
||||||
Name: "liveness-exec",
|
|
||||||
Labels: map[string]string{"test": "liveness"},
|
|
||||||
},
|
|
||||||
Spec: api.PodSpec{
|
|
||||||
Containers: []api.Container{
|
|
||||||
{
|
|
||||||
Name: "liveness",
|
|
||||||
Image: "busybox",
|
|
||||||
Command: []string{"/bin/sh", "-c", "echo ok >/tmp/health; sleep 10; echo fail >/tmp/health; sleep 600"},
|
|
||||||
LivenessProbe: &api.Probe{
|
|
||||||
Handler: api.Handler{
|
|
||||||
Exec: &api.ExecAction{
|
|
||||||
Command: []string{"cat", "/tmp/health"},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
InitialDelaySeconds: 15,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
var _ = Describe("TestLivenessHttp", func() {
|
|
||||||
It("should pass", func() {
|
|
||||||
c, err := loadClient()
|
|
||||||
Expect(err).NotTo(HaveOccurred())
|
|
||||||
Expect(TestLivenessHttp(c)).To(BeTrue())
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
var _ = Describe("TestLivenessExec", func() {
|
|
||||||
It("should pass", func() {
|
|
||||||
c, err := loadClient()
|
|
||||||
Expect(err).NotTo(HaveOccurred())
|
|
||||||
Expect(TestLivenessExec(c)).To(BeTrue())
|
|
||||||
})
|
|
||||||
})
|
|
||||||
107
test/e2e/pods.go
107
test/e2e/pods.go
@ -30,10 +30,58 @@ import (
|
|||||||
. "github.com/onsi/gomega"
|
. "github.com/onsi/gomega"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func runLivenessTest(c *client.Client, podDescr *api.Pod) {
|
||||||
|
defer GinkgoRecover()
|
||||||
|
ns := "e2e-test-" + string(util.NewUUID())
|
||||||
|
|
||||||
|
By(fmt.Sprintf("Creating pod %s in namespace %s", podDescr.Name, ns))
|
||||||
|
_, err := c.Pods(ns).Create(podDescr)
|
||||||
|
Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("creating pod %s", podDescr.Name))
|
||||||
|
|
||||||
|
// At the end of the test, clean up by removing the pod.
|
||||||
|
defer func() {
|
||||||
|
By("deleting the pod")
|
||||||
|
c.Pods(ns).Delete(podDescr.Name)
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Wait until the pod is not pending. (Here we need to check for something other than
|
||||||
|
// 'Pending' other than checking for 'Running', since when failures occur, we go to
|
||||||
|
// 'Terminated' which can cause indefinite blocking.)
|
||||||
|
By("waiting for the pod to be something other than pending")
|
||||||
|
err = waitForPodNotPending(c, ns, podDescr.Name, 60*time.Second)
|
||||||
|
Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("starting pod %s in namespace %s", podDescr.Name, ns))
|
||||||
|
By(fmt.Sprintf("Started pod %s in namespace %s", podDescr.Name, ns))
|
||||||
|
|
||||||
|
// Check the pod's current state and verify that restartCount is present.
|
||||||
|
By("checking the pod's current state and verifying that restartCount is present")
|
||||||
|
pod, err := c.Pods(ns).Get(podDescr.Name)
|
||||||
|
Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("getting pod %s in namespace %s", podDescr.Name, ns))
|
||||||
|
initialRestartCount := pod.Status.Info["liveness"].RestartCount
|
||||||
|
By(fmt.Sprintf("Initial restart count of pod %s is %d", podDescr.Name, initialRestartCount))
|
||||||
|
|
||||||
|
// Wait for at most 48 * 5 = 240s = 4 minutes until restartCount is incremented
|
||||||
|
pass := false
|
||||||
|
for i := 0; i < 48; i++ {
|
||||||
|
// Wait until restartCount is incremented.
|
||||||
|
time.Sleep(5 * time.Second)
|
||||||
|
pod, err = c.Pods(ns).Get(podDescr.Name)
|
||||||
|
Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("getting pod %s", podDescr.Name))
|
||||||
|
restartCount := pod.Status.Info["liveness"].RestartCount
|
||||||
|
By(fmt.Sprintf("Restart count of pod %s in namespace %s is now %d", podDescr.Name, ns, restartCount))
|
||||||
|
if restartCount > initialRestartCount {
|
||||||
|
By(fmt.Sprintf("Restart count of pod %s in namespace %s increased from %d to %d during the test", podDescr.Name, ns, initialRestartCount, restartCount))
|
||||||
|
pass = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !pass {
|
||||||
|
Fail(fmt.Sprintf("Did not see the restart count of pod %s in namespace %s increase from %d during the test", podDescr.Name, ns, initialRestartCount))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
var _ = Describe("Pods", func() {
|
var _ = Describe("Pods", func() {
|
||||||
var (
|
var c *client.Client
|
||||||
c *client.Client
|
|
||||||
)
|
|
||||||
|
|
||||||
BeforeEach(func() {
|
BeforeEach(func() {
|
||||||
var err error
|
var err error
|
||||||
@ -314,4 +362,57 @@ var _ = Describe("Pods", func() {
|
|||||||
|
|
||||||
// We could try a wget the service from the client pod. But services.sh e2e test covers that pretty well.
|
// We could try a wget the service from the client pod. But services.sh e2e test covers that pretty well.
|
||||||
})
|
})
|
||||||
|
|
||||||
|
It("should be restarted with a docker exec \"cat /tmp/health\" liveness probe", func() {
|
||||||
|
runLivenessTest(c, &api.Pod{
|
||||||
|
ObjectMeta: api.ObjectMeta{
|
||||||
|
Name: "liveness-exec",
|
||||||
|
Labels: map[string]string{"test": "liveness"},
|
||||||
|
},
|
||||||
|
Spec: api.PodSpec{
|
||||||
|
Containers: []api.Container{
|
||||||
|
{
|
||||||
|
Name: "liveness",
|
||||||
|
Image: "busybox",
|
||||||
|
Command: []string{"/bin/sh", "-c", "echo ok >/tmp/health; sleep 10; echo fail >/tmp/health; sleep 600"},
|
||||||
|
LivenessProbe: &api.Probe{
|
||||||
|
Handler: api.Handler{
|
||||||
|
Exec: &api.ExecAction{
|
||||||
|
Command: []string{"cat", "/tmp/health"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
InitialDelaySeconds: 15,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
It("should be restarted with a /healthz http liveness probe", func() {
|
||||||
|
runLivenessTest(c, &api.Pod{
|
||||||
|
ObjectMeta: api.ObjectMeta{
|
||||||
|
Name: "liveness-http",
|
||||||
|
Labels: map[string]string{"test": "liveness"},
|
||||||
|
},
|
||||||
|
Spec: api.PodSpec{
|
||||||
|
Containers: []api.Container{
|
||||||
|
{
|
||||||
|
Name: "liveness",
|
||||||
|
Image: "kubernetes/liveness",
|
||||||
|
Command: []string{"/server"},
|
||||||
|
LivenessProbe: &api.Probe{
|
||||||
|
Handler: api.Handler{
|
||||||
|
HTTPGet: &api.HTTPGetAction{
|
||||||
|
Path: "/healthz",
|
||||||
|
Port: util.NewIntOrStringFromInt(8080),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
InitialDelaySeconds: 15,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user