github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-09-15 14:14:39 +00:00
Code Issues Projects Releases Wiki Activity

Focus on restart numbers instead of timing

Browse Source 
Signed-off-by: Laura Lorenz <lauralorenz@google.com>
This commit is contained in:
Laura Lorenz
2024-11-12 07:04:32 +00:00
parent bad037b505
commit f032068ef7
2 changed files with 24 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
test/e2e/framework/pod/wait.go
View File

@@ -871,7 +871,7 @@ func WaitForContainerTerminated(ctx context.Context, c clientset.Interface, name
// WaitForContainerRestartedNTimes waits for the given Pod container to have restarted N times
func WaitForContainerRestartedNTimes(ctx context.Context, c clientset.Interface, namespace, podName, containerName string, timeout time.Duration, target int) error {
conditionDesc := fmt.Sprintf("container %s restarted %d times", containerName, target)
conditionDesc := fmt.Sprintf("container %s restarted at least %d times", containerName, target)
return WaitForPodCondition(ctx, c, namespace, podName, conditionDesc, timeout, func(pod *v1.Pod) (bool, error) {
for _, statuses := range [][]v1.ContainerStatus{pod.Status.ContainerStatuses, pod.Status.InitContainerStatuses, pod.Status.EphemeralContainerStatuses} {
for _, cs := range statuses {

45
test/e2e_node/container_restart_test.go
View File

@@ -13,13 +13,14 @@ import (
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/uuid"
kubeletevents "k8s.io/kubernetes/pkg/kubelet/events"
"k8s.io/kubernetes/test/e2e/feature"
"k8s.io/kubernetes/test/e2e/framework"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
admissionapi "k8s.io/pod-security-admission/api"
)
const containerName = "restarts"
var _ = SIGDescribe("Container Restart", feature.CriProxy, framework.WithSerial(), func() {
f := framework.NewDefaultFramework("container-restart")
f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
@@ -38,8 +39,8 @@ var _ = SIGDescribe("Container Restart", feature.CriProxy, framework.WithSerial(
})
ginkgo.It("Container restart backs off.", func(ctx context.Context) {
// 3 would take 10s best case, 6 would take 150s best case
doTest(ctx, f, 5, time.Duration(80*time.Second), time.Duration(10*time.Second))
// 0s, 0s, 10s, 30s, 70s, 150s, 310s
doTest(ctx, f, 5, containerName, 7)
})
})
@@ -62,41 +63,42 @@ var _ = SIGDescribe("Container Restart", feature.CriProxy, framework.WithSerial(
})
ginkgo.It("Alternate restart backs off.", func(ctx context.Context) {
doTest(ctx, f, 7, time.Duration(120*time.Second), time.Duration(10*time.Second))
// 0s, 0s, 10s, 30s, 60s, 90s, 120s, 150, 180, 210)
doTest(ctx, f, 7, containerName, 10)
})
})
})
func doTest(ctx context.Context, f *framework.Framework, maxRestarts int, target time.Duration, threshold time.Duration) {
func doTest(ctx context.Context, f *framework.Framework, targetRestarts int, containerName string, maxRestarts int) {
pod := e2epod.NewPodClient(f).Create(ctx, newFailAlwaysPod())
podErr := e2epod.WaitForPodContainerToFail(ctx, f.ClientSet, f.Namespace.Name, pod.Name, 0, "CrashLoopBackOff", 1*time.Minute)
gomega.Expect(podErr).To(gomega.HaveOccurred())
// Wait for 120s worth of backoffs to occur so we can confirm the backoff growth.
podErr = e2epod.WaitForContainerRestartedNTimes(ctx, f.ClientSet, f.Namespace.Name, pod.Name, "restart", 150*time.Second, maxRestarts)
// Wait for 150s worth of backoffs to occur so we can confirm the backoff growth.
podErr = e2epod.WaitForContainerRestartedNTimes(ctx, f.ClientSet, f.Namespace.Name, pod.Name, "restart", 150*time.Second, targetRestarts)
gomega.Expect(podErr).ShouldNot(gomega.HaveOccurred(), "Expected container to repeatedly back off container failures")
d, err := getContainerRetryDuration(ctx, f, pod.Name)
r, err := extractObservedBackoff(ctx, f, pod.Name, containerName)
framework.ExpectNoError(err)
gomega.Expect(d).Should(gomega.BeNumerically("~", target, threshold))
gomega.Expect(r).Should(gomega.BeNumerically("<=", maxRestarts))
}
func getContainerRetryDuration(ctx context.Context, f *framework.Framework, podName string) (time.Duration, error) {
var d time.Duration
e, err := f.ClientSet.CoreV1().Events(f.Namespace.Name).List(ctx, metav1.ListOptions{})
func extractObservedBackoff(ctx context.Context, f *framework.Framework, podName string, containerName string) (int32, error) {
var r int32
pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(ctx, podName, metav1.GetOptions{})
if err != nil {
return d, err
return r, err
}
for _, event := range e.Items {
if event.InvolvedObject.Name == podName && event.Reason == kubeletevents.StartedContainer {
return event.LastTimestamp.Time.Sub(event.FirstTimestamp.Time), nil
for _, statuses := range [][]v1.ContainerStatus{pod.Status.ContainerStatuses, pod.Status.InitContainerStatuses, pod.Status.EphemeralContainerStatuses} {
for _, cs := range statuses {
if cs.Name == containerName {
return r, nil
}
}
}
return d, nil
return r, nil
}
func newFailAlwaysPod() *v1.Pod {
@@ -108,10 +110,9 @@ func newFailAlwaysPod() *v1.Pod {
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "restart",
Name: containerName,
Image: imageutils.GetBusyBoxImageName(),
ImagePullPolicy: v1.PullAlways,
Command: []string{"exit 1"},
ImagePullPolicy: v1.PullIfNotPresent,
},
},
},