diff --git a/pkg/apis/core/validation/validation.go b/pkg/apis/core/validation/validation.go index d925231aae7..634b1b45bdb 100644 --- a/pkg/apis/core/validation/validation.go +++ b/pkg/apis/core/validation/validation.go @@ -2859,6 +2859,45 @@ func validatePodResourceClaimSource(claimSource core.ClaimSource, fldPath *field return allErrs } +func validateLivenessProbe(probe *core.Probe, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + + if probe == nil { + return allErrs + } + allErrs = append(allErrs, validateProbe(probe, fldPath)...) + if probe.SuccessThreshold != 1 { + allErrs = append(allErrs, field.Invalid(fldPath.Child("successThreshold"), probe.SuccessThreshold, "must be 1")) + } + return allErrs +} + +func validateReadinessProbe(probe *core.Probe, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + + if probe == nil { + return allErrs + } + allErrs = append(allErrs, validateProbe(probe, fldPath)...) + if probe.TerminationGracePeriodSeconds != nil { + allErrs = append(allErrs, field.Invalid(fldPath.Child("terminationGracePeriodSeconds"), probe.TerminationGracePeriodSeconds, "must not be set for readinessProbes")) + } + return allErrs +} + +func validateStartupProbe(probe *core.Probe, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + + if probe == nil { + return allErrs + } + allErrs = append(allErrs, validateProbe(probe, fldPath)...) + if probe.SuccessThreshold != 1 { + allErrs = append(allErrs, field.Invalid(fldPath.Child("successThreshold"), probe.SuccessThreshold, "must be 1")) + } + return allErrs +} + func validateProbe(probe *core.Probe, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} @@ -3245,36 +3284,26 @@ func validateInitContainers(containers []core.Container, regularContainers []cor switch { case restartAlways: - // TODO: Allow restartable init containers to have a lifecycle hook. if ctr.Lifecycle != nil { - allErrs = append(allErrs, field.Forbidden(idxPath.Child("lifecycle"), "may not be set for init containers")) - } - // TODO: Allow restartable init containers to have a liveness probe. - if ctr.LivenessProbe != nil { - allErrs = append(allErrs, field.Forbidden(idxPath.Child("livenessProbe"), "may not be set for init containers")) - } - // TODO: Allow restartable init containers to have a readiness probe. - if ctr.ReadinessProbe != nil { - allErrs = append(allErrs, field.Forbidden(idxPath.Child("readinessProbe"), "may not be set for init containers")) - } - allErrs = append(allErrs, validateProbe(ctr.StartupProbe, idxPath.Child("startupProbe"))...) - if ctr.StartupProbe != nil && ctr.StartupProbe.SuccessThreshold != 1 { - allErrs = append(allErrs, field.Invalid(idxPath.Child("startupProbe", "successThreshold"), ctr.StartupProbe.SuccessThreshold, "must be 1")) + allErrs = append(allErrs, validateLifecycle(ctr.Lifecycle, idxPath.Child("lifecycle"))...) } + allErrs = append(allErrs, validateLivenessProbe(ctr.LivenessProbe, idxPath.Child("livenessProbe"))...) + allErrs = append(allErrs, validateReadinessProbe(ctr.ReadinessProbe, idxPath.Child("readinessProbe"))...) + allErrs = append(allErrs, validateStartupProbe(ctr.StartupProbe, idxPath.Child("startupProbe"))...) default: // These fields are disallowed for init containers. if ctr.Lifecycle != nil { - allErrs = append(allErrs, field.Forbidden(idxPath.Child("lifecycle"), "may not be set for init containers")) + allErrs = append(allErrs, field.Forbidden(idxPath.Child("lifecycle"), "may not be set for init containers without restartPolicy=Always")) } if ctr.LivenessProbe != nil { - allErrs = append(allErrs, field.Forbidden(idxPath.Child("livenessProbe"), "may not be set for init containers")) + allErrs = append(allErrs, field.Forbidden(idxPath.Child("livenessProbe"), "may not be set for init containers without restartPolicy=Always")) } if ctr.ReadinessProbe != nil { - allErrs = append(allErrs, field.Forbidden(idxPath.Child("readinessProbe"), "may not be set for init containers")) + allErrs = append(allErrs, field.Forbidden(idxPath.Child("readinessProbe"), "may not be set for init containers without restartPolicy=Always")) } if ctr.StartupProbe != nil { - allErrs = append(allErrs, field.Forbidden(idxPath.Child("startupProbe"), "may not be set for init containers")) + allErrs = append(allErrs, field.Forbidden(idxPath.Child("startupProbe"), "may not be set for init containers without restartPolicy=Always")) } } @@ -3383,23 +3412,16 @@ func validateContainers(containers []core.Container, volumes map[string]core.Vol allNames.Insert(ctr.Name) } - // These fields are only allowed for regular containers, so only check supported values here. - // Init and ephemeral container validation will return field.Forbidden() for these paths. + // These fields are allowed for regular containers and restartable init + // containers. + // Regular init container and ephemeral container validation will return + // field.Forbidden() for these paths. if ctr.Lifecycle != nil { allErrs = append(allErrs, validateLifecycle(ctr.Lifecycle, path.Child("lifecycle"))...) } - allErrs = append(allErrs, validateProbe(ctr.LivenessProbe, path.Child("livenessProbe"))...) - if ctr.LivenessProbe != nil && ctr.LivenessProbe.SuccessThreshold != 1 { - allErrs = append(allErrs, field.Invalid(path.Child("livenessProbe", "successThreshold"), ctr.LivenessProbe.SuccessThreshold, "must be 1")) - } - allErrs = append(allErrs, validateProbe(ctr.ReadinessProbe, path.Child("readinessProbe"))...) - if ctr.ReadinessProbe != nil && ctr.ReadinessProbe.TerminationGracePeriodSeconds != nil { - allErrs = append(allErrs, field.Invalid(path.Child("readinessProbe", "terminationGracePeriodSeconds"), ctr.ReadinessProbe.TerminationGracePeriodSeconds, "must not be set for readinessProbes")) - } - allErrs = append(allErrs, validateProbe(ctr.StartupProbe, path.Child("startupProbe"))...) - if ctr.StartupProbe != nil && ctr.StartupProbe.SuccessThreshold != 1 { - allErrs = append(allErrs, field.Invalid(path.Child("startupProbe", "successThreshold"), ctr.StartupProbe.SuccessThreshold, "must be 1")) - } + allErrs = append(allErrs, validateLivenessProbe(ctr.LivenessProbe, path.Child("livenessProbe"))...) + allErrs = append(allErrs, validateReadinessProbe(ctr.ReadinessProbe, path.Child("readinessProbe"))...) + allErrs = append(allErrs, validateStartupProbe(ctr.StartupProbe, path.Child("startupProbe"))...) // These fields are disallowed for regular containers if ctr.RestartPolicy != nil { diff --git a/pkg/apis/core/validation/validation_test.go b/pkg/apis/core/validation/validation_test.go index bd12b18e0c6..702631b7458 100644 --- a/pkg/apis/core/validation/validation_test.go +++ b/pkg/apis/core/validation/validation_test.go @@ -8162,14 +8162,43 @@ func TestValidateInitContainers(t *testing.T) { ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File", }, { - Name: "container-3-restart-always-with-startup-probe", + Name: "container-3-restart-always-with-lifecycle-hook-and-probes", Image: "image", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File", RestartPolicy: &containerRestartPolicyAlways, + Lifecycle: &core.Lifecycle{ + PostStart: &core.LifecycleHandler{ + Exec: &core.ExecAction{ + Command: []string{"echo", "post start"}, + }, + }, + PreStop: &core.LifecycleHandler{ + Exec: &core.ExecAction{ + Command: []string{"echo", "pre stop"}, + }, + }, + }, + LivenessProbe: &core.Probe{ + ProbeHandler: core.ProbeHandler{ + TCPSocket: &core.TCPSocketAction{ + Port: intstr.FromInt32(80), + }, + }, + SuccessThreshold: 1, + }, + ReadinessProbe: &core.Probe{ + ProbeHandler: core.ProbeHandler{ + TCPSocket: &core.TCPSocketAction{ + Port: intstr.FromInt32(80), + }, + }, + }, StartupProbe: &core.Probe{ ProbeHandler: core.ProbeHandler{ - TCPSocket: &core.TCPSocketAction{Port: intstr.FromInt(80)}, + TCPSocket: &core.TCPSocketAction{ + Port: intstr.FromInt(80), + }, }, SuccessThreshold: 1, }, @@ -8390,6 +8419,165 @@ func TestValidateInitContainers(t *testing.T) { }, }}, field.ErrorList{{Type: field.ErrorTypeInvalid, Field: "initContainers[0].startupProbe.successThreshold", BadValue: int32(2)}}, + }, { + "invalid readiness probe, terminationGracePeriodSeconds set.", + line(), + []core.Container{{ + Name: "life-123", + Image: "image", + ImagePullPolicy: "IfNotPresent", + TerminationMessagePolicy: "File", + RestartPolicy: &containerRestartPolicyAlways, + ReadinessProbe: &core.Probe{ + ProbeHandler: core.ProbeHandler{ + TCPSocket: &core.TCPSocketAction{ + Port: intstr.FromInt32(80), + }, + }, + TerminationGracePeriodSeconds: utilpointer.Int64(10), + }, + }}, + field.ErrorList{{Type: field.ErrorTypeInvalid, Field: "initContainers[0].readinessProbe.terminationGracePeriodSeconds", BadValue: utilpointer.Int64(10)}}, + }, { + "invalid liveness probe, successThreshold != 1", + line(), + []core.Container{{ + Name: "live-123", + Image: "image", + ImagePullPolicy: "IfNotPresent", + TerminationMessagePolicy: "File", + RestartPolicy: &containerRestartPolicyAlways, + LivenessProbe: &core.Probe{ + ProbeHandler: core.ProbeHandler{ + TCPSocket: &core.TCPSocketAction{ + Port: intstr.FromInt32(80), + }, + }, + SuccessThreshold: 2, + }, + }}, + field.ErrorList{{Type: field.ErrorTypeInvalid, Field: "initContainers[0].livenessProbe.successThreshold", BadValue: int32(2)}}, + }, { + "invalid lifecycle, no exec command.", + line(), + []core.Container{{ + Name: "life-123", + Image: "image", + ImagePullPolicy: "IfNotPresent", + TerminationMessagePolicy: "File", + RestartPolicy: &containerRestartPolicyAlways, + Lifecycle: &core.Lifecycle{ + PreStop: &core.LifecycleHandler{ + Exec: &core.ExecAction{}, + }, + }, + }}, + field.ErrorList{{Type: field.ErrorTypeRequired, Field: "initContainers[0].lifecycle.preStop.exec.command", BadValue: ""}}, + }, { + "invalid lifecycle, no http path.", + line(), + []core.Container{{ + Name: "life-123", + Image: "image", + ImagePullPolicy: "IfNotPresent", + TerminationMessagePolicy: "File", + RestartPolicy: &containerRestartPolicyAlways, + Lifecycle: &core.Lifecycle{ + PreStop: &core.LifecycleHandler{ + HTTPGet: &core.HTTPGetAction{ + Port: intstr.FromInt32(80), + Scheme: "HTTP", + }, + }, + }, + }}, + field.ErrorList{{Type: field.ErrorTypeRequired, Field: "initContainers[0].lifecycle.preStop.httpGet.path", BadValue: ""}}, + }, { + "invalid lifecycle, no http port.", + line(), + []core.Container{{ + Name: "life-123", + Image: "image", + ImagePullPolicy: "IfNotPresent", + TerminationMessagePolicy: "File", + RestartPolicy: &containerRestartPolicyAlways, + Lifecycle: &core.Lifecycle{ + PreStop: &core.LifecycleHandler{ + HTTPGet: &core.HTTPGetAction{ + Path: "/", + Scheme: "HTTP", + }, + }, + }, + }}, + field.ErrorList{{Type: field.ErrorTypeInvalid, Field: "initContainers[0].lifecycle.preStop.httpGet.port", BadValue: 0}}, + }, { + "invalid lifecycle, no http scheme.", + line(), + []core.Container{{ + Name: "life-123", + Image: "image", + ImagePullPolicy: "IfNotPresent", + TerminationMessagePolicy: "File", + RestartPolicy: &containerRestartPolicyAlways, + Lifecycle: &core.Lifecycle{ + PreStop: &core.LifecycleHandler{ + HTTPGet: &core.HTTPGetAction{ + Path: "/", + Port: intstr.FromInt32(80), + }, + }, + }, + }}, + field.ErrorList{{Type: field.ErrorTypeNotSupported, Field: "initContainers[0].lifecycle.preStop.httpGet.scheme", BadValue: core.URIScheme("")}}, + }, { + "invalid lifecycle, no tcp socket port.", + line(), + []core.Container{{ + Name: "life-123", + Image: "image", + ImagePullPolicy: "IfNotPresent", + TerminationMessagePolicy: "File", + RestartPolicy: &containerRestartPolicyAlways, + Lifecycle: &core.Lifecycle{ + PreStop: &core.LifecycleHandler{ + TCPSocket: &core.TCPSocketAction{}, + }, + }, + }}, + field.ErrorList{{Type: field.ErrorTypeInvalid, Field: "initContainers[0].lifecycle.preStop.tcpSocket.port", BadValue: 0}}, + }, { + "invalid lifecycle, zero tcp socket port.", + line(), + []core.Container{{ + Name: "life-123", + Image: "image", + ImagePullPolicy: "IfNotPresent", + TerminationMessagePolicy: "File", + RestartPolicy: &containerRestartPolicyAlways, + Lifecycle: &core.Lifecycle{ + PreStop: &core.LifecycleHandler{ + TCPSocket: &core.TCPSocketAction{ + Port: intstr.FromInt32(0), + }, + }, + }, + }}, + field.ErrorList{{Type: field.ErrorTypeInvalid, Field: "initContainers[0].lifecycle.preStop.tcpSocket.port", BadValue: 0}}, + }, { + "invalid lifecycle, no action.", + line(), + []core.Container{{ + Name: "life-123", + Image: "image", + ImagePullPolicy: "IfNotPresent", + TerminationMessagePolicy: "File", + RestartPolicy: &containerRestartPolicyAlways, + Lifecycle: &core.Lifecycle{ + PreStop: &core.LifecycleHandler{}, + }, + }}, + field.ErrorList{{Type: field.ErrorTypeRequired, Field: "initContainers[0].lifecycle.preStop", BadValue: ""}}, }, } for _, tc := range errorCases { diff --git a/pkg/kubelet/kubelet_pods.go b/pkg/kubelet/kubelet_pods.go index bbe15aacfd9..03d8dc987b2 100644 --- a/pkg/kubelet/kubelet_pods.go +++ b/pkg/kubelet/kubelet_pods.go @@ -1738,9 +1738,10 @@ func (kl *Kubelet) generateAPIPodStatus(pod *v1.Pod, podStatus *kubecontainer.Po if utilfeature.DefaultFeatureGate.Enabled(features.PodReadyToStartContainersCondition) { s.Conditions = append(s.Conditions, status.GeneratePodReadyToStartContainersCondition(pod, podStatus)) } - s.Conditions = append(s.Conditions, status.GeneratePodInitializedCondition(&pod.Spec, append(s.InitContainerStatuses, s.ContainerStatuses...), s.Phase)) - s.Conditions = append(s.Conditions, status.GeneratePodReadyCondition(&pod.Spec, s.Conditions, s.ContainerStatuses, s.Phase)) - s.Conditions = append(s.Conditions, status.GenerateContainersReadyCondition(&pod.Spec, s.ContainerStatuses, s.Phase)) + allContainerStatuses := append(s.InitContainerStatuses, s.ContainerStatuses...) + s.Conditions = append(s.Conditions, status.GeneratePodInitializedCondition(&pod.Spec, allContainerStatuses, s.Phase)) + s.Conditions = append(s.Conditions, status.GeneratePodReadyCondition(&pod.Spec, s.Conditions, allContainerStatuses, s.Phase)) + s.Conditions = append(s.Conditions, status.GenerateContainersReadyCondition(&pod.Spec, allContainerStatuses, s.Phase)) s.Conditions = append(s.Conditions, v1.PodCondition{ Type: v1.PodScheduled, Status: v1.ConditionTrue, diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container.go b/pkg/kubelet/kuberuntime/kuberuntime_container.go index 5628620d72b..eb25241cc11 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container.go @@ -877,7 +877,7 @@ func hasAnyRegularContainerCreated(pod *v1.Pod, podStatus *kubecontainer.PodStat // The actions include: // - Start the first init container that has not been started. // - Restart all restartable init containers that have started but are not running. -// - Kill the restartable init containers that have failed the startup probe. +// - Kill the restartable init containers that are not alive or started. func (m *kubeGenericRuntimeManager) computeInitContainerActions(pod *v1.Pod, podStatus *kubecontainer.PodStatus, changes *podActions) bool { if len(pod.Spec.InitContainers) == 0 { return true @@ -960,6 +960,27 @@ func (m *kubeGenericRuntimeManager) computeInitContainerActions(pod *v1.Pod, pod // this init container is initialized for the first time, start the next one changes.InitContainersToStart = append(changes.InitContainersToStart, i+1) } + + // A restartable init container does not have to take into account its + // liveness probe when it determines to start the next init container. + if container.LivenessProbe != nil { + liveness, found := m.livenessManager.Get(status.ID) + if !found { + // If the liveness probe has not been run, wait for it. + break + } + if liveness == proberesults.Failure { + // If the restartable init container failed the liveness probe, + // restart it. + changes.ContainersToKill[status.ID] = containerToKillInfo{ + name: container.Name, + container: container, + message: fmt.Sprintf("Init container %s failed liveness probe", container.Name), + reason: reasonLivenessProbe, + } + changes.InitContainersToStart = append(changes.InitContainersToStart, i) + } + } } else { // init container // nothing do to but wait for it to finish break diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container_test.go b/pkg/kubelet/kuberuntime/kuberuntime_container_test.go index 2afae7bf205..c788bc0d379 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container_test.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container_test.go @@ -341,7 +341,7 @@ func TestToKubeContainerStatusWithResources(t *testing.T) { } } -func TestLifeCycleHook(t *testing.T) { +func testLifeCycleHook(t *testing.T, testPod *v1.Pod, testContainer *v1.Container) { // Setup fakeRuntime, _, m, _ := createTestRuntimeManager() @@ -352,23 +352,6 @@ func TestLifeCycleHook(t *testing.T) { ID: "foo", } - testPod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "bar", - Namespace: "default", - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "foo", - Image: "busybox", - ImagePullPolicy: v1.PullIfNotPresent, - Command: []string{"testCommand"}, - WorkingDir: "testWorkingDir", - }, - }, - }, - } cmdPostStart := &v1.Lifecycle{ PostStart: &v1.LifecycleHandler{ Exec: &v1.ExecAction{ @@ -418,7 +401,7 @@ func TestLifeCycleHook(t *testing.T) { // Configured and works as expected t.Run("PreStop-CMDExec", func(t *testing.T) { ctx := context.Background() - testPod.Spec.Containers[0].Lifecycle = cmdLifeCycle + testContainer.Lifecycle = cmdLifeCycle m.killContainer(ctx, testPod, cID, "foo", "testKill", "", &gracePeriod) if fakeRunner.Cmd[0] != cmdLifeCycle.PreStop.Exec.Command[0] { t.Errorf("CMD Prestop hook was not invoked") @@ -432,7 +415,7 @@ func TestLifeCycleHook(t *testing.T) { defer func() { fakeHTTP.req = nil }() defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentHTTPGetHandlers, false)() httpLifeCycle.PreStop.HTTPGet.Port = intstr.IntOrString{} - testPod.Spec.Containers[0].Lifecycle = httpLifeCycle + testContainer.Lifecycle = httpLifeCycle m.killContainer(ctx, testPod, cID, "foo", "testKill", "", &gracePeriod) if fakeHTTP.req == nil || !strings.Contains(fakeHTTP.req.URL.String(), httpLifeCycle.PreStop.HTTPGet.Host) { @@ -443,7 +426,7 @@ func TestLifeCycleHook(t *testing.T) { ctx := context.Background() defer func() { fakeHTTP.req = nil }() httpLifeCycle.PreStop.HTTPGet.Port = intstr.FromInt32(80) - testPod.Spec.Containers[0].Lifecycle = httpLifeCycle + testContainer.Lifecycle = httpLifeCycle m.killContainer(ctx, testPod, cID, "foo", "testKill", "", &gracePeriod) if fakeHTTP.req == nil || !strings.Contains(fakeHTTP.req.URL.String(), httpLifeCycle.PreStop.HTTPGet.Host) { @@ -473,8 +456,7 @@ func TestLifeCycleHook(t *testing.T) { // Fake all the things you need before trying to create a container fakeSandBox, _ := makeAndSetFakePod(t, m, fakeRuntime, testPod) fakeSandBoxConfig, _ := m.generatePodSandboxConfig(testPod, 0) - testPod.Spec.Containers[0].Lifecycle = cmdPostStart - testContainer := &testPod.Spec.Containers[0] + testContainer.Lifecycle = cmdPostStart fakePodStatus := &kubecontainer.PodStatus{ ContainerStatuses: []*kubecontainer.Status{ { @@ -500,6 +482,51 @@ func TestLifeCycleHook(t *testing.T) { }) } +func TestLifeCycleHook(t *testing.T) { + testPod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bar", + Namespace: "default", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "foo", + Image: "busybox", + ImagePullPolicy: v1.PullIfNotPresent, + Command: []string{"testCommand"}, + WorkingDir: "testWorkingDir", + }, + }, + }, + } + + testLifeCycleHook(t, testPod, &testPod.Spec.Containers[0]) +} + +func TestLifeCycleHookForRestartableInitContainer(t *testing.T) { + testPod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bar", + Namespace: "default", + }, + Spec: v1.PodSpec{ + InitContainers: []v1.Container{ + { + Name: "foo", + Image: "busybox", + ImagePullPolicy: v1.PullIfNotPresent, + Command: []string{"testCommand"}, + WorkingDir: "testWorkingDir", + RestartPolicy: &containerRestartPolicyAlways, + }, + }, + }, + } + + testLifeCycleHook(t, testPod, &testPod.Spec.InitContainers[0]) +} + func TestStartSpec(t *testing.T) { podStatus := &kubecontainer.PodStatus{ ContainerStatuses: []*kubecontainer.Status{ diff --git a/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go b/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go index f9fe3a9c666..83df1ec0271 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go @@ -1518,6 +1518,62 @@ func TestComputePodActionsWithRestartableInitContainers(t *testing.T) { ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}), }, }, + "livenessProbe has not been run; start the nothing": { + mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways }, + mutateStatusFn: func(pod *v1.Pod, status *kubecontainer.PodStatus) { + m.livenessManager.Remove(status.ContainerStatuses[1].ID) + status.ContainerStatuses = status.ContainerStatuses[:2] + }, + actions: podActions{ + SandboxID: baseStatus.SandboxStatuses[0].Id, + InitContainersToStart: []int{2}, + ContainersToStart: []int{}, + ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}), + }, + }, + "livenessProbe in progress; start the next": { + mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways }, + mutateStatusFn: func(pod *v1.Pod, status *kubecontainer.PodStatus) { + m.livenessManager.Set(status.ContainerStatuses[1].ID, proberesults.Unknown, basePod) + status.ContainerStatuses = status.ContainerStatuses[:2] + }, + actions: podActions{ + SandboxID: baseStatus.SandboxStatuses[0].Id, + InitContainersToStart: []int{2}, + ContainersToStart: []int{}, + ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}), + }, + resetStatusFn: func(status *kubecontainer.PodStatus) { + m.livenessManager.Remove(status.ContainerStatuses[1].ID) + }, + }, + "livenessProbe has completed; start the next": { + mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways }, + mutateStatusFn: func(pod *v1.Pod, status *kubecontainer.PodStatus) { + status.ContainerStatuses = status.ContainerStatuses[:2] + }, + actions: podActions{ + SandboxID: baseStatus.SandboxStatuses[0].Id, + InitContainersToStart: []int{2}, + ContainersToStart: []int{}, + ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}), + }, + }, + "kill and recreate the restartable init container if the liveness check has failed": { + mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways }, + mutateStatusFn: func(pod *v1.Pod, status *kubecontainer.PodStatus) { + m.livenessManager.Set(status.ContainerStatuses[2].ID, proberesults.Failure, basePod) + }, + actions: podActions{ + SandboxID: baseStatus.SandboxStatuses[0].Id, + InitContainersToStart: []int{2}, + ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{2}), + ContainersToStart: []int{0, 1, 2}, + }, + resetStatusFn: func(status *kubecontainer.PodStatus) { + m.livenessManager.Remove(status.ContainerStatuses[2].ID) + }, + }, "startupProbe has not been run; do nothing": { mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways }, mutateStatusFn: func(pod *v1.Pod, status *kubecontainer.PodStatus) { @@ -1740,7 +1796,9 @@ func TestComputePodActionsWithRestartableInitContainers(t *testing.T) { }, } { pod, status := makeBasePodAndStatusWithRestartableInitContainers() + m.livenessManager.Set(status.ContainerStatuses[1].ID, proberesults.Success, basePod) m.startupManager.Set(status.ContainerStatuses[1].ID, proberesults.Success, basePod) + m.livenessManager.Set(status.ContainerStatuses[2].ID, proberesults.Success, basePod) m.startupManager.Set(status.ContainerStatuses[2].ID, proberesults.Success, basePod) if test.mutatePodFn != nil { test.mutatePodFn(pod) @@ -1769,12 +1827,14 @@ func makeBasePodAndStatusWithRestartableInitContainers() (*v1.Pod, *kubecontaine Name: "restartable-init-2", Image: "bar-image", RestartPolicy: &containerRestartPolicyAlways, + LivenessProbe: &v1.Probe{}, StartupProbe: &v1.Probe{}, }, { Name: "restartable-init-3", Image: "bar-image", RestartPolicy: &containerRestartPolicyAlways, + LivenessProbe: &v1.Probe{}, StartupProbe: &v1.Probe{}, }, } diff --git a/pkg/kubelet/prober/prober_manager_test.go b/pkg/kubelet/prober/prober_manager_test.go index 29d5551c443..16ef20a5371 100644 --- a/pkg/kubelet/prober/prober_manager_test.go +++ b/pkg/kubelet/prober/prober_manager_test.go @@ -152,8 +152,12 @@ func TestAddRemovePodsWithRestartableInitContainer(t *testing.T) { enableSidecarContainers: false, }, { - desc: "pod with sidecar (sidecar containers feature enabled)", - probePaths: []probeKey{{"restartable_init_container_pod", "restartable-init", readiness}}, + desc: "pod with sidecar (sidecar containers feature enabled)", + probePaths: []probeKey{ + {"restartable_init_container_pod", "restartable-init", liveness}, + {"restartable_init_container_pod", "restartable-init", readiness}, + {"restartable_init_container_pod", "restartable-init", startup}, + }, enableSidecarContainers: true, }, } @@ -179,7 +183,9 @@ func TestAddRemovePodsWithRestartableInitContainer(t *testing.T) { Name: "init", }, { Name: "restartable-init", + LivenessProbe: defaultProbe, ReadinessProbe: defaultProbe, + StartupProbe: defaultProbe, RestartPolicy: containerRestartPolicy(tc.enableSidecarContainers), }}, Containers: []v1.Container{{ diff --git a/pkg/kubelet/status/generate.go b/pkg/kubelet/status/generate.go index 7c6c1179a20..c6707345b67 100644 --- a/pkg/kubelet/status/generate.go +++ b/pkg/kubelet/status/generate.go @@ -55,6 +55,21 @@ func GenerateContainersReadyCondition(spec *v1.PodSpec, containerStatuses []v1.C } unknownContainers := []string{} unreadyContainers := []string{} + + for _, container := range spec.InitContainers { + if !kubetypes.IsRestartableInitContainer(&container) { + continue + } + + if containerStatus, ok := podutil.GetContainerStatus(containerStatuses, container.Name); ok { + if !containerStatus.Ready { + unreadyContainers = append(unreadyContainers, container.Name) + } + } else { + unknownContainers = append(unknownContainers, container.Name) + } + } + for _, container := range spec.Containers { if containerStatus, ok := podutil.GetContainerStatus(containerStatuses, container.Name); ok { if !containerStatus.Ready { diff --git a/pkg/kubelet/status/generate_test.go b/pkg/kubelet/status/generate_test.go index d185db2cc23..6bb32079998 100644 --- a/pkg/kubelet/status/generate_test.go +++ b/pkg/kubelet/status/generate_test.go @@ -30,6 +30,10 @@ import ( "k8s.io/utils/pointer" ) +var ( + containerRestartPolicyAlways = v1.ContainerRestartPolicyAlways +) + func TestGenerateContainersReadyCondition(t *testing.T) { tests := []struct { spec *v1.PodSpec @@ -112,6 +116,74 @@ func TestGenerateContainersReadyCondition(t *testing.T) { podPhase: v1.PodSucceeded, expectReady: getPodCondition(v1.ContainersReady, v1.ConditionFalse, PodCompleted, ""), }, + { + spec: &v1.PodSpec{ + InitContainers: []v1.Container{ + {Name: "restartable-init-1", RestartPolicy: &containerRestartPolicyAlways}, + }, + Containers: []v1.Container{ + {Name: "regular-1"}, + }, + }, + containerStatuses: []v1.ContainerStatus{ + getReadyStatus("regular-1"), + }, + podPhase: v1.PodRunning, + expectReady: getPodCondition(v1.ContainersReady, v1.ConditionFalse, ContainersNotReady, "containers with unknown status: [restartable-init-1]"), + }, + { + spec: &v1.PodSpec{ + InitContainers: []v1.Container{ + {Name: "restartable-init-1", RestartPolicy: &containerRestartPolicyAlways}, + {Name: "restartable-init-2", RestartPolicy: &containerRestartPolicyAlways}, + }, + Containers: []v1.Container{ + {Name: "regular-1"}, + }, + }, + containerStatuses: []v1.ContainerStatus{ + getReadyStatus("restartable-init-1"), + getReadyStatus("restartable-init-2"), + getReadyStatus("regular-1"), + }, + podPhase: v1.PodRunning, + expectReady: getPodCondition(v1.ContainersReady, v1.ConditionTrue, "", ""), + }, + { + spec: &v1.PodSpec{ + InitContainers: []v1.Container{ + {Name: "restartable-init-1", RestartPolicy: &containerRestartPolicyAlways}, + {Name: "restartable-init-2", RestartPolicy: &containerRestartPolicyAlways}, + }, + Containers: []v1.Container{ + {Name: "regular-1"}, + }, + }, + containerStatuses: []v1.ContainerStatus{ + getReadyStatus("restartable-init-1"), + getReadyStatus("regular-1"), + }, + podPhase: v1.PodRunning, + expectReady: getPodCondition(v1.ContainersReady, v1.ConditionFalse, ContainersNotReady, "containers with unknown status: [restartable-init-2]"), + }, + { + spec: &v1.PodSpec{ + InitContainers: []v1.Container{ + {Name: "restartable-init-1", RestartPolicy: &containerRestartPolicyAlways}, + {Name: "restartable-init-2", RestartPolicy: &containerRestartPolicyAlways}, + }, + Containers: []v1.Container{ + {Name: "regular-1"}, + }, + }, + containerStatuses: []v1.ContainerStatus{ + getReadyStatus("restartable-init-1"), + getNotReadyStatus("restartable-init-2"), + getReadyStatus("regular-1"), + }, + podPhase: v1.PodRunning, + expectReady: getPodCondition(v1.ContainersReady, v1.ConditionFalse, ContainersNotReady, "containers with unready status: [restartable-init-2]"), + }, } for i, test := range tests { diff --git a/pkg/kubelet/status/status_manager.go b/pkg/kubelet/status/status_manager.go index ed553215c1e..e43c5874481 100644 --- a/pkg/kubelet/status/status_manager.go +++ b/pkg/kubelet/status/status_manager.go @@ -349,8 +349,9 @@ func (m *manager) SetContainerReadiness(podUID types.UID, containerID kubecontai status.Conditions = append(status.Conditions, condition) } } - updateConditionFunc(v1.PodReady, GeneratePodReadyCondition(&pod.Spec, status.Conditions, status.ContainerStatuses, status.Phase)) - updateConditionFunc(v1.ContainersReady, GenerateContainersReadyCondition(&pod.Spec, status.ContainerStatuses, status.Phase)) + allContainerStatuses := append(status.InitContainerStatuses, status.ContainerStatuses...) + updateConditionFunc(v1.PodReady, GeneratePodReadyCondition(&pod.Spec, status.Conditions, allContainerStatuses, status.Phase)) + updateConditionFunc(v1.ContainersReady, GenerateContainersReadyCondition(&pod.Spec, allContainerStatuses, status.Phase)) m.updateStatusInternal(pod, status, false, false) } diff --git a/test/e2e/common/node/container_probe.go b/test/e2e/common/node/container_probe.go index edec437f1c4..20c365f700f 100644 --- a/test/e2e/common/node/container_probe.go +++ b/test/e2e/common/node/container_probe.go @@ -253,7 +253,7 @@ var _ = SIGDescribe("Probing container", func() { FailureThreshold: 1, } pod := busyBoxPodSpec(readinessProbe, nil, cmd) - runReadinessFailTest(ctx, f, pod, time.Minute) + runReadinessFailTest(ctx, f, pod, time.Minute, true) }) /* @@ -729,6 +729,762 @@ done }) }) +var _ = SIGDescribe("[NodeAlphaFeature:SidecarContainers][Feature:SidecarContainers] Probing restartable init container", func() { + f := framework.NewDefaultFramework("container-probe") + f.NamespacePodSecurityLevel = admissionapi.LevelBaseline + var podClient *e2epod.PodClient + probe := webserverProbeBuilder{} + + ginkgo.BeforeEach(func() { + podClient = e2epod.NewPodClient(f) + }) + + /* + Release: v1.28 + Testname: Pod restartable init container readiness probe, with initial delay + Description: Create a Pod that is configured with a initial delay set on + the readiness probe. Check the Pod Start time to compare to the initial + delay. The Pod MUST be ready only after the specified initial delay. + */ + ginkgo.It("with readiness probe should not be ready before initial delay and never restart", func(ctx context.Context) { + containerName := "test-webserver" + p := podClient.Create(ctx, testWebServerSidecarPodSpec(probe.withInitialDelay().build(), nil, containerName, 80)) + framework.ExpectNoError(e2epod.WaitTimeoutForPodReadyInNamespace(ctx, f.ClientSet, p.Name, f.Namespace.Name, framework.PodStartTimeout)) + + p, err := podClient.Get(ctx, p.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + isReady, err := testutils.PodRunningReady(p) + framework.ExpectNoError(err) + if !isReady { + framework.Failf("pod %s/%s should be ready", f.Namespace.Name, p.Name) + } + + // We assume the pod became ready when the container became ready. This + // is true for a single container pod. + readyTime, err := GetTransitionTimeForReadyCondition(p) + framework.ExpectNoError(err) + startedTime, err := GetContainerStartedTime(p, containerName) + framework.ExpectNoError(err) + + framework.Logf("Container started at %v, pod became ready at %v", startedTime, readyTime) + initialDelay := probeTestInitialDelaySeconds * time.Second + if readyTime.Sub(startedTime) < initialDelay { + framework.Failf("Pod became ready before it's %v initial delay", initialDelay) + } + + restartCount := getRestartCount(p) + framework.ExpectEqual(restartCount, 0, "pod should have a restart count of 0 but got %v", restartCount) + }) + + /* + Release: v1.28 + Testname: Pod restartable init container readiness probe, failure + Description: Create a Pod with a readiness probe that fails consistently. + When this Pod is created, then the Pod MUST never be ready, never be + running and restart count MUST be zero. + */ + ginkgo.It("with readiness probe that fails should never be ready and never restart", func(ctx context.Context) { + p := podClient.Create(ctx, testWebServerSidecarPodSpec(probe.withFailing().build(), nil, "test-webserver", 80)) + gomega.Consistently(ctx, func() (bool, error) { + p, err := podClient.Get(ctx, p.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + return podutil.IsPodReady(p), nil + }, 1*time.Minute, 1*time.Second).ShouldNot(gomega.BeTrue(), "pod should not be ready") + + p, err := podClient.Get(ctx, p.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + + isReady, _ := testutils.PodRunningReady(p) + if isReady { + framework.Failf("pod %s/%s should be not ready", f.Namespace.Name, p.Name) + } + + restartCount := getRestartCount(p) + framework.ExpectEqual(restartCount, 0, "pod should have a restart count of 0 but got %v", restartCount) + }) + + /* + Release: v1.28 + Testname: Pod restartable init container liveness probe, using local file, restart + Description: Create a Pod with liveness probe that uses ExecAction handler + to cat /temp/health file. The Container deletes the file /temp/health after + 10 second, triggering liveness probe to fail. The Pod MUST now be killed + and restarted incrementing restart count to 1. + */ + ginkgo.It("should be restarted with a exec \"cat /tmp/health\" liveness probe", func(ctx context.Context) { + cmd := []string{"/bin/sh", "-c", "echo ok >/tmp/health; sleep 10; rm -rf /tmp/health; sleep 600"} + livenessProbe := &v1.Probe{ + ProbeHandler: execHandler([]string{"cat", "/tmp/health"}), + InitialDelaySeconds: 15, + TimeoutSeconds: 5, // default 1s can be pretty aggressive in CI environments with low resources + FailureThreshold: 1, + } + pod := busyBoxSidecarPodSpec(nil, livenessProbe, cmd) + RunSidecarLivenessTest(ctx, f, pod, 1, defaultObservationTimeout) + }) + + /* + Release: v1.28 + Testname: Pod restartable init container liveness probe, using local file, no restart + Description: Pod is created with liveness probe that uses 'exec' command + to cat /temp/health file. Liveness probe MUST not fail to check health and + the restart count should remain 0. + */ + ginkgo.It("should *not* be restarted with a exec \"cat /tmp/health\" liveness probe", func(ctx context.Context) { + cmd := []string{"/bin/sh", "-c", "echo ok >/tmp/health; sleep 600"} + livenessProbe := &v1.Probe{ + ProbeHandler: execHandler([]string{"cat", "/tmp/health"}), + InitialDelaySeconds: 15, + TimeoutSeconds: 5, // default 1s can be pretty aggressive in CI environments with low resources + FailureThreshold: 1, + } + pod := busyBoxSidecarPodSpec(nil, livenessProbe, cmd) + RunSidecarLivenessTest(ctx, f, pod, 0, defaultObservationTimeout) + }) + + /* + Release: v1.28 + Testname: Pod restartable init container liveness probe, using http endpoint, restart + Description: A Pod is created with liveness probe on http endpoint + /healthz. The http handler on the /healthz will return a http error after + 10 seconds since the Pod is started. This MUST result in liveness check + failure. The Pod MUST now be killed and restarted incrementing restart + count to 1. + */ + ginkgo.It("should be restarted with a /healthz http liveness probe", func(ctx context.Context) { + livenessProbe := &v1.Probe{ + ProbeHandler: httpGetHandler("/healthz", 8080), + InitialDelaySeconds: 15, + FailureThreshold: 1, + } + pod := livenessSidecarPodSpec(f.Namespace.Name, nil, livenessProbe) + RunSidecarLivenessTest(ctx, f, pod, 1, defaultObservationTimeout) + }) + + /* + Release: v1.28 + Testname: Pod restartable init container liveness probe, using tcp socket, no restart + Description: A Pod is created with liveness probe on tcp socket 8080. The + http handler on port 8080 will return http errors after 10 seconds, but the + socket will remain open. Liveness probe MUST not fail to check health and + the restart count should remain 0. + */ + ginkgo.It("should *not* be restarted with a tcp:8080 liveness probe", func(ctx context.Context) { + livenessProbe := &v1.Probe{ + ProbeHandler: tcpSocketHandler(8080), + InitialDelaySeconds: 15, + FailureThreshold: 1, + } + pod := livenessSidecarPodSpec(f.Namespace.Name, nil, livenessProbe) + RunSidecarLivenessTest(ctx, f, pod, 0, defaultObservationTimeout) + }) + + /* + Release: v1.28 + Testname: Pod restartable init container liveness probe, using http endpoint, multiple restarts (slow) + Description: A Pod is created with liveness probe on http endpoint + /healthz. The http handler on the /healthz will return a http error after + 10 seconds since the Pod is started. This MUST result in liveness check + failure. The Pod MUST now be killed and restarted incrementing restart + count to 1. The liveness probe must fail again after restart once the http + handler for /healthz enpoind on the Pod returns an http error after 10 + seconds from the start. Restart counts MUST increment every time health + check fails, measure up to 5 restart. + */ + ginkgo.It("should have monotonically increasing restart count", func(ctx context.Context) { + livenessProbe := &v1.Probe{ + ProbeHandler: httpGetHandler("/healthz", 8080), + InitialDelaySeconds: 5, + FailureThreshold: 1, + } + pod := livenessSidecarPodSpec(f.Namespace.Name, nil, livenessProbe) + // ~2 minutes backoff timeouts + 4 minutes defaultObservationTimeout + 2 minutes for each pod restart + RunSidecarLivenessTest(ctx, f, pod, 5, 2*time.Minute+defaultObservationTimeout+4*2*time.Minute) + }) + + /* + Release: v1.28 + Testname: Pod restartable init container liveness probe, using http endpoint, failure + Description: A Pod is created with liveness probe on http endpoint '/'. + Liveness probe on this endpoint will not fail. When liveness probe does not + fail then the restart count MUST remain zero. + */ + ginkgo.It("should *not* be restarted with a /healthz http liveness probe", func(ctx context.Context) { + livenessProbe := &v1.Probe{ + ProbeHandler: httpGetHandler("/", 80), + InitialDelaySeconds: 15, + TimeoutSeconds: 5, + FailureThreshold: 5, // to accommodate nodes which are slow in bringing up containers. + } + pod := testWebServerSidecarPodSpec(nil, livenessProbe, "test-webserver", 80) + RunSidecarLivenessTest(ctx, f, pod, 0, defaultObservationTimeout) + }) + + /* + Release: v1.28 + Testname: Pod restartable init container liveness probe, container exec timeout, restart + Description: A Pod is created with liveness probe with a Exec action on the + Pod. If the liveness probe call does not return within the timeout + specified, liveness probe MUST restart the Pod. + */ + ginkgo.It("should be restarted with an exec liveness probe with timeout [MinimumKubeletVersion:1.20]", func(ctx context.Context) { + cmd := []string{"/bin/sh", "-c", "sleep 600"} + livenessProbe := &v1.Probe{ + ProbeHandler: execHandler([]string{"/bin/sh", "-c", "sleep 10"}), + InitialDelaySeconds: 15, + TimeoutSeconds: 1, + FailureThreshold: 1, + } + pod := busyBoxSidecarPodSpec(nil, livenessProbe, cmd) + RunSidecarLivenessTest(ctx, f, pod, 1, defaultObservationTimeout) + }) + + /* + Release: v1.28 + Testname: Pod restartable init container readiness probe, container exec timeout, not ready + Description: A Pod is created with readiness probe with a Exec action on + the Pod. If the readiness probe call does not return within the timeout + specified, readiness probe MUST not be Ready. + */ + ginkgo.It("should not be ready with an exec readiness probe timeout [MinimumKubeletVersion:1.20]", func(ctx context.Context) { + cmd := []string{"/bin/sh", "-c", "sleep 600"} + readinessProbe := &v1.Probe{ + ProbeHandler: execHandler([]string{"/bin/sh", "-c", "sleep 10"}), + InitialDelaySeconds: 15, + TimeoutSeconds: 1, + FailureThreshold: 1, + } + pod := busyBoxSidecarPodSpec(readinessProbe, nil, cmd) + runReadinessFailTest(ctx, f, pod, time.Minute, false) + }) + + /* + Release: v1.28 + Testname: Pod restartalbe init container liveness probe, container exec timeout, restart + Description: A Pod is created with liveness probe with a Exec action on the + Pod. If the liveness probe call does not return within the timeout + specified, liveness probe MUST restart the Pod. When ExecProbeTimeout + feature gate is disabled and cluster is using dockershim, the timeout is + ignored BUT a failing liveness probe MUST restart the Pod. + */ + ginkgo.It("should be restarted with a failing exec liveness probe that took longer than the timeout", func(ctx context.Context) { + cmd := []string{"/bin/sh", "-c", "sleep 600"} + livenessProbe := &v1.Probe{ + ProbeHandler: execHandler([]string{"/bin/sh", "-c", "sleep 10 & exit 1"}), + InitialDelaySeconds: 15, + TimeoutSeconds: 1, + FailureThreshold: 1, + } + pod := busyBoxSidecarPodSpec(nil, livenessProbe, cmd) + RunSidecarLivenessTest(ctx, f, pod, 1, defaultObservationTimeout) + }) + + /* + Release: v1.28 + Testname: Pod restartable init container http liveness probe, redirected to a local address + Description: A Pod is created with liveness probe on http endpoint + /redirect?loc=healthz. The http handler on the /redirect will redirect to + the /healthz endpoint, which will return a http error after 10 seconds + since the Pod is started. This MUST result in liveness check failure. The + Pod MUST now be killed and restarted incrementing restart count to 1. + */ + ginkgo.It("should be restarted with a local redirect http liveness probe", func(ctx context.Context) { + livenessProbe := &v1.Probe{ + ProbeHandler: httpGetHandler("/redirect?loc="+url.QueryEscape("/healthz"), 8080), + InitialDelaySeconds: 15, + FailureThreshold: 1, + } + pod := livenessSidecarPodSpec(f.Namespace.Name, nil, livenessProbe) + RunSidecarLivenessTest(ctx, f, pod, 1, defaultObservationTimeout) + }) + + /* + Release: v1.28 + Testname: Pod restartable init container http liveness probe, redirected to a non-local address + Description: A Pod is created with liveness probe on http endpoint + /redirect with a redirect to http://0.0.0.0/. The http handler on the + /redirect should not follow the redirect, but instead treat it as a success + and generate an event. + */ + ginkgo.It("should *not* be restarted with a non-local redirect http liveness probe", func(ctx context.Context) { + livenessProbe := &v1.Probe{ + ProbeHandler: httpGetHandler("/redirect?loc="+url.QueryEscape("http://0.0.0.0/"), 8080), + InitialDelaySeconds: 15, + FailureThreshold: 1, + } + pod := livenessSidecarPodSpec(f.Namespace.Name, nil, livenessProbe) + RunSidecarLivenessTest(ctx, f, pod, 0, defaultObservationTimeout) + // Expect an event of type "ProbeWarning". + expectedEvent := fields.Set{ + "involvedObject.kind": "Pod", + "involvedObject.name": pod.Name, + "involvedObject.namespace": f.Namespace.Name, + "reason": events.ContainerProbeWarning, + }.AsSelector().String() + framework.ExpectNoError(e2eevents.WaitTimeoutForEvent( + ctx, f.ClientSet, f.Namespace.Name, expectedEvent, "Probe terminated redirects, Response body: Found.", framework.PodEventTimeout)) + }) + + /* + Release: v1.28 + Testname: Pod restartable init container startup probe restart + Description: A Pod is created with a failing startup probe. The Pod MUST be + killed and restarted incrementing restart count to 1, even if liveness + would succeed. + */ + ginkgo.It("should be restarted startup probe fails", func(ctx context.Context) { + cmd := []string{"/bin/sh", "-c", "sleep 600"} + livenessProbe := &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{"/bin/true"}, + }, + }, + InitialDelaySeconds: 15, + FailureThreshold: 1, + } + startupProbe := &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{"/bin/false"}, + }, + }, + InitialDelaySeconds: 15, + FailureThreshold: 3, + } + pod := startupSidecarPodSpec(startupProbe, nil, livenessProbe, cmd) + RunSidecarLivenessTest(ctx, f, pod, 1, defaultObservationTimeout) + }) + + /* + Release: v1.28 + Testname: Pod restartable init container liveness probe delayed (long) by startup probe + Description: A Pod is created with failing liveness and startup probes. + Liveness probe MUST NOT fail until startup probe expires. + */ + ginkgo.It("should *not* be restarted by liveness probe because startup probe delays it", func(ctx context.Context) { + cmd := []string{"/bin/sh", "-c", "sleep 600"} + livenessProbe := &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{"/bin/false"}, + }, + }, + InitialDelaySeconds: 15, + FailureThreshold: 1, + } + startupProbe := &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{"/bin/false"}, + }, + }, + InitialDelaySeconds: 15, + FailureThreshold: 60, + } + pod := startupSidecarPodSpec(startupProbe, nil, livenessProbe, cmd) + RunSidecarLivenessTest(ctx, f, pod, 0, defaultObservationTimeout) + }) + + /* + Release: v1.28 + Testname: Pod restartable init container liveness probe fails after startup success + Description: A Pod is created with failing liveness probe and delayed + startup probe that uses 'exec' command to cat /tmp/health file. The + Container is started by creating /tmp/startup after 10 seconds, triggering + liveness probe to fail. The Pod MUST not be killed and restarted + incrementing restart count to 1. + */ + ginkgo.It("should be restarted by liveness probe after startup probe enables it", func(ctx context.Context) { + cmd := []string{"/bin/sh", "-c", "sleep 10; echo ok >/tmp/startup; sleep 600"} + livenessProbe := &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{"/bin/false"}, + }, + }, + InitialDelaySeconds: 15, + FailureThreshold: 1, + } + startupProbe := &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{"cat", "/tmp/startup"}, + }, + }, + InitialDelaySeconds: 15, + FailureThreshold: 60, + } + pod := startupSidecarPodSpec(startupProbe, nil, livenessProbe, cmd) + RunSidecarLivenessTest(ctx, f, pod, 1, defaultObservationTimeout) + }) + + /* + Release: v1.28 + Testname: Pod restartable init container readiness probe, delayed by startup probe + Description: A Pod is created with startup and readiness probes. The + Container is started by creating /tmp/startup after 45 seconds, delaying + the ready state by this amount of time. This is similar to the "Pod + readiness probe, with initial delay" test. + */ + ginkgo.It("should be ready immediately after startupProbe succeeds", func(ctx context.Context) { + // Probe workers sleep at Kubelet start for a random time which is at most PeriodSeconds + // this test requires both readiness and startup workers running before updating statuses + // to avoid flakes, ensure sleep before startup (32s) > readinessProbe.PeriodSeconds + cmd := []string{"/bin/sh", "-c", "echo ok >/tmp/health; sleep 32; echo ok >/tmp/startup; sleep 600"} + readinessProbe := &v1.Probe{ + ProbeHandler: execHandler([]string{"/bin/cat", "/tmp/health"}), + InitialDelaySeconds: 0, + PeriodSeconds: 30, + } + startupProbe := &v1.Probe{ + ProbeHandler: execHandler([]string{"/bin/cat", "/tmp/startup"}), + InitialDelaySeconds: 0, + FailureThreshold: 120, + PeriodSeconds: 5, + } + p := podClient.Create(ctx, startupSidecarPodSpec(startupProbe, readinessProbe, nil, cmd)) + + p, err := podClient.Get(ctx, p.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + + err = e2epod.WaitForPodContainerStarted(ctx, f.ClientSet, f.Namespace.Name, p.Name, 0, framework.PodStartTimeout) + framework.ExpectNoError(err) + startedTime := time.Now() + + // We assume the pod became ready when the container became ready. This + // is true for a single container pod. + err = e2epod.WaitTimeoutForPodReadyInNamespace(ctx, f.ClientSet, p.Name, f.Namespace.Name, framework.PodStartTimeout) + framework.ExpectNoError(err) + readyTime := time.Now() + + p, err = podClient.Get(ctx, p.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + + isReady, err := testutils.PodRunningReady(p) + framework.ExpectNoError(err) + if !isReady { + framework.Failf("pod %s/%s should be ready", f.Namespace.Name, p.Name) + } + + readyIn := readyTime.Sub(startedTime) + framework.Logf("Container started at %v, pod became ready at %v, %v after startupProbe succeeded", startedTime, readyTime, readyIn) + if readyIn < 0 { + framework.Failf("Pod became ready before startupProbe succeeded") + } + if readyIn > 25*time.Second { + framework.Failf("Pod became ready in %v, more than 25s after startupProbe succeeded. It means that the delay readiness probes were not initiated immediately after startup finished.", readyIn) + } + }) + + // TODO: Update tests after implementing termination ordering of restartable + // init containers + /* + Release: v1.28 + Testname: Set terminationGracePeriodSeconds for livenessProbe of restartable init container + Description: A pod with a long terminationGracePeriod is created with a + shorter livenessProbe-level terminationGracePeriodSeconds. We confirm the + shorter termination period is used. + */ + ginkgo.It("should override timeoutGracePeriodSeconds when LivenessProbe field is set", func(ctx context.Context) { + cmd := []string{"/bin/sh", "-c", "sleep 1000"} + // probe will fail since pod has no http endpoints + shortGracePeriod := int64(5) + livenessProbe := &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + HTTPGet: &v1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromInt(8080), + }, + }, + InitialDelaySeconds: 10, + FailureThreshold: 1, + TerminationGracePeriodSeconds: &shortGracePeriod, + } + pod := busyBoxSidecarPodSpec(nil, livenessProbe, cmd) + longGracePeriod := int64(500) + pod.Spec.TerminationGracePeriodSeconds = &longGracePeriod + + // 10s delay + 10s period + 5s grace period = 25s < 30s << pod-level timeout 500 + // add defaultObservationTimeout(4min) more for kubelet syncing information + // to apiserver + RunSidecarLivenessTest(ctx, f, pod, 1, time.Second*40+defaultObservationTimeout) + }) + + /* + Release: v1.28 + Testname: Set terminationGracePeriodSeconds for startupProbe of restartable init container + Description: A pod with a long terminationGracePeriod is created with a + shorter startupProbe-level terminationGracePeriodSeconds. We confirm the + shorter termination period is used. + */ + ginkgo.It("should override timeoutGracePeriodSeconds when StartupProbe field is set", func(ctx context.Context) { + cmd := []string{"/bin/sh", "-c", "sleep 1000"} + // startup probe will fail since pod will sleep for 1000s before becoming ready + livenessProbe := &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{"/bin/true"}, + }, + }, + InitialDelaySeconds: 15, + FailureThreshold: 1, + } + pod := busyBoxSidecarPodSpec(nil, livenessProbe, cmd) + longGracePeriod := int64(500) + pod.Spec.TerminationGracePeriodSeconds = &longGracePeriod + + shortGracePeriod := int64(5) + pod.Spec.InitContainers[0].StartupProbe = &v1.Probe{ + ProbeHandler: execHandler([]string{"/bin/cat", "/tmp/startup"}), + InitialDelaySeconds: 10, + FailureThreshold: 1, + TerminationGracePeriodSeconds: &shortGracePeriod, + } + + // 10s delay + 10s period + 5s grace period = 25s < 30s << pod-level timeout 500 + // add defaultObservationTimeout(4min) more for kubelet syncing information + // to apiserver + RunSidecarLivenessTest(ctx, f, pod, 1, time.Second*40+defaultObservationTimeout) + }) + + /* + Release: v1.28 + Testname: Pod restartable init container liveness probe, using grpc call, success + Description: A Pod is created with liveness probe on grpc service. Liveness + probe on this endpoint will not fail. When liveness probe does not fail + then the restart count MUST remain zero. + */ + ginkgo.It("should *not* be restarted with a GRPC liveness probe", func(ctx context.Context) { + livenessProbe := &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + GRPC: &v1.GRPCAction{ + Port: 5000, + Service: nil, + }, + }, + InitialDelaySeconds: probeTestInitialDelaySeconds, + TimeoutSeconds: 5, // default 1s can be pretty aggressive in CI environments with low resources + FailureThreshold: 1, + } + + pod := gRPCServerSidecarPodSpec(nil, livenessProbe, "agnhost") + RunSidecarLivenessTest(ctx, f, pod, 0, defaultObservationTimeout) + }) + + /* + Release: v1.28 + Testname: Pod restartable init container liveness probe, using grpc call, failure + Description: A Pod is created with liveness probe on grpc service. + Liveness probe on this endpoint should fail because of wrong probe port. + When liveness probe does fail then the restart count should +1. + */ + ginkgo.It("should be restarted with a GRPC liveness probe", func(ctx context.Context) { + livenessProbe := &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + GRPC: &v1.GRPCAction{ + Port: 2333, // this port is wrong + }, + }, + InitialDelaySeconds: probeTestInitialDelaySeconds * 4, + TimeoutSeconds: 5, // default 1s can be pretty aggressive in CI environments with low resources + FailureThreshold: 1, + } + pod := gRPCServerSidecarPodSpec(nil, livenessProbe, "agnhost") + RunSidecarLivenessTest(ctx, f, pod, 1, defaultObservationTimeout) + }) + + ginkgo.It("should mark readiness on pods to false while pod is in progress of terminating when a pod has a readiness probe", func(ctx context.Context) { + podName := "probe-test-" + string(uuid.NewUUID()) + podClient := e2epod.NewPodClient(f) + terminationGracePeriod := int64(30) + script := ` +_term() { + rm -f /tmp/ready + sleep 30 + exit 0 +} +trap _term SIGTERM + +touch /tmp/ready + +while true; do + echo \"hello\" + sleep 10 +done + ` + + // Create Pod + podClient.Create(ctx, &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + }, + Spec: v1.PodSpec{ + InitContainers: []v1.Container{ + { + Image: imageutils.GetE2EImage(imageutils.Agnhost), + Name: podName, + Command: []string{"/bin/bash"}, + Args: []string{"-c", script}, + ReadinessProbe: &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{"cat", "/tmp/ready"}, + }, + }, + FailureThreshold: 1, + InitialDelaySeconds: 5, + PeriodSeconds: 2, + }, + RestartPolicy: func() *v1.ContainerRestartPolicy { + restartPolicy := v1.ContainerRestartPolicyAlways + return &restartPolicy + }(), + }, + }, + Containers: []v1.Container{ + { + Name: "main", + Image: imageutils.GetE2EImage(imageutils.Agnhost), + Args: []string{"pause"}, + }, + }, + TerminationGracePeriodSeconds: &terminationGracePeriod, + }, + }) + + // verify pods are running and ready + err := e2epod.WaitForPodsRunningReady(ctx, f.ClientSet, f.Namespace.Name, 1, 0, f.Timeouts.PodStart) + framework.ExpectNoError(err) + + // Shutdown pod. Readiness should change to false + err = podClient.Delete(ctx, podName, metav1.DeleteOptions{}) + framework.ExpectNoError(err) + + err = waitForPodStatusByInformer(ctx, f.ClientSet, f.Namespace.Name, podName, f.Timeouts.PodDelete, func(pod *v1.Pod) (bool, error) { + if !podutil.IsPodReady(pod) { + return true, nil + } + framework.Logf("pod %s/%s is still ready, waiting until is not ready", pod.Namespace, pod.Name) + return false, nil + }) + framework.ExpectNoError(err) + }) + + ginkgo.It("should mark readiness on pods to false and disable liveness probes while pod is in progress of terminating", func(ctx context.Context) { + podName := "probe-test-" + string(uuid.NewUUID()) + podClient := e2epod.NewPodClient(f) + terminationGracePeriod := int64(30) + script := ` +_term() { + rm -f /tmp/ready + rm -f /tmp/liveness + sleep 20 + exit 0 +} +trap _term SIGTERM + +touch /tmp/ready +touch /tmp/liveness + +while true; do + echo \"hello\" + sleep 10 +done +` + + // Create Pod + podClient.Create(ctx, &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + }, + Spec: v1.PodSpec{ + InitContainers: []v1.Container{ + { + Image: imageutils.GetE2EImage(imageutils.Agnhost), + Name: podName, + Command: []string{"/bin/bash"}, + Args: []string{"-c", script}, + ReadinessProbe: &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{"cat", "/tmp/ready"}, + }, + }, + FailureThreshold: 1, + // delay startup to make sure the script script has + // time to create the ready+liveness files + InitialDelaySeconds: 5, + PeriodSeconds: 2, + }, + LivenessProbe: &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{"cat", "/tmp/liveness"}, + }, + }, + FailureThreshold: 1, + // delay startup to make sure the script script has + // time to create the ready+liveness files + InitialDelaySeconds: 5, + PeriodSeconds: 1, + }, + RestartPolicy: func() *v1.ContainerRestartPolicy { + restartPolicy := v1.ContainerRestartPolicyAlways + return &restartPolicy + }(), + }, + }, + Containers: []v1.Container{ + { + Name: "main", + Image: imageutils.GetE2EImage(imageutils.Agnhost), + Args: []string{"pause"}, + }, + }, + TerminationGracePeriodSeconds: &terminationGracePeriod, + }, + }) + + // verify pods are running and ready + err := e2epod.WaitForPodsRunningReady(ctx, f.ClientSet, f.Namespace.Name, 1, 0, f.Timeouts.PodStart) + framework.ExpectNoError(err) + + // Shutdown pod. Readiness should change to false + err = podClient.Delete(ctx, podName, metav1.DeleteOptions{}) + framework.ExpectNoError(err) + + // Wait for pod to go unready + err = waitForPodStatusByInformer(ctx, f.ClientSet, f.Namespace.Name, podName, f.Timeouts.PodDelete, func(pod *v1.Pod) (bool, error) { + if !podutil.IsPodReady(pod) { + return true, nil + } + framework.Logf("pod %s/%s is still ready, waiting until is not ready", pod.Namespace, pod.Name) + return false, nil + }) + framework.ExpectNoError(err) + + // Verify there are zero liveness failures since they are turned off + // during pod termination + gomega.Consistently(ctx, func(ctx context.Context) (bool, error) { + items, err := f.ClientSet.CoreV1().Events(f.Namespace.Name).List(ctx, metav1.ListOptions{}) + framework.ExpectNoError(err) + for _, event := range items.Items { + // Search only for the pod we are interested in + if event.InvolvedObject.Name != podName { + continue + } + if strings.Contains(event.Message, "failed liveness probe") { + return true, errors.New("should not see liveness probe failures") + } + } + return false, nil + }, 1*time.Minute, framework.Poll).ShouldNot(gomega.BeTrue(), "should not see liveness probes") + }) +}) + // waitForPodStatusByInformer waits pod status change by informer func waitForPodStatusByInformer(ctx context.Context, c clientset.Interface, podNamespace, podName string, timeout time.Duration, condition func(pod *v1.Pod) (bool, error)) error { // TODO (pohly): rewrite with gomega.Eventually to get intermediate progress reports. @@ -795,7 +1551,7 @@ func newInformerWatchPod(ctx context.Context, c clientset.Interface, podNamespac // GetContainerStartedTime returns the time when the given container started and error if any func GetContainerStartedTime(p *v1.Pod, containerName string) (time.Time, error) { - for _, status := range p.Status.ContainerStatuses { + for _, status := range append(p.Status.InitContainerStatuses, p.Status.ContainerStatuses...) { if status.Name != containerName { continue } @@ -819,7 +1575,7 @@ func GetTransitionTimeForReadyCondition(p *v1.Pod) (time.Time, error) { func getRestartCount(p *v1.Pod) int { count := 0 - for _, containerStatus := range p.Status.ContainerStatuses { + for _, containerStatus := range append(p.Status.InitContainerStatuses, p.Status.ContainerStatuses...) { count += int(containerStatus.RestartCount) } return count @@ -945,12 +1701,22 @@ func (b webserverProbeBuilder) build() *v1.Probe { return probe } -// RunLivenessTest verifies the number of restarts for pod with given expected number of restarts func RunLivenessTest(ctx context.Context, f *framework.Framework, pod *v1.Pod, expectNumRestarts int, timeout time.Duration) { - podClient := e2epod.NewPodClient(f) - ns := f.Namespace.Name gomega.Expect(pod.Spec.Containers).NotTo(gomega.BeEmpty()) containerName := pod.Spec.Containers[0].Name + runLivenessTest(ctx, f, pod, expectNumRestarts, timeout, containerName) +} + +func RunSidecarLivenessTest(ctx context.Context, f *framework.Framework, pod *v1.Pod, expectNumRestarts int, timeout time.Duration) { + gomega.Expect(pod.Spec.InitContainers).NotTo(gomega.BeEmpty()) + containerName := pod.Spec.InitContainers[0].Name + runLivenessTest(ctx, f, pod, expectNumRestarts, timeout, containerName) +} + +// RunLivenessTest verifies the number of restarts for pod with given expected number of restarts +func runLivenessTest(ctx context.Context, f *framework.Framework, pod *v1.Pod, expectNumRestarts int, timeout time.Duration, containerName string) { + podClient := e2epod.NewPodClient(f) + ns := f.Namespace.Name // At the end of the test, clean up by removing the pod. ginkgo.DeferCleanup(func(ctx context.Context) error { ginkgo.By("deleting the pod") @@ -959,18 +1725,24 @@ func RunLivenessTest(ctx context.Context, f *framework.Framework, pod *v1.Pod, e ginkgo.By(fmt.Sprintf("Creating pod %s in namespace %s", pod.Name, ns)) podClient.Create(ctx, pod) - // Wait until the pod is not pending. (Here we need to check for something other than - // 'Pending' other than checking for 'Running', since when failures occur, we go to - // 'Terminated' which can cause indefinite blocking.) - framework.ExpectNoError(e2epod.WaitForPodNotPending(ctx, f.ClientSet, ns, pod.Name), - fmt.Sprintf("starting pod %s in namespace %s", pod.Name, ns)) - framework.Logf("Started pod %s in namespace %s", pod.Name, ns) + // To check for the container is ever started, we need to wait for the + // container to be in a non-waiting state. + framework.ExpectNoError(e2epod.WaitForPodCondition(ctx, f.ClientSet, ns, pod.Name, "container not waiting", timeout, func(pod *v1.Pod) (bool, error) { + for _, c := range append(pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses...) { + if c.Name == containerName { + if c.State.Running != nil || c.State.Terminated != nil { + return true, nil + } + } + } + return false, nil + })) // Check the pod's current state and verify that restartCount is present. ginkgo.By("checking the pod's current state and verifying that restartCount is present") pod, err := podClient.Get(ctx, pod.Name, metav1.GetOptions{}) framework.ExpectNoError(err, fmt.Sprintf("getting pod %s in namespace %s", pod.Name, ns)) - initialRestartCount := podutil.GetExistingContainerStatus(pod.Status.ContainerStatuses, containerName).RestartCount + initialRestartCount := podutil.GetExistingContainerStatus(append(pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses...), containerName).RestartCount framework.Logf("Initial restart count of pod %s is %d", pod.Name, initialRestartCount) // Wait for the restart state to be as desired. @@ -983,7 +1755,7 @@ func RunLivenessTest(ctx context.Context, f *framework.Framework, pod *v1.Pod, e pod, err = podClient.Get(ctx, pod.Name, metav1.GetOptions{}) framework.Logf("Get pod %s in namespace %s", pod.Name, ns) framework.ExpectNoError(err, fmt.Sprintf("getting pod %s", pod.Name)) - restartCount := podutil.GetExistingContainerStatus(pod.Status.ContainerStatuses, containerName).RestartCount + restartCount := podutil.GetExistingContainerStatus(append(pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses...), containerName).RestartCount if restartCount != lastRestartCount { framework.Logf("Restart count of pod %s/%s is now %d (%v elapsed)", ns, pod.Name, restartCount, time.Since(start)) @@ -1009,7 +1781,7 @@ func RunLivenessTest(ctx context.Context, f *framework.Framework, pod *v1.Pod, e } } -func runReadinessFailTest(ctx context.Context, f *framework.Framework, pod *v1.Pod, notReadyUntil time.Duration) { +func runReadinessFailTest(ctx context.Context, f *framework.Framework, pod *v1.Pod, notReadyUntil time.Duration, waitForNotPending bool) { podClient := e2epod.NewPodClient(f) ns := f.Namespace.Name gomega.Expect(pod.Spec.Containers).NotTo(gomega.BeEmpty()) @@ -1022,11 +1794,13 @@ func runReadinessFailTest(ctx context.Context, f *framework.Framework, pod *v1.P ginkgo.By(fmt.Sprintf("Creating pod %s in namespace %s", pod.Name, ns)) podClient.Create(ctx, pod) - // Wait until the pod is not pending. (Here we need to check for something other than - // 'Pending', since when failures occur, we go to 'Terminated' which can cause indefinite blocking.) - framework.ExpectNoError(e2epod.WaitForPodNotPending(ctx, f.ClientSet, ns, pod.Name), - fmt.Sprintf("starting pod %s in namespace %s", pod.Name, ns)) - framework.Logf("Started pod %s in namespace %s", pod.Name, ns) + if waitForNotPending { + // Wait until the pod is not pending. (Here we need to check for something other than + // 'Pending', since when failures occur, we go to 'Terminated' which can cause indefinite blocking.) + framework.ExpectNoError(e2epod.WaitForPodNotPending(ctx, f.ClientSet, ns, pod.Name), + fmt.Sprintf("starting pod %s in namespace %s", pod.Name, ns)) + framework.Logf("Started pod %s in namespace %s", pod.Name, ns) + } // Wait for the not ready state to be true for notReadyUntil duration deadline := time.Now().Add(notReadyUntil) @@ -1061,3 +1835,159 @@ func gRPCServerPodSpec(readinessProbe, livenessProbe *v1.Probe, containerName st }, } } + +func testWebServerSidecarPodSpec(readinessProbe, livenessProbe *v1.Probe, containerName string, port int) *v1.Pod { + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "test-webserver-sidecar-" + string(uuid.NewUUID())}, + Spec: v1.PodSpec{ + InitContainers: []v1.Container{ + { + Name: containerName, + Image: imageutils.GetE2EImage(imageutils.Agnhost), + Args: []string{"test-webserver", "--port", fmt.Sprintf("%d", port)}, + Ports: []v1.ContainerPort{{ContainerPort: int32(port)}}, + LivenessProbe: livenessProbe, + ReadinessProbe: readinessProbe, + RestartPolicy: func() *v1.ContainerRestartPolicy { + restartPolicy := v1.ContainerRestartPolicyAlways + return &restartPolicy + }(), + }, + }, + Containers: []v1.Container{ + { + Name: "main", + Image: imageutils.GetE2EImage(imageutils.Agnhost), + Args: []string{"pause"}, + }, + }, + }, + } +} + +func busyBoxSidecarPodSpec(readinessProbe, livenessProbe *v1.Probe, cmd []string) *v1.Pod { + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "busybox-sidecar-" + string(uuid.NewUUID()), + Labels: map[string]string{"test": "liveness"}, + }, + Spec: v1.PodSpec{ + InitContainers: []v1.Container{ + { + Name: "busybox", + Image: imageutils.GetE2EImage(imageutils.BusyBox), + Command: cmd, + LivenessProbe: livenessProbe, + ReadinessProbe: readinessProbe, + RestartPolicy: func() *v1.ContainerRestartPolicy { + restartPolicy := v1.ContainerRestartPolicyAlways + return &restartPolicy + }(), + }, + }, + Containers: []v1.Container{ + { + Name: "main", + Image: imageutils.GetE2EImage(imageutils.Agnhost), + Args: []string{"pause"}, + }, + }, + }, + } +} + +func livenessSidecarPodSpec(namespace string, readinessProbe, livenessProbe *v1.Probe) *v1.Pod { + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-liveness-sidecar-" + string(uuid.NewUUID()), + Labels: map[string]string{"test": "liveness"}, + Namespace: namespace, + }, + Spec: v1.PodSpec{ + InitContainers: []v1.Container{ + { + Name: "sidecar", + Image: imageutils.GetE2EImage(imageutils.Agnhost), + Args: []string{"liveness"}, + LivenessProbe: livenessProbe, + ReadinessProbe: readinessProbe, + RestartPolicy: func() *v1.ContainerRestartPolicy { + restartPolicy := v1.ContainerRestartPolicyAlways + return &restartPolicy + }(), + }, + }, + Containers: []v1.Container{ + { + Name: "main", + Image: imageutils.GetE2EImage(imageutils.Agnhost), + Args: []string{"pause"}, + }, + }, + }, + } +} + +func startupSidecarPodSpec(startupProbe, readinessProbe, livenessProbe *v1.Probe, cmd []string) *v1.Pod { + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "startup-sidecar-" + string(uuid.NewUUID()), + Labels: map[string]string{"test": "startup"}, + }, + Spec: v1.PodSpec{ + InitContainers: []v1.Container{ + { + Name: "sidecar", + Image: imageutils.GetE2EImage(imageutils.BusyBox), + Command: cmd, + LivenessProbe: livenessProbe, + ReadinessProbe: readinessProbe, + StartupProbe: startupProbe, + RestartPolicy: func() *v1.ContainerRestartPolicy { + restartPolicy := v1.ContainerRestartPolicyAlways + return &restartPolicy + }(), + }, + }, + Containers: []v1.Container{ + { + Name: "main", + Image: imageutils.GetE2EImage(imageutils.Agnhost), + Args: []string{"pause"}, + }, + }, + }, + } +} + +func gRPCServerSidecarPodSpec(readinessProbe, livenessProbe *v1.Probe, containerName string) *v1.Pod { + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "test-grpc-sidecar-" + string(uuid.NewUUID())}, + Spec: v1.PodSpec{ + InitContainers: []v1.Container{ + { + Name: containerName, + Image: imageutils.GetE2EImage(imageutils.Agnhost), + Command: []string{ + "/agnhost", + "grpc-health-checking", + }, + Ports: []v1.ContainerPort{{ContainerPort: int32(5000)}, {ContainerPort: int32(8080)}}, + LivenessProbe: livenessProbe, + ReadinessProbe: readinessProbe, + RestartPolicy: func() *v1.ContainerRestartPolicy { + restartPolicy := v1.ContainerRestartPolicyAlways + return &restartPolicy + }(), + }, + }, + Containers: []v1.Container{ + { + Name: "main", + Image: imageutils.GetE2EImage(imageutils.Agnhost), + Args: []string{"pause"}, + }, + }, + }, + } +} diff --git a/test/e2e/common/node/lifecycle_hook.go b/test/e2e/common/node/lifecycle_hook.go index 6390e5ad79a..8f5a330b6eb 100644 --- a/test/e2e/common/node/lifecycle_hook.go +++ b/test/e2e/common/node/lifecycle_hook.go @@ -253,6 +253,254 @@ var _ = SIGDescribe("Container Lifecycle Hook", func() { }) }) +var _ = SIGDescribe("[NodeAlphaFeature:SidecarContainers][Feature:SidecarContainers] Restartable Init Container Lifecycle Hook", func() { + f := framework.NewDefaultFramework("restartable-init-container-lifecycle-hook") + f.NamespacePodSecurityLevel = admissionapi.LevelBaseline + var podClient *e2epod.PodClient + const ( + podCheckInterval = 1 * time.Second + postStartWaitTimeout = 2 * time.Minute + preStopWaitTimeout = 30 * time.Second + ) + ginkgo.Context("when create a pod with lifecycle hook", func() { + var ( + targetIP, targetURL, targetNode string + + httpPorts = []v1.ContainerPort{ + { + ContainerPort: 8080, + Protocol: v1.ProtocolTCP, + }, + } + httpsPorts = []v1.ContainerPort{ + { + ContainerPort: 9090, + Protocol: v1.ProtocolTCP, + }, + } + httpsArgs = []string{ + "netexec", + "--http-port", "9090", + "--udp-port", "9091", + "--tls-cert-file", "/localhost.crt", + "--tls-private-key-file", "/localhost.key", + } + ) + + podHandleHookRequest := e2epod.NewAgnhostPodFromContainers( + "", "pod-handle-http-request", nil, + e2epod.NewAgnhostContainer("container-handle-http-request", nil, httpPorts, "netexec"), + e2epod.NewAgnhostContainer("container-handle-https-request", nil, httpsPorts, httpsArgs...), + ) + + ginkgo.BeforeEach(func(ctx context.Context) { + node, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet) + framework.ExpectNoError(err) + targetNode = node.Name + nodeSelection := e2epod.NodeSelection{} + e2epod.SetAffinity(&nodeSelection, targetNode) + e2epod.SetNodeSelection(&podHandleHookRequest.Spec, nodeSelection) + + podClient = e2epod.NewPodClient(f) + ginkgo.By("create the container to handle the HTTPGet hook request.") + newPod := podClient.CreateSync(ctx, podHandleHookRequest) + targetIP = newPod.Status.PodIP + targetURL = targetIP + if strings.Contains(targetIP, ":") { + targetURL = fmt.Sprintf("[%s]", targetIP) + } + }) + testPodWithHook := func(ctx context.Context, podWithHook *v1.Pod) { + ginkgo.By("create the pod with lifecycle hook") + podClient.CreateSync(ctx, podWithHook) + const ( + defaultHandler = iota + httpsHandler + ) + handlerContainer := defaultHandler + if podWithHook.Spec.InitContainers[0].Lifecycle.PostStart != nil { + ginkgo.By("check poststart hook") + if podWithHook.Spec.InitContainers[0].Lifecycle.PostStart.HTTPGet != nil { + if v1.URISchemeHTTPS == podWithHook.Spec.InitContainers[0].Lifecycle.PostStart.HTTPGet.Scheme { + handlerContainer = httpsHandler + } + } + gomega.Eventually(ctx, func(ctx context.Context) error { + return podClient.MatchContainerOutput(ctx, podHandleHookRequest.Name, podHandleHookRequest.Spec.Containers[handlerContainer].Name, + `GET /echo\?msg=poststart`) + }, postStartWaitTimeout, podCheckInterval).Should(gomega.BeNil()) + } + ginkgo.By("delete the pod with lifecycle hook") + podClient.DeleteSync(ctx, podWithHook.Name, *metav1.NewDeleteOptions(15), e2epod.DefaultPodDeletionTimeout) + if podWithHook.Spec.InitContainers[0].Lifecycle.PreStop != nil { + ginkgo.By("check prestop hook") + if podWithHook.Spec.InitContainers[0].Lifecycle.PreStop.HTTPGet != nil { + if v1.URISchemeHTTPS == podWithHook.Spec.InitContainers[0].Lifecycle.PreStop.HTTPGet.Scheme { + handlerContainer = httpsHandler + } + } + gomega.Eventually(ctx, func(ctx context.Context) error { + return podClient.MatchContainerOutput(ctx, podHandleHookRequest.Name, podHandleHookRequest.Spec.Containers[handlerContainer].Name, + `GET /echo\?msg=prestop`) + }, preStopWaitTimeout, podCheckInterval).Should(gomega.BeNil()) + } + } + /* + Release: v1.28 + Testname: Pod Lifecycle with restartable init container, post start exec hook + Description: When a post start handler is specified in the container + lifecycle using a 'Exec' action, then the handler MUST be invoked after + the start of the container. A server pod is created that will serve http + requests, create a second pod with a container lifecycle specifying a + post start that invokes the server pod using ExecAction to validate that + the post start is executed. + */ + ginkgo.It("should execute poststart exec hook properly", func(ctx context.Context) { + lifecycle := &v1.Lifecycle{ + PostStart: &v1.LifecycleHandler{ + Exec: &v1.ExecAction{ + Command: []string{"sh", "-c", "curl http://" + targetURL + ":8080/echo?msg=poststart"}, + }, + }, + } + podWithHook := getSidecarPodWithHook("pod-with-poststart-exec-hook", imageutils.GetE2EImage(imageutils.Agnhost), lifecycle) + + testPodWithHook(ctx, podWithHook) + }) + /* + Release: v1.28 + Testname: Pod Lifecycle with restartable init container, prestop exec hook + Description: When a pre-stop handler is specified in the container + lifecycle using a 'Exec' action, then the handler MUST be invoked before + the container is terminated. A server pod is created that will serve http + requests, create a second pod with a container lifecycle specifying a + pre-stop that invokes the server pod using ExecAction to validate that + the pre-stop is executed. + */ + ginkgo.It("should execute prestop exec hook properly", func(ctx context.Context) { + lifecycle := &v1.Lifecycle{ + PreStop: &v1.LifecycleHandler{ + Exec: &v1.ExecAction{ + Command: []string{"sh", "-c", "curl http://" + targetURL + ":8080/echo?msg=prestop"}, + }, + }, + } + podWithHook := getSidecarPodWithHook("pod-with-prestop-exec-hook", imageutils.GetE2EImage(imageutils.Agnhost), lifecycle) + testPodWithHook(ctx, podWithHook) + }) + /* + Release: v1.28 + Testname: Pod Lifecycle with restartable init container, post start http hook + Description: When a post start handler is specified in the container + lifecycle using a HttpGet action, then the handler MUST be invoked after + the start of the container. A server pod is created that will serve http + requests, create a second pod on the same node with a container lifecycle + specifying a post start that invokes the server pod to validate that the + post start is executed. + */ + ginkgo.It("should execute poststart http hook properly", func(ctx context.Context) { + lifecycle := &v1.Lifecycle{ + PostStart: &v1.LifecycleHandler{ + HTTPGet: &v1.HTTPGetAction{ + Path: "/echo?msg=poststart", + Host: targetIP, + Port: intstr.FromInt(8080), + }, + }, + } + podWithHook := getSidecarPodWithHook("pod-with-poststart-http-hook", imageutils.GetPauseImageName(), lifecycle) + // make sure we spawn the test pod on the same node as the webserver. + nodeSelection := e2epod.NodeSelection{} + e2epod.SetAffinity(&nodeSelection, targetNode) + e2epod.SetNodeSelection(&podWithHook.Spec, nodeSelection) + testPodWithHook(ctx, podWithHook) + }) + /* + Release : v1.28 + Testname: Pod Lifecycle with restartable init container, poststart https hook + Description: When a post-start handler is specified in the container + lifecycle using a 'HttpGet' action, then the handler MUST be invoked + before the container is terminated. A server pod is created that will + serve https requests, create a second pod on the same node with a + container lifecycle specifying a post-start that invokes the server pod + to validate that the post-start is executed. + */ + ginkgo.It("should execute poststart https hook properly [MinimumKubeletVersion:1.23]", func(ctx context.Context) { + lifecycle := &v1.Lifecycle{ + PostStart: &v1.LifecycleHandler{ + HTTPGet: &v1.HTTPGetAction{ + Scheme: v1.URISchemeHTTPS, + Path: "/echo?msg=poststart", + Host: targetIP, + Port: intstr.FromInt(9090), + }, + }, + } + podWithHook := getSidecarPodWithHook("pod-with-poststart-https-hook", imageutils.GetPauseImageName(), lifecycle) + // make sure we spawn the test pod on the same node as the webserver. + nodeSelection := e2epod.NodeSelection{} + e2epod.SetAffinity(&nodeSelection, targetNode) + e2epod.SetNodeSelection(&podWithHook.Spec, nodeSelection) + testPodWithHook(ctx, podWithHook) + }) + /* + Release : v1.28 + Testname: Pod Lifecycle with restartable init container, prestop http hook + Description: When a pre-stop handler is specified in the container + lifecycle using a 'HttpGet' action, then the handler MUST be invoked + before the container is terminated. A server pod is created that will + serve http requests, create a second pod on the same node with a + container lifecycle specifying a pre-stop that invokes the server pod to + validate that the pre-stop is executed. + */ + ginkgo.It("should execute prestop http hook properly", func(ctx context.Context) { + lifecycle := &v1.Lifecycle{ + PreStop: &v1.LifecycleHandler{ + HTTPGet: &v1.HTTPGetAction{ + Path: "/echo?msg=prestop", + Host: targetIP, + Port: intstr.FromInt(8080), + }, + }, + } + podWithHook := getSidecarPodWithHook("pod-with-prestop-http-hook", imageutils.GetPauseImageName(), lifecycle) + // make sure we spawn the test pod on the same node as the webserver. + nodeSelection := e2epod.NodeSelection{} + e2epod.SetAffinity(&nodeSelection, targetNode) + e2epod.SetNodeSelection(&podWithHook.Spec, nodeSelection) + testPodWithHook(ctx, podWithHook) + }) + /* + Release : v1.28 + Testname: Pod Lifecycle with restartable init container, prestop https hook + Description: When a pre-stop handler is specified in the container + lifecycle using a 'HttpGet' action, then the handler MUST be invoked + before the container is terminated. A server pod is created that will + serve https requests, create a second pod on the same node with a + container lifecycle specifying a pre-stop that invokes the server pod to + validate that the pre-stop is executed. + */ + ginkgo.It("should execute prestop https hook properly [MinimumKubeletVersion:1.23]", func(ctx context.Context) { + lifecycle := &v1.Lifecycle{ + PreStop: &v1.LifecycleHandler{ + HTTPGet: &v1.HTTPGetAction{ + Scheme: v1.URISchemeHTTPS, + Path: "/echo?msg=prestop", + Host: targetIP, + Port: intstr.FromInt(9090), + }, + }, + } + podWithHook := getSidecarPodWithHook("pod-with-prestop-https-hook", imageutils.GetPauseImageName(), lifecycle) + // make sure we spawn the test pod on the same node as the webserver. + nodeSelection := e2epod.NodeSelection{} + e2epod.SetAffinity(&nodeSelection, targetNode) + e2epod.SetNodeSelection(&podWithHook.Spec, nodeSelection) + testPodWithHook(ctx, podWithHook) + }) + }) +}) + func getPodWithHook(name string, image string, lifecycle *v1.Lifecycle) *v1.Pod { return &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -269,3 +517,30 @@ func getPodWithHook(name string, image string, lifecycle *v1.Lifecycle) *v1.Pod }, } } + +func getSidecarPodWithHook(name string, image string, lifecycle *v1.Lifecycle) *v1.Pod { + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: v1.PodSpec{ + InitContainers: []v1.Container{ + { + Name: name, + Image: image, + Lifecycle: lifecycle, + RestartPolicy: func() *v1.ContainerRestartPolicy { + restartPolicy := v1.ContainerRestartPolicyAlways + return &restartPolicy + }(), + }, + }, + Containers: []v1.Container{ + { + Name: "main", + Image: imageutils.GetPauseImageName(), + }, + }, + }, + } +} diff --git a/test/e2e_node/container_lifecycle_test.go b/test/e2e_node/container_lifecycle_test.go index 66aa440e93d..29512c33947 100644 --- a/test/e2e_node/container_lifecycle_test.go +++ b/test/e2e_node/container_lifecycle_test.go @@ -845,7 +845,7 @@ var _ = SIGDescribe("[NodeAlphaFeature:SidecarContainers] Containers Lifecycle " framework.ExpectNoError(results.StartsBefore(restartableInit2, init3)) }) - ginkgo.It("should run both restartable init cotnainers and third init container together", func() { + ginkgo.It("should run both restartable init containers and third init container together", func() { framework.ExpectNoError(results.RunTogether(restartableInit2, restartableInit1)) framework.ExpectNoError(results.RunTogether(restartableInit1, init3)) framework.ExpectNoError(results.RunTogether(restartableInit2, init3)) @@ -856,7 +856,7 @@ var _ = SIGDescribe("[NodeAlphaFeature:SidecarContainers] Containers Lifecycle " framework.ExpectNoError(results.ExitsBefore(init3, regular1)) }) - ginkgo.It("should run both restartable init cotnainers and a regular container together", func() { + ginkgo.It("should run both restartable init containers and a regular container together", func() { framework.ExpectNoError(results.RunTogether(restartableInit1, regular1)) framework.ExpectNoError(results.RunTogether(restartableInit2, regular1)) }) @@ -1249,7 +1249,6 @@ var _ = SIGDescribe("[NodeAlphaFeature:SidecarContainers] Containers Lifecycle " ginkgo.It("should be running restartable init container and a failed Init container in parallel", func() { framework.ExpectNoError(results.RunTogether(restartableInit1, init1)) }) - // TODO: check preStop hooks when they are enabled }) }) @@ -1661,7 +1660,6 @@ var _ = SIGDescribe("[NodeAlphaFeature:SidecarContainers] Containers Lifecycle " ginkgo.It("should be running restartable init container and a failed Init container in parallel", func() { framework.ExpectNoError(results.RunTogether(restartableInit1, init1)) }) - // TODO: check preStop hooks when they are enabled }) }) @@ -2075,11 +2073,10 @@ var _ = SIGDescribe("[NodeAlphaFeature:SidecarContainers] Containers Lifecycle " ginkgo.It("should be running restartable init container and a failed Init container in parallel", func() { framework.ExpectNoError(results.RunTogether(restartableInit1, init1)) }) - // TODO: check preStop hooks when they are enabled }) }) - ginkgo.It("should launch restartable init cotnainers serially considering the startup probe", func() { + ginkgo.It("should launch restartable init containers serially considering the startup probe", func() { restartableInit1 := "restartable-init-1" restartableInit2 := "restartable-init-2" @@ -2158,7 +2155,7 @@ var _ = SIGDescribe("[NodeAlphaFeature:SidecarContainers] Containers Lifecycle " framework.ExpectNoError(results.StartsBefore(restartableInit2, regular1)) }) - ginkgo.It("should not launch next container if the restartable init cotnainer failed to complete startup probe", func() { + ginkgo.It("should call the container's preStop hook and not launch next container if the restartable init container's startup probe fails", func() { restartableInit1 := "restartable-init-1" regular1 := "regular-1" @@ -2174,16 +2171,30 @@ var _ = SIGDescribe("[NodeAlphaFeature:SidecarContainers] Containers Lifecycle " Name: restartableInit1, Image: busyboxImage, Command: ExecCommand(restartableInit1, execCommand{ - StartDelay: 30, - Delay: 600, - ExitCode: 0, + Delay: 600, + TerminationSeconds: 15, + ExitCode: 0, }), StartupProbe: &v1.Probe{ - PeriodSeconds: 1, - FailureThreshold: 1, + InitialDelaySeconds: 5, + FailureThreshold: 1, ProbeHandler: v1.ProbeHandler{ Exec: &v1.ExecAction{ - Command: []string{"test", "-f", "started"}, + Command: []string{ + "sh", + "-c", + "exit 1", + }, + }, + }, + }, + Lifecycle: &v1.Lifecycle{ + PreStop: &v1.LifecycleHandler{ + Exec: &v1.ExecAction{ + Command: ExecCommand(prefixedName(PreStopPrefix, restartableInit1), execCommand{ + Delay: 1, + ExitCode: 0, + }), }, }, }, @@ -2208,7 +2219,7 @@ var _ = SIGDescribe("[NodeAlphaFeature:SidecarContainers] Containers Lifecycle " client := e2epod.NewPodClient(f) pod = client.Create(context.TODO(), pod) - ginkgo.By("Waiting for the restartable init cotnainer to restart") + ginkgo.By("Waiting for the restartable init container to restart") err := WaitForPodInitContainerRestartCount(context.TODO(), f.ClientSet, pod.Namespace, pod.Name, 0, 2, 2*time.Minute) framework.ExpectNoError(err) @@ -2222,6 +2233,92 @@ var _ = SIGDescribe("[NodeAlphaFeature:SidecarContainers] Containers Lifecycle " results := parseOutput(pod) ginkgo.By("Analyzing results") + framework.ExpectNoError(results.RunTogether(restartableInit1, prefixedName(PreStopPrefix, restartableInit1))) + framework.ExpectNoError(results.Starts(prefixedName(PreStopPrefix, restartableInit1))) + framework.ExpectNoError(results.Exits(restartableInit1)) framework.ExpectNoError(results.DoesntStart(regular1)) }) + + ginkgo.It("should call the container's preStop hook and start the next container if the restartable init container's liveness probe fails", func() { + + restartableInit1 := "restartable-init-1" + regular1 := "regular-1" + + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "restartable-init-container-failed-startup", + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyAlways, + InitContainers: []v1.Container{ + { + Name: restartableInit1, + Image: busyboxImage, + Command: ExecCommand(restartableInit1, execCommand{ + Delay: 600, + TerminationSeconds: 15, + ExitCode: 0, + }), + LivenessProbe: &v1.Probe{ + InitialDelaySeconds: 5, + FailureThreshold: 1, + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{ + "sh", + "-c", + "exit 1", + }, + }, + }, + }, + Lifecycle: &v1.Lifecycle{ + PreStop: &v1.LifecycleHandler{ + Exec: &v1.ExecAction{ + Command: ExecCommand(prefixedName(PreStopPrefix, restartableInit1), execCommand{ + Delay: 1, + ExitCode: 0, + }), + }, + }, + }, + RestartPolicy: &containerRestartPolicyAlways, + }, + }, + Containers: []v1.Container{ + { + Name: regular1, + Image: busyboxImage, + Command: ExecCommand(regular1, execCommand{ + Delay: 1, + ExitCode: 0, + }), + }, + }, + }, + } + + preparePod(pod) + + client := e2epod.NewPodClient(f) + pod = client.Create(context.TODO(), pod) + + ginkgo.By("Waiting for the restartable init container to restart") + err := WaitForPodInitContainerRestartCount(context.TODO(), f.ClientSet, pod.Namespace, pod.Name, 0, 2, 2*time.Minute) + framework.ExpectNoError(err) + + err = WaitForPodContainerRestartCount(context.TODO(), f.ClientSet, pod.Namespace, pod.Name, 0, 1, 2*time.Minute) + framework.ExpectNoError(err) + + pod, err = client.Get(context.TODO(), pod.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + + results := parseOutput(pod) + + ginkgo.By("Analyzing results") + framework.ExpectNoError(results.RunTogether(restartableInit1, prefixedName(PreStopPrefix, restartableInit1))) + framework.ExpectNoError(results.Starts(prefixedName(PreStopPrefix, restartableInit1))) + framework.ExpectNoError(results.Exits(restartableInit1)) + framework.ExpectNoError(results.Starts(regular1)) + }) })