From c74ec3df094d526c0f08844652152e4ddb626884 Mon Sep 17 00:00:00 2001 From: AxeZhan Date: Sat, 23 Dec 2023 00:05:30 +0800 Subject: [PATCH] graduate PodLifecycleSleepAction to beta --- pkg/api/pod/util.go | 75 +++++--- pkg/api/pod/util_test.go | 233 +++++++++++++++++++++++++ pkg/features/kube_features.go | 3 +- pkg/kubelet/lifecycle/handlers.go | 1 + pkg/kubelet/metrics/metrics.go | 10 ++ pkg/registry/core/pod/strategy_test.go | 96 ++++++++++ test/e2e/common/node/lifecycle_hook.go | 48 ++++- test/e2e/framework/pod/wait.go | 15 ++ 8 files changed, 444 insertions(+), 37 deletions(-) diff --git a/pkg/api/pod/util.go b/pkg/api/pod/util.go index 2d73f582e7b..959f16aafc4 100644 --- a/pkg/api/pod/util.go +++ b/pkg/api/pod/util.go @@ -594,39 +594,56 @@ func dropDisabledFields( // For other types of containers, validateContainers will handle them. } - if !utilfeature.DefaultFeatureGate.Enabled(features.PodLifecycleSleepAction) && !podLifecycleSleepActionInUse(oldPodSpec) { - for i := range podSpec.Containers { - if podSpec.Containers[i].Lifecycle == nil { - continue - } - if podSpec.Containers[i].Lifecycle.PreStop != nil { - podSpec.Containers[i].Lifecycle.PreStop.Sleep = nil - } - if podSpec.Containers[i].Lifecycle.PostStart != nil { - podSpec.Containers[i].Lifecycle.PostStart.Sleep = nil + dropPodLifecycleSleepAction(podSpec, oldPodSpec) +} + +func dropPodLifecycleSleepAction(podSpec, oldPodSpec *api.PodSpec) { + if utilfeature.DefaultFeatureGate.Enabled(features.PodLifecycleSleepAction) || podLifecycleSleepActionInUse(oldPodSpec) { + return + } + + adjustLifecycle := func(lifecycle *api.Lifecycle) { + if lifecycle.PreStop != nil && lifecycle.PreStop.Sleep != nil { + lifecycle.PreStop.Sleep = nil + if lifecycle.PreStop.Exec == nil && lifecycle.PreStop.HTTPGet == nil && lifecycle.PreStop.TCPSocket == nil { + lifecycle.PreStop = nil } } - for i := range podSpec.InitContainers { - if podSpec.InitContainers[i].Lifecycle == nil { - continue - } - if podSpec.InitContainers[i].Lifecycle.PreStop != nil { - podSpec.InitContainers[i].Lifecycle.PreStop.Sleep = nil - } - if podSpec.InitContainers[i].Lifecycle.PostStart != nil { - podSpec.InitContainers[i].Lifecycle.PostStart.Sleep = nil + if lifecycle.PostStart != nil && lifecycle.PostStart.Sleep != nil { + lifecycle.PostStart.Sleep = nil + if lifecycle.PostStart.Exec == nil && lifecycle.PostStart.HTTPGet == nil && lifecycle.PostStart.TCPSocket == nil { + lifecycle.PostStart = nil } } - for i := range podSpec.EphemeralContainers { - if podSpec.EphemeralContainers[i].Lifecycle == nil { - continue - } - if podSpec.EphemeralContainers[i].Lifecycle.PreStop != nil { - podSpec.EphemeralContainers[i].Lifecycle.PreStop.Sleep = nil - } - if podSpec.EphemeralContainers[i].Lifecycle.PostStart != nil { - podSpec.EphemeralContainers[i].Lifecycle.PostStart.Sleep = nil - } + } + + for i := range podSpec.Containers { + if podSpec.Containers[i].Lifecycle == nil { + continue + } + adjustLifecycle(podSpec.Containers[i].Lifecycle) + if podSpec.Containers[i].Lifecycle.PreStop == nil && podSpec.Containers[i].Lifecycle.PostStart == nil { + podSpec.Containers[i].Lifecycle = nil + } + } + + for i := range podSpec.InitContainers { + if podSpec.InitContainers[i].Lifecycle == nil { + continue + } + adjustLifecycle(podSpec.InitContainers[i].Lifecycle) + if podSpec.InitContainers[i].Lifecycle.PreStop == nil && podSpec.InitContainers[i].Lifecycle.PostStart == nil { + podSpec.InitContainers[i].Lifecycle = nil + } + } + + for i := range podSpec.EphemeralContainers { + if podSpec.EphemeralContainers[i].Lifecycle == nil { + continue + } + adjustLifecycle(podSpec.EphemeralContainers[i].Lifecycle) + if podSpec.EphemeralContainers[i].Lifecycle.PreStop == nil && podSpec.EphemeralContainers[i].Lifecycle.PostStart == nil { + podSpec.EphemeralContainers[i].Lifecycle = nil } } } diff --git a/pkg/api/pod/util_test.go b/pkg/api/pod/util_test.go index 4c92a197518..e25a84143ca 100644 --- a/pkg/api/pod/util_test.go +++ b/pkg/api/pod/util_test.go @@ -3472,3 +3472,236 @@ func TestDropClusterTrustBundleProjectedVolumes(t *testing.T) { }) } } + +func TestDropPodLifecycleSleepAction(t *testing.T) { + makeSleepHandler := func() *api.LifecycleHandler { + return &api.LifecycleHandler{ + Sleep: &api.SleepAction{Seconds: 1}, + } + } + + makeExecHandler := func() *api.LifecycleHandler { + return &api.LifecycleHandler{ + Exec: &api.ExecAction{Command: []string{"foo"}}, + } + } + + makeHTTPGetHandler := func() *api.LifecycleHandler { + return &api.LifecycleHandler{ + HTTPGet: &api.HTTPGetAction{Host: "foo"}, + } + } + + makeContainer := func(preStop, postStart *api.LifecycleHandler) api.Container { + container := api.Container{Name: "foo"} + if preStop != nil || postStart != nil { + container.Lifecycle = &api.Lifecycle{ + PostStart: postStart, + PreStop: preStop, + } + } + return container + } + + makeEphemeralContainer := func(preStop, postStart *api.LifecycleHandler) api.EphemeralContainer { + container := api.EphemeralContainer{ + EphemeralContainerCommon: api.EphemeralContainerCommon{Name: "foo"}, + } + if preStop != nil || postStart != nil { + container.Lifecycle = &api.Lifecycle{ + PostStart: postStart, + PreStop: preStop, + } + } + return container + } + + makePod := func(containers []api.Container, initContainers []api.Container, ephemeralContainers []api.EphemeralContainer) *api.PodSpec { + return &api.PodSpec{ + Containers: containers, + InitContainers: initContainers, + EphemeralContainers: ephemeralContainers, + } + } + + testCases := []struct { + gateEnabled bool + oldLifecycleHandler *api.LifecycleHandler + newLifecycleHandler *api.LifecycleHandler + expectLifecycleHandler *api.LifecycleHandler + }{ + // nil -> nil + { + gateEnabled: false, + oldLifecycleHandler: nil, + newLifecycleHandler: nil, + expectLifecycleHandler: nil, + }, + { + gateEnabled: true, + oldLifecycleHandler: nil, + newLifecycleHandler: nil, + expectLifecycleHandler: nil, + }, + // nil -> exec + { + gateEnabled: false, + oldLifecycleHandler: nil, + newLifecycleHandler: makeExecHandler(), + expectLifecycleHandler: makeExecHandler(), + }, + { + gateEnabled: true, + oldLifecycleHandler: nil, + newLifecycleHandler: makeExecHandler(), + expectLifecycleHandler: makeExecHandler(), + }, + // nil -> sleep + { + gateEnabled: false, + oldLifecycleHandler: nil, + newLifecycleHandler: makeSleepHandler(), + expectLifecycleHandler: nil, + }, + { + gateEnabled: true, + oldLifecycleHandler: nil, + newLifecycleHandler: makeSleepHandler(), + expectLifecycleHandler: makeSleepHandler(), + }, + // exec -> exec + { + gateEnabled: false, + oldLifecycleHandler: makeExecHandler(), + newLifecycleHandler: makeExecHandler(), + expectLifecycleHandler: makeExecHandler(), + }, + { + gateEnabled: true, + oldLifecycleHandler: makeExecHandler(), + newLifecycleHandler: makeExecHandler(), + expectLifecycleHandler: makeExecHandler(), + }, + // exec -> http + { + gateEnabled: false, + oldLifecycleHandler: makeExecHandler(), + newLifecycleHandler: makeHTTPGetHandler(), + expectLifecycleHandler: makeHTTPGetHandler(), + }, + { + gateEnabled: true, + oldLifecycleHandler: makeExecHandler(), + newLifecycleHandler: makeHTTPGetHandler(), + expectLifecycleHandler: makeHTTPGetHandler(), + }, + // exec -> sleep + { + gateEnabled: false, + oldLifecycleHandler: makeExecHandler(), + newLifecycleHandler: makeSleepHandler(), + expectLifecycleHandler: nil, + }, + { + gateEnabled: true, + oldLifecycleHandler: makeExecHandler(), + newLifecycleHandler: makeSleepHandler(), + expectLifecycleHandler: makeSleepHandler(), + }, + // sleep -> exec + { + gateEnabled: false, + oldLifecycleHandler: makeSleepHandler(), + newLifecycleHandler: makeExecHandler(), + expectLifecycleHandler: makeExecHandler(), + }, + { + gateEnabled: true, + oldLifecycleHandler: makeSleepHandler(), + newLifecycleHandler: makeExecHandler(), + expectLifecycleHandler: makeExecHandler(), + }, + // sleep -> sleep + { + gateEnabled: false, + oldLifecycleHandler: makeSleepHandler(), + newLifecycleHandler: makeSleepHandler(), + expectLifecycleHandler: makeSleepHandler(), + }, + { + gateEnabled: true, + oldLifecycleHandler: makeSleepHandler(), + newLifecycleHandler: makeSleepHandler(), + expectLifecycleHandler: makeSleepHandler(), + }, + } + + for i, tc := range testCases { + t.Run(fmt.Sprintf("test_%d", i), func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodLifecycleSleepAction, tc.gateEnabled)() + + // preStop + // container + { + oldPod := makePod([]api.Container{makeContainer(tc.oldLifecycleHandler.DeepCopy(), nil)}, nil, nil) + newPod := makePod([]api.Container{makeContainer(tc.newLifecycleHandler.DeepCopy(), nil)}, nil, nil) + expectPod := makePod([]api.Container{makeContainer(tc.expectLifecycleHandler.DeepCopy(), nil)}, nil, nil) + dropDisabledFields(newPod, nil, oldPod, nil) + if diff := cmp.Diff(expectPod, newPod); diff != "" { + t.Fatalf("Unexpected modification to new pod; diff (-got +want)\n%s", diff) + } + } + // InitContainer + { + oldPod := makePod(nil, []api.Container{makeContainer(tc.oldLifecycleHandler.DeepCopy(), nil)}, nil) + newPod := makePod(nil, []api.Container{makeContainer(tc.newLifecycleHandler.DeepCopy(), nil)}, nil) + expectPod := makePod(nil, []api.Container{makeContainer(tc.expectLifecycleHandler.DeepCopy(), nil)}, nil) + dropDisabledFields(newPod, nil, oldPod, nil) + if diff := cmp.Diff(expectPod, newPod); diff != "" { + t.Fatalf("Unexpected modification to new pod; diff (-got +want)\n%s", diff) + } + } + // EphemeralContainer + { + oldPod := makePod(nil, nil, []api.EphemeralContainer{makeEphemeralContainer(tc.oldLifecycleHandler.DeepCopy(), nil)}) + newPod := makePod(nil, nil, []api.EphemeralContainer{makeEphemeralContainer(tc.newLifecycleHandler.DeepCopy(), nil)}) + expectPod := makePod(nil, nil, []api.EphemeralContainer{makeEphemeralContainer(tc.expectLifecycleHandler.DeepCopy(), nil)}) + dropDisabledFields(newPod, nil, oldPod, nil) + if diff := cmp.Diff(expectPod, newPod); diff != "" { + t.Fatalf("Unexpected modification to new pod; diff (-got +want)\n%s", diff) + } + } + // postStart + // container + { + oldPod := makePod([]api.Container{makeContainer(nil, tc.oldLifecycleHandler.DeepCopy())}, nil, nil) + newPod := makePod([]api.Container{makeContainer(nil, tc.newLifecycleHandler.DeepCopy())}, nil, nil) + expectPod := makePod([]api.Container{makeContainer(nil, tc.expectLifecycleHandler.DeepCopy())}, nil, nil) + dropDisabledFields(newPod, nil, oldPod, nil) + if diff := cmp.Diff(expectPod, newPod); diff != "" { + t.Fatalf("Unexpected modification to new pod; diff (-got +want)\n%s", diff) + } + } + // InitContainer + { + oldPod := makePod(nil, []api.Container{makeContainer(nil, tc.oldLifecycleHandler.DeepCopy())}, nil) + newPod := makePod(nil, []api.Container{makeContainer(nil, tc.newLifecycleHandler.DeepCopy())}, nil) + expectPod := makePod(nil, []api.Container{makeContainer(nil, tc.expectLifecycleHandler.DeepCopy())}, nil) + dropDisabledFields(newPod, nil, oldPod, nil) + if diff := cmp.Diff(expectPod, newPod); diff != "" { + t.Fatalf("Unexpected modification to new pod; diff (-got +want)\n%s", diff) + } + } + // EphemeralContainer + { + oldPod := makePod(nil, nil, []api.EphemeralContainer{makeEphemeralContainer(nil, tc.oldLifecycleHandler.DeepCopy())}) + newPod := makePod(nil, nil, []api.EphemeralContainer{makeEphemeralContainer(nil, tc.newLifecycleHandler.DeepCopy())}) + expectPod := makePod(nil, nil, []api.EphemeralContainer{makeEphemeralContainer(nil, tc.expectLifecycleHandler.DeepCopy())}) + dropDisabledFields(newPod, nil, oldPod, nil) + if diff := cmp.Diff(expectPod, newPod); diff != "" { + t.Fatalf("Unexpected modification to new pod; diff (-got +want)\n%s", diff) + } + } + }) + } +} diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 26f215a56da..8040a7e9010 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -603,6 +603,7 @@ const ( // owner: @AxeZhan // kep: http://kep.k8s.io/3960 // alpha: v1.29 + // beta: v1.30 // // Enables SleepAction in container lifecycle hooks PodLifecycleSleepAction featuregate.Feature = "PodLifecycleSleepAction" @@ -1070,7 +1071,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS PodHostIPs: {Default: true, PreRelease: featuregate.Beta}, - PodLifecycleSleepAction: {Default: false, PreRelease: featuregate.Alpha}, + PodLifecycleSleepAction: {Default: true, PreRelease: featuregate.Beta}, PodSchedulingReadiness: {Default: true, PreRelease: featuregate.Beta}, diff --git a/pkg/kubelet/lifecycle/handlers.go b/pkg/kubelet/lifecycle/handlers.go index e15669965ea..14ca376eab7 100644 --- a/pkg/kubelet/lifecycle/handlers.go +++ b/pkg/kubelet/lifecycle/handlers.go @@ -133,6 +133,7 @@ func (hr *handlerRunner) runSleepHandler(ctx context.Context, seconds int64) err select { case <-ctx.Done(): // unexpected termination + metrics.LifecycleHandlerSleepTerminated.Inc() return fmt.Errorf("container terminated before sleep hook finished") case <-c: return nil diff --git a/pkg/kubelet/metrics/metrics.go b/pkg/kubelet/metrics/metrics.go index 05e7b8895f4..145f8a9a4a3 100644 --- a/pkg/kubelet/metrics/metrics.go +++ b/pkg/kubelet/metrics/metrics.go @@ -859,6 +859,15 @@ var ( }, []string{"image_size_in_bytes"}, ) + + LifecycleHandlerSleepTerminated = metrics.NewCounter( + &metrics.CounterOpts{ + Subsystem: KubeletSubsystem, + Name: "sleep_action_terminated_early_total", + Help: "The number of times lifecycle sleep handler got terminated before it finishes", + StabilityLevel: metrics.ALPHA, + }, + ) ) var registerMetrics sync.Once @@ -942,6 +951,7 @@ func Register(collectors ...metrics.StableCollector) { } legacyregistry.MustRegister(LifecycleHandlerHTTPFallbacks) + legacyregistry.MustRegister(LifecycleHandlerSleepTerminated) }) } diff --git a/pkg/registry/core/pod/strategy_test.go b/pkg/registry/core/pod/strategy_test.go index 6317af86da6..da48f25fbf7 100644 --- a/pkg/registry/core/pod/strategy_test.go +++ b/pkg/registry/core/pod/strategy_test.go @@ -2031,3 +2031,99 @@ func Test_mutatePodAffinity(t *testing.T) { }) } } + +func TestPodLifecycleSleepActionEnablement(t *testing.T) { + getLifecycle := func(pod *api.Pod) *api.Lifecycle { + return pod.Spec.Containers[0].Lifecycle + } + + defaultTerminationGracePeriodSeconds := int64(30) + + podWithHandler := func() *api.Pod { + return &api.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "foo", + ResourceVersion: "1", + }, + Spec: api.PodSpec{ + RestartPolicy: api.RestartPolicyAlways, + DNSPolicy: api.DNSDefault, + Containers: []api.Container{ + { + Name: "container", + Image: "image", + ImagePullPolicy: "IfNotPresent", + TerminationMessagePolicy: "File", + Lifecycle: &api.Lifecycle{ + PreStop: &api.LifecycleHandler{ + Sleep: &api.SleepAction{Seconds: 1}, + }, + }, + }, + }, + TerminationGracePeriodSeconds: &defaultTerminationGracePeriodSeconds, + }, + } + } + + podWithoutHandler := func() *api.Pod { + return &api.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "foo", + ResourceVersion: "1", + }, + Spec: api.PodSpec{ + RestartPolicy: api.RestartPolicyAlways, + DNSPolicy: api.DNSDefault, + Containers: []api.Container{ + { + Name: "container", + Image: "image", + ImagePullPolicy: "IfNotPresent", + TerminationMessagePolicy: "File", + }, + }, + TerminationGracePeriodSeconds: &defaultTerminationGracePeriodSeconds, + }, + } + } + + testCases := []struct { + description string + gateEnabled bool + newPod *api.Pod + wantPod *api.Pod + }{ + { + description: "gate enabled, creating pods with sleep action", + gateEnabled: true, + newPod: podWithHandler(), + wantPod: podWithHandler(), + }, + { + description: "gate disabled, creating pods with sleep action", + gateEnabled: false, + newPod: podWithHandler(), + wantPod: podWithoutHandler(), + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodLifecycleSleepAction, tc.gateEnabled)() + + newPod := tc.newPod + + Strategy.PrepareForCreate(genericapirequest.NewContext(), newPod) + if errs := Strategy.Validate(genericapirequest.NewContext(), newPod); len(errs) != 0 { + t.Errorf("Unexpected error: %v", errs.ToAggregate()) + } + + if diff := cmp.Diff(getLifecycle(newPod), getLifecycle(tc.wantPod)); diff != "" { + t.Fatalf("Unexpected modification to life cycle; diff (-got +want)\n%s", diff) + } + }) + } +} diff --git a/test/e2e/common/node/lifecycle_hook.go b/test/e2e/common/node/lifecycle_hook.go index e633b79772c..b89b59de2ed 100644 --- a/test/e2e/common/node/lifecycle_hook.go +++ b/test/e2e/common/node/lifecycle_hook.go @@ -552,6 +552,10 @@ var _ = SIGDescribe(feature.PodLifecycleSleepAction, func() { f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline var podClient *e2epod.PodClient + validDuration := func(duration time.Duration, low, high int64) bool { + return duration >= time.Second*time.Duration(low) && duration <= time.Second*time.Duration(high) + } + ginkgo.Context("when create a pod with lifecycle hook using sleep action", func() { ginkgo.BeforeEach(func(ctx context.Context) { podClient = e2epod.NewPodClient(f) @@ -569,16 +573,18 @@ var _ = SIGDescribe(feature.PodLifecycleSleepAction, func() { start := time.Now() podClient.DeleteSync(ctx, podWithHook.Name, metav1.DeleteOptions{}, e2epod.DefaultPodDeletionTimeout) cost := time.Since(start) - // verify that deletion was delayed by sleep seconds - if cost < time.Second*5 || cost > time.Second*10 { - framework.Failf("unexpected delay duration before killing the pod") + // cost should be + // longer than 5 seconds (pod should sleep for 5 seconds) + // shorter than gracePeriodSeconds (default 30 seconds here) + if !validDuration(cost, 5, 30) { + framework.Failf("unexpected delay duration before killing the pod, cost = %v", cost) } }) ginkgo.It("reduce GracePeriodSeconds during runtime", func(ctx context.Context) { lifecycle := &v1.Lifecycle{ PreStop: &v1.LifecycleHandler{ - Sleep: &v1.SleepAction{Seconds: 10}, + Sleep: &v1.SleepAction{Seconds: 15}, }, } podWithHook := getPodWithHook("pod-with-prestop-sleep-hook", imageutils.GetPauseImageName(), lifecycle) @@ -588,10 +594,38 @@ var _ = SIGDescribe(feature.PodLifecycleSleepAction, func() { start := time.Now() podClient.DeleteSync(ctx, podWithHook.Name, *metav1.NewDeleteOptions(2), e2epod.DefaultPodDeletionTimeout) cost := time.Since(start) - // verify that deletion was delayed by sleep seconds - if cost <= time.Second || cost >= time.Second*5 { - framework.Failf("unexpected delay duration before killing the pod") + // cost should be + // longer than 2 seconds (we change gracePeriodSeconds to 2 seconds here, and it's less than sleep action) + // shorter than sleep action (to make sure it doesn't take effect) + if !validDuration(cost, 2, 15) { + framework.Failf("unexpected delay duration before killing the pod, cost = %v", cost) } }) + + ginkgo.It("ignore terminated container", func(ctx context.Context) { + lifecycle := &v1.Lifecycle{ + PreStop: &v1.LifecycleHandler{ + Sleep: &v1.SleepAction{Seconds: 20}, + }, + } + name := "pod-with-prestop-sleep-hook" + podWithHook := getPodWithHook(name, imageutils.GetE2EImage(imageutils.BusyBox), lifecycle) + podWithHook.Spec.Containers[0].Command = []string{"/bin/sh"} + podWithHook.Spec.Containers[0].Args = []string{"-c", "exit 0"} + podWithHook.Spec.RestartPolicy = v1.RestartPolicyNever + ginkgo.By("create the pod with lifecycle hook using sleep action") + p := podClient.Create(ctx, podWithHook) + framework.ExpectNoError(e2epod.WaitForContainerTerminated(ctx, f.ClientSet, f.Namespace.Name, p.Name, name, 3*time.Minute)) + ginkgo.By("delete the pod with lifecycle hook using sleep action") + start := time.Now() + podClient.DeleteSync(ctx, podWithHook.Name, metav1.DeleteOptions{}, e2epod.DefaultPodDeletionTimeout) + cost := time.Since(start) + // cost should be + // shorter than sleep action (container is terminated and sleep action should be ignored) + if !validDuration(cost, 0, 15) { + framework.Failf("unexpected delay duration before killing the pod, cost = %v", cost) + } + }) + }) }) diff --git a/test/e2e/framework/pod/wait.go b/test/e2e/framework/pod/wait.go index b760edc43a9..44bf7716bfd 100644 --- a/test/e2e/framework/pod/wait.go +++ b/test/e2e/framework/pod/wait.go @@ -774,3 +774,18 @@ func WaitForContainerRunning(ctx context.Context, c clientset.Interface, namespa return false, nil }) } + +// WaitForContainerTerminated waits for the given Pod container to have a state of terminated +func WaitForContainerTerminated(ctx context.Context, c clientset.Interface, namespace, podName, containerName string, timeout time.Duration) error { + conditionDesc := fmt.Sprintf("container %s terminated", containerName) + return WaitForPodCondition(ctx, c, namespace, podName, conditionDesc, timeout, func(pod *v1.Pod) (bool, error) { + for _, statuses := range [][]v1.ContainerStatus{pod.Status.ContainerStatuses, pod.Status.InitContainerStatuses, pod.Status.EphemeralContainerStatuses} { + for _, cs := range statuses { + if cs.Name == containerName { + return cs.State.Terminated != nil, nil + } + } + } + return false, nil + }) +}