graduate PodLifecycleSleepAction to beta

pkg/api/pod/util.go

@@ -594,39 +594,56 @@ func dropDisabledFields(
 		// For other types of containers, validateContainers will handle them.
 	}
 
-	if !utilfeature.DefaultFeatureGate.Enabled(features.PodLifecycleSleepAction) && !podLifecycleSleepActionInUse(oldPodSpec) {
-		for i := range podSpec.Containers {
-			if podSpec.Containers[i].Lifecycle == nil {
-				continue
-			}
-			if podSpec.Containers[i].Lifecycle.PreStop != nil {
-				podSpec.Containers[i].Lifecycle.PreStop.Sleep = nil
-			}
-			if podSpec.Containers[i].Lifecycle.PostStart != nil {
-				podSpec.Containers[i].Lifecycle.PostStart.Sleep = nil
+	dropPodLifecycleSleepAction(podSpec, oldPodSpec)
+}
+
+func dropPodLifecycleSleepAction(podSpec, oldPodSpec *api.PodSpec) {
+	if utilfeature.DefaultFeatureGate.Enabled(features.PodLifecycleSleepAction) || podLifecycleSleepActionInUse(oldPodSpec) {
+		return
+	}
+
+	adjustLifecycle := func(lifecycle *api.Lifecycle) {
+		if lifecycle.PreStop != nil && lifecycle.PreStop.Sleep != nil {
+			lifecycle.PreStop.Sleep = nil
+			if lifecycle.PreStop.Exec == nil && lifecycle.PreStop.HTTPGet == nil && lifecycle.PreStop.TCPSocket == nil {
+				lifecycle.PreStop = nil
 			}
 		}
-		for i := range podSpec.InitContainers {
-			if podSpec.InitContainers[i].Lifecycle == nil {
-				continue
-			}
-			if podSpec.InitContainers[i].Lifecycle.PreStop != nil {
-				podSpec.InitContainers[i].Lifecycle.PreStop.Sleep = nil
-			}
-			if podSpec.InitContainers[i].Lifecycle.PostStart != nil {
-				podSpec.InitContainers[i].Lifecycle.PostStart.Sleep = nil
+		if lifecycle.PostStart != nil && lifecycle.PostStart.Sleep != nil {
+			lifecycle.PostStart.Sleep = nil
+			if lifecycle.PostStart.Exec == nil && lifecycle.PostStart.HTTPGet == nil && lifecycle.PostStart.TCPSocket == nil {
+				lifecycle.PostStart = nil
 			}
 		}
-		for i := range podSpec.EphemeralContainers {
-			if podSpec.EphemeralContainers[i].Lifecycle == nil {
-				continue
-			}
-			if podSpec.EphemeralContainers[i].Lifecycle.PreStop != nil {
-				podSpec.EphemeralContainers[i].Lifecycle.PreStop.Sleep = nil
-			}
-			if podSpec.EphemeralContainers[i].Lifecycle.PostStart != nil {
-				podSpec.EphemeralContainers[i].Lifecycle.PostStart.Sleep = nil
-			}
+	}
+
+	for i := range podSpec.Containers {
+		if podSpec.Containers[i].Lifecycle == nil {
+			continue
+		}
+		adjustLifecycle(podSpec.Containers[i].Lifecycle)
+		if podSpec.Containers[i].Lifecycle.PreStop == nil && podSpec.Containers[i].Lifecycle.PostStart == nil {
+			podSpec.Containers[i].Lifecycle = nil
+		}
+	}
+
+	for i := range podSpec.InitContainers {
+		if podSpec.InitContainers[i].Lifecycle == nil {
+			continue
+		}
+		adjustLifecycle(podSpec.InitContainers[i].Lifecycle)
+		if podSpec.InitContainers[i].Lifecycle.PreStop == nil && podSpec.InitContainers[i].Lifecycle.PostStart == nil {
+			podSpec.InitContainers[i].Lifecycle = nil
+		}
+	}
+
+	for i := range podSpec.EphemeralContainers {
+		if podSpec.EphemeralContainers[i].Lifecycle == nil {
+			continue
+		}
+		adjustLifecycle(podSpec.EphemeralContainers[i].Lifecycle)
+		if podSpec.EphemeralContainers[i].Lifecycle.PreStop == nil && podSpec.EphemeralContainers[i].Lifecycle.PostStart == nil {
+			podSpec.EphemeralContainers[i].Lifecycle = nil
 		}
 	}
 }

pkg/api/pod/util_test.go

@@ -3472,3 +3472,236 @@ func TestDropClusterTrustBundleProjectedVolumes(t *testing.T) {
 		})
 	}
 }
+
+func TestDropPodLifecycleSleepAction(t *testing.T) {
+	makeSleepHandler := func() *api.LifecycleHandler {
+		return &api.LifecycleHandler{
+			Sleep: &api.SleepAction{Seconds: 1},
+		}
+	}
+
+	makeExecHandler := func() *api.LifecycleHandler {
+		return &api.LifecycleHandler{
+			Exec: &api.ExecAction{Command: []string{"foo"}},
+		}
+	}
+
+	makeHTTPGetHandler := func() *api.LifecycleHandler {
+		return &api.LifecycleHandler{
+			HTTPGet: &api.HTTPGetAction{Host: "foo"},
+		}
+	}
+
+	makeContainer := func(preStop, postStart *api.LifecycleHandler) api.Container {
+		container := api.Container{Name: "foo"}
+		if preStop != nil || postStart != nil {
+			container.Lifecycle = &api.Lifecycle{
+				PostStart: postStart,
+				PreStop:   preStop,
+			}
+		}
+		return container
+	}
+
+	makeEphemeralContainer := func(preStop, postStart *api.LifecycleHandler) api.EphemeralContainer {
+		container := api.EphemeralContainer{
+			EphemeralContainerCommon: api.EphemeralContainerCommon{Name: "foo"},
+		}
+		if preStop != nil || postStart != nil {
+			container.Lifecycle = &api.Lifecycle{
+				PostStart: postStart,
+				PreStop:   preStop,
+			}
+		}
+		return container
+	}
+
+	makePod := func(containers []api.Container, initContainers []api.Container, ephemeralContainers []api.EphemeralContainer) *api.PodSpec {
+		return &api.PodSpec{
+			Containers:          containers,
+			InitContainers:      initContainers,
+			EphemeralContainers: ephemeralContainers,
+		}
+	}
+
+	testCases := []struct {
+		gateEnabled            bool
+		oldLifecycleHandler    *api.LifecycleHandler
+		newLifecycleHandler    *api.LifecycleHandler
+		expectLifecycleHandler *api.LifecycleHandler
+	}{
+		// nil -> nil
+		{
+			gateEnabled:            false,
+			oldLifecycleHandler:    nil,
+			newLifecycleHandler:    nil,
+			expectLifecycleHandler: nil,
+		},
+		{
+			gateEnabled:            true,
+			oldLifecycleHandler:    nil,
+			newLifecycleHandler:    nil,
+			expectLifecycleHandler: nil,
+		},
+		// nil -> exec
+		{
+			gateEnabled:            false,
+			oldLifecycleHandler:    nil,
+			newLifecycleHandler:    makeExecHandler(),
+			expectLifecycleHandler: makeExecHandler(),
+		},
+		{
+			gateEnabled:            true,
+			oldLifecycleHandler:    nil,
+			newLifecycleHandler:    makeExecHandler(),
+			expectLifecycleHandler: makeExecHandler(),
+		},
+		// nil -> sleep
+		{
+			gateEnabled:            false,
+			oldLifecycleHandler:    nil,
+			newLifecycleHandler:    makeSleepHandler(),
+			expectLifecycleHandler: nil,
+		},
+		{
+			gateEnabled:            true,
+			oldLifecycleHandler:    nil,
+			newLifecycleHandler:    makeSleepHandler(),
+			expectLifecycleHandler: makeSleepHandler(),
+		},
+		// exec -> exec
+		{
+			gateEnabled:            false,
+			oldLifecycleHandler:    makeExecHandler(),
+			newLifecycleHandler:    makeExecHandler(),
+			expectLifecycleHandler: makeExecHandler(),
+		},
+		{
+			gateEnabled:            true,
+			oldLifecycleHandler:    makeExecHandler(),
+			newLifecycleHandler:    makeExecHandler(),
+			expectLifecycleHandler: makeExecHandler(),
+		},
+		// exec -> http
+		{
+			gateEnabled:            false,
+			oldLifecycleHandler:    makeExecHandler(),
+			newLifecycleHandler:    makeHTTPGetHandler(),
+			expectLifecycleHandler: makeHTTPGetHandler(),
+		},
+		{
+			gateEnabled:            true,
+			oldLifecycleHandler:    makeExecHandler(),
+			newLifecycleHandler:    makeHTTPGetHandler(),
+			expectLifecycleHandler: makeHTTPGetHandler(),
+		},
+		// exec -> sleep
+		{
+			gateEnabled:            false,
+			oldLifecycleHandler:    makeExecHandler(),
+			newLifecycleHandler:    makeSleepHandler(),
+			expectLifecycleHandler: nil,
+		},
+		{
+			gateEnabled:            true,
+			oldLifecycleHandler:    makeExecHandler(),
+			newLifecycleHandler:    makeSleepHandler(),
+			expectLifecycleHandler: makeSleepHandler(),
+		},
+		// sleep -> exec
+		{
+			gateEnabled:            false,
+			oldLifecycleHandler:    makeSleepHandler(),
+			newLifecycleHandler:    makeExecHandler(),
+			expectLifecycleHandler: makeExecHandler(),
+		},
+		{
+			gateEnabled:            true,
+			oldLifecycleHandler:    makeSleepHandler(),
+			newLifecycleHandler:    makeExecHandler(),
+			expectLifecycleHandler: makeExecHandler(),
+		},
+		// sleep -> sleep
+		{
+			gateEnabled:            false,
+			oldLifecycleHandler:    makeSleepHandler(),
+			newLifecycleHandler:    makeSleepHandler(),
+			expectLifecycleHandler: makeSleepHandler(),
+		},
+		{
+			gateEnabled:            true,
+			oldLifecycleHandler:    makeSleepHandler(),
+			newLifecycleHandler:    makeSleepHandler(),
+			expectLifecycleHandler: makeSleepHandler(),
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("test_%d", i), func(t *testing.T) {
+			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodLifecycleSleepAction, tc.gateEnabled)()
+
+			// preStop
+			// container
+			{
+				oldPod := makePod([]api.Container{makeContainer(tc.oldLifecycleHandler.DeepCopy(), nil)}, nil, nil)
+				newPod := makePod([]api.Container{makeContainer(tc.newLifecycleHandler.DeepCopy(), nil)}, nil, nil)
+				expectPod := makePod([]api.Container{makeContainer(tc.expectLifecycleHandler.DeepCopy(), nil)}, nil, nil)
+				dropDisabledFields(newPod, nil, oldPod, nil)
+				if diff := cmp.Diff(expectPod, newPod); diff != "" {
+					t.Fatalf("Unexpected modification to new pod; diff (-got +want)\n%s", diff)
+				}
+			}
+			// InitContainer
+			{
+				oldPod := makePod(nil, []api.Container{makeContainer(tc.oldLifecycleHandler.DeepCopy(), nil)}, nil)
+				newPod := makePod(nil, []api.Container{makeContainer(tc.newLifecycleHandler.DeepCopy(), nil)}, nil)
+				expectPod := makePod(nil, []api.Container{makeContainer(tc.expectLifecycleHandler.DeepCopy(), nil)}, nil)
+				dropDisabledFields(newPod, nil, oldPod, nil)
+				if diff := cmp.Diff(expectPod, newPod); diff != "" {
+					t.Fatalf("Unexpected modification to new pod; diff (-got +want)\n%s", diff)
+				}
+			}
+			// EphemeralContainer
+			{
+				oldPod := makePod(nil, nil, []api.EphemeralContainer{makeEphemeralContainer(tc.oldLifecycleHandler.DeepCopy(), nil)})
+				newPod := makePod(nil, nil, []api.EphemeralContainer{makeEphemeralContainer(tc.newLifecycleHandler.DeepCopy(), nil)})
+				expectPod := makePod(nil, nil, []api.EphemeralContainer{makeEphemeralContainer(tc.expectLifecycleHandler.DeepCopy(), nil)})
+				dropDisabledFields(newPod, nil, oldPod, nil)
+				if diff := cmp.Diff(expectPod, newPod); diff != "" {
+					t.Fatalf("Unexpected modification to new pod; diff (-got +want)\n%s", diff)
+				}
+			}
+			// postStart
+			// container
+			{
+				oldPod := makePod([]api.Container{makeContainer(nil, tc.oldLifecycleHandler.DeepCopy())}, nil, nil)
+				newPod := makePod([]api.Container{makeContainer(nil, tc.newLifecycleHandler.DeepCopy())}, nil, nil)
+				expectPod := makePod([]api.Container{makeContainer(nil, tc.expectLifecycleHandler.DeepCopy())}, nil, nil)
+				dropDisabledFields(newPod, nil, oldPod, nil)
+				if diff := cmp.Diff(expectPod, newPod); diff != "" {
+					t.Fatalf("Unexpected modification to new pod; diff (-got +want)\n%s", diff)
+				}
+			}
+			// InitContainer
+			{
+				oldPod := makePod(nil, []api.Container{makeContainer(nil, tc.oldLifecycleHandler.DeepCopy())}, nil)
+				newPod := makePod(nil, []api.Container{makeContainer(nil, tc.newLifecycleHandler.DeepCopy())}, nil)
+				expectPod := makePod(nil, []api.Container{makeContainer(nil, tc.expectLifecycleHandler.DeepCopy())}, nil)
+				dropDisabledFields(newPod, nil, oldPod, nil)
+				if diff := cmp.Diff(expectPod, newPod); diff != "" {
+					t.Fatalf("Unexpected modification to new pod; diff (-got +want)\n%s", diff)
+				}
+			}
+			// EphemeralContainer
+			{
+				oldPod := makePod(nil, nil, []api.EphemeralContainer{makeEphemeralContainer(nil, tc.oldLifecycleHandler.DeepCopy())})
+				newPod := makePod(nil, nil, []api.EphemeralContainer{makeEphemeralContainer(nil, tc.newLifecycleHandler.DeepCopy())})
+				expectPod := makePod(nil, nil, []api.EphemeralContainer{makeEphemeralContainer(nil, tc.expectLifecycleHandler.DeepCopy())})
+				dropDisabledFields(newPod, nil, oldPod, nil)
+				if diff := cmp.Diff(expectPod, newPod); diff != "" {
+					t.Fatalf("Unexpected modification to new pod; diff (-got +want)\n%s", diff)
+				}
+			}
+		})
+	}
+}

pkg/features/kube_features.go

@@ -603,6 +603,7 @@ const (
 	// owner: @AxeZhan
 	// kep: http://kep.k8s.io/3960
 	// alpha: v1.29
+	// beta: v1.30
 	//
 	// Enables SleepAction in container lifecycle hooks
 	PodLifecycleSleepAction featuregate.Feature = "PodLifecycleSleepAction"
@@ -1070,7 +1071,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
 
 	PodHostIPs: {Default: true, PreRelease: featuregate.Beta},
 
-	PodLifecycleSleepAction: {Default: false, PreRelease: featuregate.Alpha},
+	PodLifecycleSleepAction: {Default: true, PreRelease: featuregate.Beta},
 
 	PodSchedulingReadiness: {Default: true, PreRelease: featuregate.Beta},
 

pkg/kubelet/lifecycle/handlers.go

@@ -133,6 +133,7 @@ func (hr *handlerRunner) runSleepHandler(ctx context.Context, seconds int64) err
 	select {
 	case <-ctx.Done():
 		// unexpected termination
+		metrics.LifecycleHandlerSleepTerminated.Inc()
 		return fmt.Errorf("container terminated before sleep hook finished")
 	case <-c:
 		return nil

pkg/kubelet/metrics/metrics.go

@@ -859,6 +859,15 @@ var (
 		},
 		[]string{"image_size_in_bytes"},
 	)
+
+	LifecycleHandlerSleepTerminated = metrics.NewCounter(
+		&metrics.CounterOpts{
+			Subsystem:      KubeletSubsystem,
+			Name:           "sleep_action_terminated_early_total",
+			Help:           "The number of times lifecycle sleep handler got terminated before it finishes",
+			StabilityLevel: metrics.ALPHA,
+		},
+	)
 )
 
 var registerMetrics sync.Once
@@ -942,6 +951,7 @@ func Register(collectors ...metrics.StableCollector) {
 		}
 
 		legacyregistry.MustRegister(LifecycleHandlerHTTPFallbacks)
+		legacyregistry.MustRegister(LifecycleHandlerSleepTerminated)
 	})
 }
 

pkg/registry/core/pod/strategy_test.go

@@ -2031,3 +2031,99 @@ func Test_mutatePodAffinity(t *testing.T) {
 		})
 	}
 }
+
+func TestPodLifecycleSleepActionEnablement(t *testing.T) {
+	getLifecycle := func(pod *api.Pod) *api.Lifecycle {
+		return pod.Spec.Containers[0].Lifecycle
+	}
+
+	defaultTerminationGracePeriodSeconds := int64(30)
+
+	podWithHandler := func() *api.Pod {
+		return &api.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace:       "default",
+				Name:            "foo",
+				ResourceVersion: "1",
+			},
+			Spec: api.PodSpec{
+				RestartPolicy: api.RestartPolicyAlways,
+				DNSPolicy:     api.DNSDefault,
+				Containers: []api.Container{
+					{
+						Name:                     "container",
+						Image:                    "image",
+						ImagePullPolicy:          "IfNotPresent",
+						TerminationMessagePolicy: "File",
+						Lifecycle: &api.Lifecycle{
+							PreStop: &api.LifecycleHandler{
+								Sleep: &api.SleepAction{Seconds: 1},
+							},
+						},
+					},
+				},
+				TerminationGracePeriodSeconds: &defaultTerminationGracePeriodSeconds,
+			},
+		}
+	}
+
+	podWithoutHandler := func() *api.Pod {
+		return &api.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace:       "default",
+				Name:            "foo",
+				ResourceVersion: "1",
+			},
+			Spec: api.PodSpec{
+				RestartPolicy: api.RestartPolicyAlways,
+				DNSPolicy:     api.DNSDefault,
+				Containers: []api.Container{
+					{
+						Name:                     "container",
+						Image:                    "image",
+						ImagePullPolicy:          "IfNotPresent",
+						TerminationMessagePolicy: "File",
+					},
+				},
+				TerminationGracePeriodSeconds: &defaultTerminationGracePeriodSeconds,
+			},
+		}
+	}
+
+	testCases := []struct {
+		description string
+		gateEnabled bool
+		newPod      *api.Pod
+		wantPod     *api.Pod
+	}{
+		{
+			description: "gate enabled, creating pods with sleep action",
+			gateEnabled: true,
+			newPod:      podWithHandler(),
+			wantPod:     podWithHandler(),
+		},
+		{
+			description: "gate disabled, creating pods with sleep action",
+			gateEnabled: false,
+			newPod:      podWithHandler(),
+			wantPod:     podWithoutHandler(),
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodLifecycleSleepAction, tc.gateEnabled)()
+
+			newPod := tc.newPod
+
+			Strategy.PrepareForCreate(genericapirequest.NewContext(), newPod)
+			if errs := Strategy.Validate(genericapirequest.NewContext(), newPod); len(errs) != 0 {
+				t.Errorf("Unexpected error: %v", errs.ToAggregate())
+			}
+
+			if diff := cmp.Diff(getLifecycle(newPod), getLifecycle(tc.wantPod)); diff != "" {
+				t.Fatalf("Unexpected modification to life cycle; diff (-got +want)\n%s", diff)
+			}
+		})
+	}
+}

test/e2e/common/node/lifecycle_hook.go

@@ -552,6 +552,10 @@ var _ = SIGDescribe(feature.PodLifecycleSleepAction, func() {
 	f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline
 	var podClient *e2epod.PodClient
 
+	validDuration := func(duration time.Duration, low, high int64) bool {
+		return duration >= time.Second*time.Duration(low) && duration <= time.Second*time.Duration(high)
+	}
+
 	ginkgo.Context("when create a pod with lifecycle hook using sleep action", func() {
 		ginkgo.BeforeEach(func(ctx context.Context) {
 			podClient = e2epod.NewPodClient(f)
@@ -569,16 +573,18 @@ var _ = SIGDescribe(feature.PodLifecycleSleepAction, func() {
 			start := time.Now()
 			podClient.DeleteSync(ctx, podWithHook.Name, metav1.DeleteOptions{}, e2epod.DefaultPodDeletionTimeout)
 			cost := time.Since(start)
-			// verify that deletion was delayed by sleep seconds
-			if cost < time.Second*5 || cost > time.Second*10 {
-				framework.Failf("unexpected delay duration before killing the pod")
+			// cost should be
+			// longer than 5 seconds (pod should sleep for 5 seconds)
+			// shorter than gracePeriodSeconds (default 30 seconds here)
+			if !validDuration(cost, 5, 30) {
+				framework.Failf("unexpected delay duration before killing the pod, cost = %v", cost)
 			}
 		})
 
 		ginkgo.It("reduce GracePeriodSeconds during runtime", func(ctx context.Context) {
 			lifecycle := &v1.Lifecycle{
 				PreStop: &v1.LifecycleHandler{
-					Sleep: &v1.SleepAction{Seconds: 10},
+					Sleep: &v1.SleepAction{Seconds: 15},
 				},
 			}
 			podWithHook := getPodWithHook("pod-with-prestop-sleep-hook", imageutils.GetPauseImageName(), lifecycle)
@@ -588,10 +594,38 @@ var _ = SIGDescribe(feature.PodLifecycleSleepAction, func() {
 			start := time.Now()
 			podClient.DeleteSync(ctx, podWithHook.Name, *metav1.NewDeleteOptions(2), e2epod.DefaultPodDeletionTimeout)
 			cost := time.Since(start)
-			// verify that deletion was delayed by sleep seconds
-			if cost <= time.Second || cost >= time.Second*5 {
-				framework.Failf("unexpected delay duration before killing the pod")
+			// cost should be
+			// longer than 2 seconds (we change gracePeriodSeconds to 2 seconds here, and it's less than sleep action)
+			// shorter than sleep action (to make sure it doesn't take effect)
+			if !validDuration(cost, 2, 15) {
+				framework.Failf("unexpected delay duration before killing the pod, cost = %v", cost)
 			}
 		})
+
+		ginkgo.It("ignore terminated container", func(ctx context.Context) {
+			lifecycle := &v1.Lifecycle{
+				PreStop: &v1.LifecycleHandler{
+					Sleep: &v1.SleepAction{Seconds: 20},
+				},
+			}
+			name := "pod-with-prestop-sleep-hook"
+			podWithHook := getPodWithHook(name, imageutils.GetE2EImage(imageutils.BusyBox), lifecycle)
+			podWithHook.Spec.Containers[0].Command = []string{"/bin/sh"}
+			podWithHook.Spec.Containers[0].Args = []string{"-c", "exit 0"}
+			podWithHook.Spec.RestartPolicy = v1.RestartPolicyNever
+			ginkgo.By("create the pod with lifecycle hook using sleep action")
+			p := podClient.Create(ctx, podWithHook)
+			framework.ExpectNoError(e2epod.WaitForContainerTerminated(ctx, f.ClientSet, f.Namespace.Name, p.Name, name, 3*time.Minute))
+			ginkgo.By("delete the pod with lifecycle hook using sleep action")
+			start := time.Now()
+			podClient.DeleteSync(ctx, podWithHook.Name, metav1.DeleteOptions{}, e2epod.DefaultPodDeletionTimeout)
+			cost := time.Since(start)
+			// cost should be
+			// shorter than sleep action (container is terminated and sleep action should be ignored)
+			if !validDuration(cost, 0, 15) {
+				framework.Failf("unexpected delay duration before killing the pod, cost = %v", cost)
+			}
+		})
+
 	})
 })

test/e2e/framework/pod/wait.go

@@ -774,3 +774,18 @@ func WaitForContainerRunning(ctx context.Context, c clientset.Interface, namespa
 		return false, nil
 	})
 }
+
+// WaitForContainerTerminated waits for the given Pod container to have a state of terminated
+func WaitForContainerTerminated(ctx context.Context, c clientset.Interface, namespace, podName, containerName string, timeout time.Duration) error {
+	conditionDesc := fmt.Sprintf("container %s terminated", containerName)
+	return WaitForPodCondition(ctx, c, namespace, podName, conditionDesc, timeout, func(pod *v1.Pod) (bool, error) {
+		for _, statuses := range [][]v1.ContainerStatus{pod.Status.ContainerStatuses, pod.Status.InitContainerStatuses, pod.Status.EphemeralContainerStatuses} {
+			for _, cs := range statuses {
+				if cs.Name == containerName {
+					return cs.State.Terminated != nil, nil
+				}
+			}
+		}
+		return false, nil
+	})
+}