Merge pull request #94160 from matthyx/startupprobe-ga

Remove StartupProbe featuregate and related logic

api/openapi-spec/swagger.json

@@ -5626,7 +5626,7 @@
         },
         "startupProbe": {
           "$ref": "#/definitions/io.k8s.api.core.v1.Probe",
-          "description": "StartupProbe indicates that the Pod has successfully initialized. If specified, no other probes are executed until this completes successfully. If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. This cannot be updated. This is a beta feature enabled by the StartupProbe feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes"
+          "description": "StartupProbe indicates that the Pod has successfully initialized. If specified, no other probes are executed until this completes successfully. If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. This cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes"
         },
         "stdin": {
           "description": "Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false.",

pkg/api/pod/util.go

@@ -406,14 +406,6 @@ func dropDisabledFields(
 		})
 	}
 
-	if !utilfeature.DefaultFeatureGate.Enabled(features.StartupProbe) && !startupProbeInUse(oldPodSpec) {
-		// drop startupProbe from all containers if the feature is disabled
-		VisitContainers(podSpec, AllContainers, func(c *api.Container, containerType ContainerType) bool {
-			c.StartupProbe = nil
-			return true
-		})
-	}
-
 	dropDisabledRunAsGroupField(podSpec, oldPodSpec)
 
 	dropDisabledFSGroupFields(podSpec, oldPodSpec)
@@ -696,24 +688,6 @@ func subpathExprInUse(podSpec *api.PodSpec) bool {
 	return inUse
 }
 
-// startupProbeInUse returns true if the pod spec is non-nil and has a container that has a startupProbe defined
-func startupProbeInUse(podSpec *api.PodSpec) bool {
-	if podSpec == nil {
-		return false
-	}
-
-	var inUse bool
-	VisitContainers(podSpec, AllContainers, func(c *api.Container, containerType ContainerType) bool {
-		if c.StartupProbe != nil {
-			inUse = true
-			return false
-		}
-		return true
-	})
-
-	return inUse
-}
-
 // csiInUse returns true if any pod's spec include inline CSI volumes.
 func csiInUse(podSpec *api.PodSpec) bool {
 	if podSpec == nil {

pkg/features/kube_features.go

@@ -555,6 +555,7 @@ const (
 	// owner: @matthyx
 	// alpha: v1.16
 	// beta: v1.18
+	// GA: v1.20
 	//
 	// Enables the startupProbe in kubelet worker.
 	StartupProbe featuregate.Feature = "StartupProbe"
@@ -729,7 +730,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
 	EndpointSliceProxying:                          {Default: true, PreRelease: featuregate.Beta},
 	WindowsEndpointSliceProxying:                   {Default: false, PreRelease: featuregate.Alpha},
 	EvenPodsSpread:                                 {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.21
-	StartupProbe:                                   {Default: true, PreRelease: featuregate.Beta},
+	StartupProbe:                                   {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.21
 	AllowInsecureBackendProxy:                      {Default: true, PreRelease: featuregate.Beta},
 	PodDisruptionBudget:                            {Default: true, PreRelease: featuregate.Beta},
 	ServiceTopology:                                {Default: false, PreRelease: featuregate.Alpha},

pkg/kubelet/prober/BUILD

@@ -16,7 +16,6 @@ go_library(
     importpath = "k8s.io/kubernetes/pkg/kubelet/prober",
     deps = [
         "//pkg/api/v1/pod:go_default_library",
-        "//pkg/features:go_default_library",
         "//pkg/kubelet/container:go_default_library",
         "//pkg/kubelet/events:go_default_library",
         "//pkg/kubelet/prober/results:go_default_library",
@@ -32,7 +31,6 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
-        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//staging/src/k8s.io/client-go/tools/record:go_default_library",
         "//staging/src/k8s.io/component-base/metrics:go_default_library",
         "//vendor/k8s.io/klog/v2:go_default_library",
@@ -50,7 +48,6 @@ go_test(
     ],
     embed = [":go_default_library"],
     deps = [
-        "//pkg/features:go_default_library",
         "//pkg/kubelet/container:go_default_library",
         "//pkg/kubelet/container/testing:go_default_library",
         "//pkg/kubelet/pod:go_default_library",
@@ -67,10 +64,8 @@ go_test(
         "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
-        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
         "//staging/src/k8s.io/client-go/tools/record:go_default_library",
-        "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library",
         "//vendor/k8s.io/klog/v2:go_default_library",
         "//vendor/k8s.io/utils/exec:go_default_library",
     ],

pkg/kubelet/prober/prober_manager.go

@@ -23,11 +23,9 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/component-base/metrics"
 	"k8s.io/klog/v2"
-	"k8s.io/kubernetes/pkg/features"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 	"k8s.io/kubernetes/pkg/kubelet/prober/results"
 	"k8s.io/kubernetes/pkg/kubelet/status"
@@ -168,7 +166,7 @@ func (m *manager) AddPod(pod *v1.Pod) {
 	for _, c := range pod.Spec.Containers {
 		key.containerName = c.Name
 
-		if c.StartupProbe != nil && utilfeature.DefaultFeatureGate.Enabled(features.StartupProbe) {
+		if c.StartupProbe != nil {
 			key.probeType = startup
 			if _, ok := m.workers[key]; ok {
 				klog.Errorf("Startup probe already exists! %v - %v",
@@ -238,9 +236,6 @@ func (m *manager) UpdatePodStatus(podUID types.UID, podStatus *v1.PodStatus) {
 		var started bool
 		if c.State.Running == nil {
 			started = false
-		} else if !utilfeature.DefaultFeatureGate.Enabled(features.StartupProbe) {
-			// the container is running, assume it is started if the StartupProbe feature is disabled
-			started = true
 		} else if result, ok := m.startupManager.Get(kubecontainer.ParseContainerID(c.ContainerID)); ok {
 			started = result == results.Success
 		} else {

pkg/kubelet/prober/prober_manager_test.go

@@ -28,10 +28,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	featuregatetesting "k8s.io/component-base/featuregate/testing"
 	"k8s.io/klog/v2"
-	"k8s.io/kubernetes/pkg/features"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 	"k8s.io/kubernetes/pkg/kubelet/prober/results"
 	"k8s.io/kubernetes/pkg/probe"
@@ -92,7 +89,6 @@ func TestAddRemovePods(t *testing.T) {
 	}
 
 	m := newTestManager()
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StartupProbe, true)()
 	defer cleanup(t, m)
 	if err := expectProbes(m, nil); err != nil {
 		t.Error(err)
@@ -139,7 +135,6 @@ func TestAddRemovePods(t *testing.T) {
 
 func TestCleanupPods(t *testing.T) {
 	m := newTestManager()
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StartupProbe, true)()
 	defer cleanup(t, m)
 	podToCleanup := v1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
@@ -202,7 +197,6 @@ func TestCleanupPods(t *testing.T) {
 
 func TestCleanupRepeated(t *testing.T) {
 	m := newTestManager()
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StartupProbe, true)()
 	defer cleanup(t, m)
 	podTemplate := v1.Pod{
 		Spec: v1.PodSpec{

staging/src/k8s.io/api/core/v1/generated.proto

@@ -718,7 +718,6 @@ message Container {
   // This can be used to provide different probe parameters at the beginning of a Pod's lifecycle,
   // when it might take a long time to load data or warm a cache, than during steady-state operation.
   // This cannot be updated.
-  // This is a beta feature enabled by the StartupProbe feature flag.
   // More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
   // +optional
   optional Probe startupProbe = 22;

staging/src/k8s.io/api/core/v1/types.go

@@ -2287,7 +2287,6 @@ type Container struct {
 	// This can be used to provide different probe parameters at the beginning of a Pod's lifecycle,
 	// when it might take a long time to load data or warm a cache, than during steady-state operation.
 	// This cannot be updated.
-	// This is a beta feature enabled by the StartupProbe feature flag.
 	// More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
 	// +optional
 	StartupProbe *Probe `json:"startupProbe,omitempty" protobuf:"bytes,22,opt,name=startupProbe"`

staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go

@@ -339,7 +339,7 @@ var map_Container = map[string]string{
 	"volumeDevices":            "volumeDevices is the list of block devices to be used by the container.",
 	"livenessProbe":            "Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes",
 	"readinessProbe":           "Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes",
-	"startupProbe":             "StartupProbe indicates that the Pod has successfully initialized. If specified, no other probes are executed until this completes successfully. If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. This cannot be updated. This is a beta feature enabled by the StartupProbe feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes",
+	"startupProbe":             "StartupProbe indicates that the Pod has successfully initialized. If specified, no other probes are executed until this completes successfully. If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. This cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes",
 	"lifecycle":                "Actions that the management system should take in response to container lifecycle events. Cannot be updated.",
 	"terminationMessagePath":   "Optional: Path at which the file to which the container's termination message will be written is mounted into the container's filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.",
 	"terminationMessagePolicy": "Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated.",

test/e2e/common/container_probe.go

@@ -264,6 +264,144 @@ var _ = framework.KubeDescribe("Probing container", func() {
 		framework.ExpectNoError(e2eevents.WaitTimeoutForEvent(
 			f.ClientSet, f.Namespace.Name, expectedEvent, "0.0.0.0", framework.PodEventTimeout))
 	})
+
+	/*
+		Release: v1.16
+		Testname: Pod startup probe restart
+		Description: A Pod is created with a failing startup probe. The Pod MUST be killed and restarted incrementing restart count to 1, even if liveness would succeed.
+	*/
+	ginkgo.It("should be restarted startup probe fails", func() {
+		cmd := []string{"/bin/sh", "-c", "sleep 600"}
+		livenessProbe := &v1.Probe{
+			Handler: v1.Handler{
+				Exec: &v1.ExecAction{
+					Command: []string{"/bin/true"},
+				},
+			},
+			InitialDelaySeconds: 15,
+			FailureThreshold:    1,
+		}
+		startupProbe := &v1.Probe{
+			Handler: v1.Handler{
+				Exec: &v1.ExecAction{
+					Command: []string{"/bin/false"},
+				},
+			},
+			InitialDelaySeconds: 15,
+			FailureThreshold:    3,
+		}
+		pod := startupPodSpec(startupProbe, nil, livenessProbe, cmd)
+		RunLivenessTest(f, pod, 1, defaultObservationTimeout)
+	})
+
+	/*
+		Release: v1.16
+		Testname: Pod liveness probe delayed (long) by startup probe
+		Description: A Pod is created with failing liveness and startup probes. Liveness probe MUST NOT fail until startup probe expires.
+	*/
+	ginkgo.It("should *not* be restarted by liveness probe because startup probe delays it", func() {
+		cmd := []string{"/bin/sh", "-c", "sleep 600"}
+		livenessProbe := &v1.Probe{
+			Handler: v1.Handler{
+				Exec: &v1.ExecAction{
+					Command: []string{"/bin/false"},
+				},
+			},
+			InitialDelaySeconds: 15,
+			FailureThreshold:    1,
+		}
+		startupProbe := &v1.Probe{
+			Handler: v1.Handler{
+				Exec: &v1.ExecAction{
+					Command: []string{"/bin/false"},
+				},
+			},
+			InitialDelaySeconds: 15,
+			FailureThreshold:    60,
+		}
+		pod := startupPodSpec(startupProbe, nil, livenessProbe, cmd)
+		RunLivenessTest(f, pod, 0, defaultObservationTimeout)
+	})
+
+	/*
+		Release: v1.16
+		Testname: Pod liveness probe fails after startup success
+		Description: A Pod is created with failing liveness probe and delayed startup probe that uses 'exec' command to cat /temp/health file. The Container is started by creating /tmp/startup after 10 seconds, triggering liveness probe to fail. The Pod MUST now be killed and restarted incrementing restart count to 1.
+	*/
+	ginkgo.It("should be restarted by liveness probe after startup probe enables it", func() {
+		cmd := []string{"/bin/sh", "-c", "sleep 10; echo ok >/tmp/startup; sleep 600"}
+		livenessProbe := &v1.Probe{
+			Handler: v1.Handler{
+				Exec: &v1.ExecAction{
+					Command: []string{"/bin/false"},
+				},
+			},
+			InitialDelaySeconds: 15,
+			FailureThreshold:    1,
+		}
+		startupProbe := &v1.Probe{
+			Handler: v1.Handler{
+				Exec: &v1.ExecAction{
+					Command: []string{"cat", "/tmp/startup"},
+				},
+			},
+			InitialDelaySeconds: 15,
+			FailureThreshold:    60,
+		}
+		pod := startupPodSpec(startupProbe, nil, livenessProbe, cmd)
+		RunLivenessTest(f, pod, 1, defaultObservationTimeout)
+	})
+
+	/*
+		Release: v1.16
+		Testname: Pod readiness probe, delayed by startup probe
+		Description: A Pod is created with startup and readiness probes. The Container is started by creating /tmp/startup after 45 seconds, delaying the ready state by this amount of time. This is similar to the "Pod readiness probe, with initial delay" test.
+	*/
+	ginkgo.It("should not be ready until startupProbe succeeds", func() {
+		cmd := []string{"/bin/sh", "-c", "echo ok >/tmp/health; sleep 45; echo ok >/tmp/startup; sleep 600"}
+		readinessProbe := &v1.Probe{
+			Handler: v1.Handler{
+				Exec: &v1.ExecAction{
+					Command: []string{"cat", "/tmp/health"},
+				},
+			},
+			InitialDelaySeconds: 0,
+		}
+		startupProbe := &v1.Probe{
+			Handler: v1.Handler{
+				Exec: &v1.ExecAction{
+					Command: []string{"cat", "/tmp/startup"},
+				},
+			},
+			InitialDelaySeconds: 0,
+			FailureThreshold:    60,
+		}
+		p := podClient.Create(startupPodSpec(startupProbe, readinessProbe, nil, cmd))
+
+		p, err := podClient.Get(context.TODO(), p.Name, metav1.GetOptions{})
+		framework.ExpectNoError(err)
+
+		e2epod.WaitTimeoutForPodReadyInNamespace(f.ClientSet, p.Name, f.Namespace.Name, framework.PodStartTimeout)
+
+		p, err = podClient.Get(context.TODO(), p.Name, metav1.GetOptions{})
+		framework.ExpectNoError(err)
+
+		isReady, err := testutils.PodRunningReady(p)
+		framework.ExpectNoError(err)
+		framework.ExpectEqual(isReady, true, "pod should be ready")
+
+		// We assume the pod became ready when the container became ready. This
+		// is true for a single container pod.
+		readyTime, err := GetTransitionTimeForReadyCondition(p)
+		framework.ExpectNoError(err)
+		startedTime, err := GetContainerStartedTime(p, "busybox")
+		framework.ExpectNoError(err)
+
+		framework.Logf("Container started at %v, pod became ready at %v", startedTime, readyTime)
+		if readyTime.Sub(startedTime) < 40*time.Second {
+			framework.Failf("Pod became ready before startupProbe succeeded")
+		}
+	})
 })
 
 func GetContainerStartedTime(p *v1.Pod, containerName string) (time.Time, error) {
@@ -354,6 +492,27 @@ func livenessPodSpec(readinessProbe, livenessProbe *v1.Probe) *v1.Pod {
 	}
 }
 
+func startupPodSpec(startupProbe, readinessProbe, livenessProbe *v1.Probe, cmd []string) *v1.Pod {
+	return &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:   "startup-" + string(uuid.NewUUID()),
+			Labels: map[string]string{"test": "startup"},
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:           "busybox",
+					Image:          imageutils.GetE2EImage(imageutils.BusyBox),
+					Command:        cmd,
+					LivenessProbe:  livenessProbe,
+					ReadinessProbe: readinessProbe,
+					StartupProbe:   startupProbe,
+				},
+			},
+		},
+	}
+}
+
 func execHandler(cmd []string) v1.Handler {
 	return v1.Handler{
 		Exec: &v1.ExecAction{

test/e2e_node/BUILD

@@ -140,7 +140,6 @@ go_test(
         "runtime_conformance_test.go",
         "runtimeclass_test.go",
         "security_context_test.go",
-        "startup_probe_test.go",
         "summary_test.go",
         "system_node_critical_test.go",
         "topology_manager_test.go",

test/e2e_node/startup_probe_test.go

@@ -1,219 +0,0 @@
-/*
-Copyright 2019 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package e2enode
-
-import (
-	"context"
-	"time"
-
-	v1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/uuid"
-	"k8s.io/kubernetes/pkg/features"
-	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
-	"k8s.io/kubernetes/test/e2e/common"
-	"k8s.io/kubernetes/test/e2e/framework"
-	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
-	testutils "k8s.io/kubernetes/test/utils"
-	imageutils "k8s.io/kubernetes/test/utils/image"
-
-	"github.com/onsi/ginkgo"
-)
-
-const (
-	defaultObservationTimeout = time.Minute * 4
-)
-
-var _ = framework.KubeDescribe("StartupProbe [Serial] [Disruptive]", func() {
-	f := framework.NewDefaultFramework("startup-probe-test")
-	var podClient *framework.PodClient
-	ginkgo.BeforeEach(func() {
-		podClient = f.PodClient()
-	})
-
-	/*
-		These tests are located here as they require tempSetCurrentKubeletConfig to enable the feature gate for startupProbe.
-		Once the feature gate has been removed, these tests should come back to test/e2e/common/container_probe.go.
-	*/
-	ginkgo.Context("when a container has a startup probe", func() {
-		tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
-			if initialConfig.FeatureGates == nil {
-				initialConfig.FeatureGates = make(map[string]bool)
-			}
-			initialConfig.FeatureGates[string(features.StartupProbe)] = true
-		})
-
-		/*
-			Release: v1.16
-			Testname: Pod startup probe restart
-			Description: A Pod is created with a failing startup probe. The Pod MUST be killed and restarted incrementing restart count to 1, even if liveness would succeed.
-		*/
-		ginkgo.It("should be restarted startup probe fails", func() {
-			cmd := []string{"/bin/sh", "-c", "sleep 600"}
-			livenessProbe := &v1.Probe{
-				Handler: v1.Handler{
-					Exec: &v1.ExecAction{
-						Command: []string{"/bin/true"},
-					},
-				},
-				InitialDelaySeconds: 15,
-				FailureThreshold:    1,
-			}
-			startupProbe := &v1.Probe{
-				Handler: v1.Handler{
-					Exec: &v1.ExecAction{
-						Command: []string{"/bin/false"},
-					},
-				},
-				InitialDelaySeconds: 15,
-				FailureThreshold:    3,
-			}
-			pod := startupPodSpec(startupProbe, nil, livenessProbe, cmd)
-			common.RunLivenessTest(f, pod, 1, defaultObservationTimeout)
-		})
-
-		/*
-			Release: v1.16
-			Testname: Pod liveness probe delayed (long) by startup probe
-			Description: A Pod is created with failing liveness and startup probes. Liveness probe MUST NOT fail until startup probe expires.
-		*/
-		ginkgo.It("should *not* be restarted by liveness probe because startup probe delays it", func() {
-			cmd := []string{"/bin/sh", "-c", "sleep 600"}
-			livenessProbe := &v1.Probe{
-				Handler: v1.Handler{
-					Exec: &v1.ExecAction{
-						Command: []string{"/bin/false"},
-					},
-				},
-				InitialDelaySeconds: 15,
-				FailureThreshold:    1,
-			}
-			startupProbe := &v1.Probe{
-				Handler: v1.Handler{
-					Exec: &v1.ExecAction{
-						Command: []string{"/bin/false"},
-					},
-				},
-				InitialDelaySeconds: 15,
-				FailureThreshold:    60,
-			}
-			pod := startupPodSpec(startupProbe, nil, livenessProbe, cmd)
-			common.RunLivenessTest(f, pod, 0, defaultObservationTimeout)
-		})
-
-		/*
-			Release: v1.16
-			Testname: Pod liveness probe fails after startup success
-			Description: A Pod is created with failing liveness probe and delayed startup probe that uses 'exec' command to cat /temp/health file. The Container is started by creating /tmp/startup after 10 seconds, triggering liveness probe to fail. The Pod MUST now be killed and restarted incrementing restart count to 1.
-		*/
-		ginkgo.It("should be restarted by liveness probe after startup probe enables it", func() {
-			cmd := []string{"/bin/sh", "-c", "sleep 10; echo ok >/tmp/startup; sleep 600"}
-			livenessProbe := &v1.Probe{
-				Handler: v1.Handler{
-					Exec: &v1.ExecAction{
-						Command: []string{"/bin/false"},
-					},
-				},
-				InitialDelaySeconds: 15,
-				FailureThreshold:    1,
-			}
-			startupProbe := &v1.Probe{
-				Handler: v1.Handler{
-					Exec: &v1.ExecAction{
-						Command: []string{"cat", "/tmp/startup"},
-					},
-				},
-				InitialDelaySeconds: 15,
-				FailureThreshold:    60,
-			}
-			pod := startupPodSpec(startupProbe, nil, livenessProbe, cmd)
-			common.RunLivenessTest(f, pod, 1, defaultObservationTimeout)
-		})
-
-		/*
-			Release: v1.16
-			Testname: Pod readiness probe, delayed by startup probe
-			Description: A Pod is created with startup and readiness probes. The Container is started by creating /tmp/startup after 45 seconds, delaying the ready state by this amount of time. This is similar to the "Pod readiness probe, with initial delay" test.
-		*/
-		ginkgo.It("should not be ready until startupProbe succeeds", func() {
-			cmd := []string{"/bin/sh", "-c", "echo ok >/tmp/health; sleep 45; echo ok >/tmp/startup; sleep 600"}
-			readinessProbe := &v1.Probe{
-				Handler: v1.Handler{
-					Exec: &v1.ExecAction{
-						Command: []string{"cat", "/tmp/health"},
-					},
-				},
-				InitialDelaySeconds: 0,
-			}
-			startupProbe := &v1.Probe{
-				Handler: v1.Handler{
-					Exec: &v1.ExecAction{
-						Command: []string{"cat", "/tmp/startup"},
-					},
-				},
-				InitialDelaySeconds: 0,
-				FailureThreshold:    60,
-			}
-			p := podClient.Create(startupPodSpec(startupProbe, readinessProbe, nil, cmd))
-
-			p, err := podClient.Get(context.TODO(), p.Name, metav1.GetOptions{})
-			framework.ExpectNoError(err)
-
-			e2epod.WaitTimeoutForPodReadyInNamespace(f.ClientSet, p.Name, f.Namespace.Name, framework.PodStartTimeout)
-
-			p, err = podClient.Get(context.TODO(), p.Name, metav1.GetOptions{})
-			framework.ExpectNoError(err)
-
-			isReady, err := testutils.PodRunningReady(p)
-			framework.ExpectNoError(err)
-			framework.ExpectEqual(isReady, true, "pod should be ready")
-
-			// We assume the pod became ready when the container became ready. This
-			// is true for a single container pod.
-			readyTime, err := common.GetTransitionTimeForReadyCondition(p)
-			framework.ExpectNoError(err)
-			startedTime, err := common.GetContainerStartedTime(p, "busybox")
-			framework.ExpectNoError(err)
-
-			framework.Logf("Container started at %v, pod became ready at %v", startedTime, readyTime)
-			if readyTime.Sub(startedTime) < 40*time.Second {
-				framework.Failf("Pod became ready before startupProbe succeeded")
-			}
-		})
-	})
-})
-
-func startupPodSpec(startupProbe, readinessProbe, livenessProbe *v1.Probe, cmd []string) *v1.Pod {
-	return &v1.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:   "startup-" + string(uuid.NewUUID()),
-			Labels: map[string]string{"test": "startup"},
-		},
-		Spec: v1.PodSpec{
-			Containers: []v1.Container{
-				{
-					Name:           "busybox",
-					Image:          imageutils.GetE2EImage(imageutils.BusyBox),
-					Command:        cmd,
-					LivenessProbe:  livenessProbe,
-					ReadinessProbe: readinessProbe,
-					StartupProbe:   startupProbe,
-				},
-			},
-		},
-	}
-}