diff --git a/test/e2e/framework/deployment/BUILD b/test/e2e/framework/deployment/BUILD
index e5aef9109fa..c0b346e03bc 100644
--- a/test/e2e/framework/deployment/BUILD
+++ b/test/e2e/framework/deployment/BUILD
@@ -10,9 +10,9 @@ go_library(
     ],
     importpath = "k8s.io/kubernetes/test/e2e/framework/deployment",
     deps = [
-        "//pkg/controller/deployment/util:go_default_library",
         "//staging/src/k8s.io/api/apps/v1:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
diff --git a/test/e2e/framework/deployment/fixtures.go b/test/e2e/framework/deployment/fixtures.go
index 7792c6dd16a..54e99865904 100644
--- a/test/e2e/framework/deployment/fixtures.go
+++ b/test/e2e/framework/deployment/fixtures.go
@@ -19,13 +19,14 @@ package deployment
 import (
 	"context"
 	"fmt"
+	"sort"
 
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/uuid"
 	clientset "k8s.io/client-go/kubernetes"
-	deploymentutil "k8s.io/kubernetes/pkg/controller/deployment/util"
 	"k8s.io/kubernetes/test/e2e/framework"
 	testutils "k8s.io/kubernetes/test/utils"
 	imageutils "k8s.io/kubernetes/test/utils/image"
@@ -86,22 +87,90 @@ func CreateDeployment(client clientset.Interface, replicas int32, podLabels map[
 
 // GetPodsForDeployment gets pods for the given deployment
 func GetPodsForDeployment(client clientset.Interface, deployment *appsv1.Deployment) (*v1.PodList, error) {
-	replicaSet, err := deploymentutil.GetNewReplicaSet(deployment, client.AppsV1())
+	replicaSetSelector, err := metav1.LabelSelectorAsSelector(deployment.Spec.Selector)
 	if err != nil {
-		return nil, fmt.Errorf("Failed to get new replica set for deployment %q: %v", deployment.Name, err)
+		return nil, err
 	}
+
+	replicaSetListOptions := metav1.ListOptions{LabelSelector: replicaSetSelector.String()}
+	allReplicaSets, err := client.AppsV1().ReplicaSets(deployment.Namespace).List(context.TODO(), replicaSetListOptions)
+	if err != nil {
+		return nil, err
+	}
+
+	ownedReplicaSets := make([]*appsv1.ReplicaSet, 0, len(allReplicaSets.Items))
+	for _, rs := range allReplicaSets.Items {
+		if !metav1.IsControlledBy(&rs, deployment) {
+			continue
+		}
+
+		ownedReplicaSets = append(ownedReplicaSets, &rs)
+	}
+
+	// We ignore pod-template-hash because:
+	// 1. The hash result would be different upon podTemplateSpec API changes
+	//    (e.g. the addition of a new field will cause the hash code to change)
+	// 2. The deployment template won't have hash labels
+	podTemplatesEqualsIgnoringHash := func(template1, template2 *v1.PodTemplateSpec) bool {
+		t1Copy := template1.DeepCopy()
+		t2Copy := template2.DeepCopy()
+		// Remove hash labels from template.Labels before comparing
+		delete(t1Copy.Labels, appsv1.DefaultDeploymentUniqueLabelKey)
+		delete(t2Copy.Labels, appsv1.DefaultDeploymentUniqueLabelKey)
+		return apiequality.Semantic.DeepEqual(t1Copy, t2Copy)
+	}
+
+	var replicaSet *appsv1.ReplicaSet
+	// In rare cases, such as after cluster upgrades, Deployment may end up with
+	// having more than one new ReplicaSets that have the same template as its template,
+	// see https://github.com/kubernetes/kubernetes/issues/40415
+	// We deterministically choose the oldest new ReplicaSet.
+	sort.Sort(replicaSetsByCreationTimestamp(ownedReplicaSets))
+	for _, rs := range ownedReplicaSets {
+		if !podTemplatesEqualsIgnoringHash(&rs.Spec.Template, &deployment.Spec.Template) {
+			continue
+		}
+
+		replicaSet = rs
+		break
+	}
+
 	if replicaSet == nil {
 		return nil, fmt.Errorf("expected a new replica set for deployment %q, found none", deployment.Name)
 	}
-	podListFunc := func(namespace string, options metav1.ListOptions) (*v1.PodList, error) {
-		return client.CoreV1().Pods(namespace).List(context.TODO(), options)
-	}
-	rsList := []*appsv1.ReplicaSet{replicaSet}
-	podList, err := deploymentutil.ListPods(deployment, rsList, podListFunc)
+
+	podSelector, err := metav1.LabelSelectorAsSelector(deployment.Spec.Selector)
 	if err != nil {
-		return nil, fmt.Errorf("Failed to list Pods of Deployment %q: %v", deployment.Name, err)
+		return nil, err
 	}
-	return podList, nil
+	podListOptions := metav1.ListOptions{LabelSelector: podSelector.String()}
+	allPods, err := client.CoreV1().Pods(deployment.Namespace).List(context.TODO(), podListOptions)
+	if err != nil {
+		return nil, err
+	}
+
+	replicaSetUID := replicaSet.UID
+	ownedPods := &v1.PodList{Items: make([]v1.Pod, 0, len(allPods.Items))}
+	for _, pod := range allPods.Items {
+		controllerRef := metav1.GetControllerOf(&pod)
+		if controllerRef != nil && controllerRef.UID == replicaSetUID {
+			ownedPods.Items = append(ownedPods.Items, pod)
+		}
+	}
+
+	return ownedPods, nil
+}
+
+// replicaSetsByCreationTimestamp sorts a list of ReplicaSet by creation timestamp, using their names as a tie breaker.
+type replicaSetsByCreationTimestamp []*appsv1.ReplicaSet
+
+func (o replicaSetsByCreationTimestamp) Len() int      { return len(o) }
+func (o replicaSetsByCreationTimestamp) Swap(i, j int) { o[i], o[j] = o[j], o[i] }
+func (o replicaSetsByCreationTimestamp) Less(i, j int) bool {
+	if o[i].CreationTimestamp.Equal(&o[j].CreationTimestamp) {
+		return o[i].Name < o[j].Name
+	}
+	return o[i].CreationTimestamp.Before(&o[j].CreationTimestamp)
 }
 
 // testDeployment creates a deployment definition based on the namespace. The deployment references the PVC's