diff --git a/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go b/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go
index 3b6aad3c707..5fd44f9062f 100644
--- a/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go
+++ b/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go
@@ -417,7 +417,7 @@ var _ = SIGDescribe("[Feature:HPA] [Serial] [Slow] Horizontal pod autoscaling (n
 			ginkgo.DeferCleanup(e2eautoscaling.DeleteHPAWithBehavior, rc, hpa.Name)
 
 			ginkgo.By("triggering scale up by increasing consumption")
-			rc.ConsumeCPU(usageForReplicas(4))
+			rc.ConsumeCPU(usageForReplicas(3))
 			waitDeadline := upScaleStabilization
 
 			ginkgo.By("verifying number of replicas stay in desired range within stabilisation window")
@@ -426,7 +426,7 @@ var _ = SIGDescribe("[Feature:HPA] [Serial] [Slow] Horizontal pod autoscaling (n
 			ginkgo.By("waiting for replicas to scale up after stabilisation window passed")
 			waitStart := time.Now()
 			waitDeadline = maxHPAReactionTime + maxResourceConsumerDelay + waitBuffer
-			rc.WaitForReplicas(ctx, 4, waitDeadline)
+			rc.WaitForReplicas(ctx, 3, waitDeadline)
 			timeWaited := time.Now().Sub(waitStart)
 			framework.Logf("time waited for scale up: %s", timeWaited)
 			gomega.Expect(timeWaited).To(gomega.BeNumerically("<", waitDeadline), "waited %s, wanted less than %s", timeWaited, waitDeadline)
@@ -436,7 +436,7 @@ var _ = SIGDescribe("[Feature:HPA] [Serial] [Slow] Horizontal pod autoscaling (n
 			waitDeadline = downScaleStabilization
 
 			ginkgo.By("verifying number of replicas stay in desired range within stabilisation window")
-			rc.EnsureDesiredReplicasInRange(ctx, 4, 4, waitDeadline, hpa.Name)
+			rc.EnsureDesiredReplicasInRange(ctx, 3, 3, waitDeadline, hpa.Name)
 
 			ginkgo.By("waiting for replicas to scale down after stabilisation window passed")
 			waitStart = time.Now()