diff --git a/pkg/scheduler/backend/queue/active_queue.go b/pkg/scheduler/backend/queue/active_queue.go
index 207f052962c..f44b552d8ab 100644
--- a/pkg/scheduler/backend/queue/active_queue.go
+++ b/pkg/scheduler/backend/queue/active_queue.go
@@ -20,6 +20,7 @@ import (
 	"container/list"
 	"fmt"
 	"sync"
+	"time"
 
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -252,6 +253,7 @@ func (aq *activeQueue) unlockedPop(logger klog.Logger) (*framework.QueuedPodInfo
 		return nil, err
 	}
 	pInfo.Attempts++
+	pInfo.BackoffExpiration = time.Time{}
 	// In flight, no concurrent events yet.
 	if aq.isSchedulingQueueHintEnabled {
 		// If the pod is already in the map, we shouldn't overwrite the inFlightPods otherwise it'd lead to a memory leak.
diff --git a/pkg/scheduler/backend/queue/backoff_queue.go b/pkg/scheduler/backend/queue/backoff_queue.go
index 27663c359ae..656e5098e70 100644
--- a/pkg/scheduler/backend/queue/backoff_queue.go
+++ b/pkg/scheduler/backend/queue/backoff_queue.go
@@ -32,8 +32,6 @@ type backoffQueuer interface {
 	// isPodBackingoff returns true if a pod is still waiting for its backoff timer.
 	// If this returns true, the pod should not be re-tried.
 	isPodBackingoff(podInfo *framework.QueuedPodInfo) bool
-	// getBackoffTime returns the time that podInfo completes backoff
-	getBackoffTime(podInfo *framework.QueuedPodInfo) time.Time
 	// popEachBackoffCompleted run fn for all pods from podBackoffQ and podErrorBackoffQ that completed backoff while popping them.
 	popEachBackoffCompleted(logger klog.Logger, fn func(pInfo *framework.QueuedPodInfo))
 
@@ -113,11 +111,17 @@ func (bq *backoffQueue) isPodBackingoff(podInfo *framework.QueuedPodInfo) bool {
 	return boTime.After(bq.clock.Now())
 }
 
-// getBackoffTime returns the time that podInfo completes backoff
+// getBackoffTime returns the time that podInfo completes backoff.
+// It caches the result in podInfo.BackoffExpiration and returns this value in subsequent calls.
+// The cache will be cleared when this pod is poped from the scheduling queue again (i.e., at activeQ's pop),
+// because of the fact that the backoff time is calculated based on podInfo.Attempts,
+// which doesn't get changed until the pod's scheduling is retried.
 func (bq *backoffQueue) getBackoffTime(podInfo *framework.QueuedPodInfo) time.Time {
-	duration := bq.calculateBackoffDuration(podInfo)
-	backoffTime := podInfo.Timestamp.Add(duration)
-	return backoffTime
+	if podInfo.BackoffExpiration.IsZero() {
+		duration := bq.calculateBackoffDuration(podInfo)
+		podInfo.BackoffExpiration = podInfo.Timestamp.Add(duration)
+	}
+	return podInfo.BackoffExpiration
 }
 
 // calculateBackoffDuration is a helper function for calculating the backoffDuration
diff --git a/pkg/scheduler/backend/queue/scheduling_queue_test.go b/pkg/scheduler/backend/queue/scheduling_queue_test.go
index aaf04e66c0c..f779b022eb3 100644
--- a/pkg/scheduler/backend/queue/scheduling_queue_test.go
+++ b/pkg/scheduler/backend/queue/scheduling_queue_test.go
@@ -3640,7 +3640,7 @@ func TestBackOffFlow(t *testing.T) {
 			}
 
 			// Check backoff duration.
-			deadline := q.backoffQ.getBackoffTime(podInfo)
+			deadline := podInfo.BackoffExpiration
 			backoff := deadline.Sub(timestamp)
 			if backoff != step.wantBackoff {
 				t.Errorf("got backoff %s, want %s", backoff, step.wantBackoff)
diff --git a/pkg/scheduler/framework/types.go b/pkg/scheduler/framework/types.go
index 5487902f4f5..5fd10a33ddf 100644
--- a/pkg/scheduler/framework/types.go
+++ b/pkg/scheduler/framework/types.go
@@ -366,6 +366,8 @@ type QueuedPodInfo struct {
 	// Number of schedule attempts before successfully scheduled.
 	// It's used to record the # attempts metric and calculate the backoff time this Pod is obliged to get before retrying.
 	Attempts int
+	// BackoffExpiration is the time when the Pod will complete its backoff.
+	BackoffExpiration time.Time
 	// The time when the pod is added to the queue for the first time. The pod may be added
 	// back to the queue multiple times before it's successfully scheduled.
 	// It shouldn't be updated once initialized. It's used to record the e2e scheduling