diff --git a/pkg/scheduler/backend/queue/backoff_queue.go b/pkg/scheduler/backend/queue/backoff_queue.go
index 6e0bbd6808c..0a0f6b6df1c 100644
--- a/pkg/scheduler/backend/queue/backoff_queue.go
+++ b/pkg/scheduler/backend/queue/backoff_queue.go
@@ -239,15 +239,11 @@ func (bq *backoffQueue) calculateBackoffDuration(podInfo *framework.QueuedPodInf
 		return 0
 	}
 
-	duration := bq.podInitialBackoff
-	for i := 1; i < podInfo.Attempts; i++ {
-		// Use subtraction instead of addition or multiplication to avoid overflow.
-		if duration > bq.podMaxBackoff-duration {
-			return bq.podMaxBackoff
-		}
-		duration += duration
+	shift := podInfo.Attempts - 1
+	if bq.podInitialBackoff > bq.podMaxBackoff>>shift {
+		return bq.podMaxBackoff
 	}
-	return duration
+	return time.Duration(bq.podInitialBackoff << shift)
 }
 
 func (bq *backoffQueue) popAllBackoffCompletedWithQueue(logger klog.Logger, queue *heap.Heap[*framework.QueuedPodInfo]) []*framework.QueuedPodInfo {
diff --git a/pkg/scheduler/backend/queue/backoff_queue_test.go b/pkg/scheduler/backend/queue/backoff_queue_test.go
index 8ebf7799729..9862bcdbcf7 100644
--- a/pkg/scheduler/backend/queue/backoff_queue_test.go
+++ b/pkg/scheduler/backend/queue/backoff_queue_test.go
@@ -48,6 +48,13 @@ func TestBackoffQueue_calculateBackoffDuration(t *testing.T) {
 		},
 		{
 			name:                   "normal",
+			initialBackoffDuration: 3 * time.Nanosecond,
+			maxBackoffDuration:     1000 * time.Nanosecond,
+			podInfo:                &framework.QueuedPodInfo{Attempts: 5},
+			want:                   48 * time.Nanosecond, // 3 * 2^4 = 48
+		},
+		{
+			name:                   "hitting max backoff duration",
 			initialBackoffDuration: 1 * time.Nanosecond,
 			maxBackoffDuration:     32 * time.Nanosecond,
 			podInfo:                &framework.QueuedPodInfo{Attempts: 16},