diff --git a/pkg/controller/garbagecollector/garbagecollector_test.go b/pkg/controller/garbagecollector/garbagecollector_test.go
index 15d96d73499..fdcf11ececa 100644
--- a/pkg/controller/garbagecollector/garbagecollector_test.go
+++ b/pkg/controller/garbagecollector/garbagecollector_test.go
@@ -409,6 +409,16 @@ func TestProcessEvent(t *testing.T) {
 	}
 }
 
+func BenchmarkReferencesDiffs(t *testing.B) {
+	t.ReportAllocs()
+	t.ResetTimer()
+	for n := 0; n < t.N; n++ {
+		old := []metav1.OwnerReference{{UID: "1"}, {UID: "2"}}
+		new := []metav1.OwnerReference{{UID: "2"}, {UID: "3"}}
+		referencesDiffs(old, new)
+	}
+}
+
 // TestDependentsRace relies on golang's data race detector to check if there is
 // data race among in the dependents field.
 func TestDependentsRace(t *testing.T) {
diff --git a/pkg/controller/garbagecollector/graph_builder.go b/pkg/controller/garbagecollector/graph_builder.go
index 525e357c201..b43ed70d3c8 100644
--- a/pkg/controller/garbagecollector/graph_builder.go
+++ b/pkg/controller/garbagecollector/graph_builder.go
@@ -403,27 +403,21 @@ func referencesDiffs(old []metav1.OwnerReference, new []metav1.OwnerReference) (
 		oldUIDToRef[string(value.UID)] = value
 	}
 	oldUIDSet := sets.StringKeySet(oldUIDToRef)
-	newUIDToRef := make(map[string]metav1.OwnerReference)
 	for _, value := range new {
-		newUIDToRef[string(value.UID)] = value
-	}
-	newUIDSet := sets.StringKeySet(newUIDToRef)
-
-	addedUID := newUIDSet.Difference(oldUIDSet)
-	removedUID := oldUIDSet.Difference(newUIDSet)
-	intersection := oldUIDSet.Intersection(newUIDSet)
-
-	for uid := range addedUID {
-		added = append(added, newUIDToRef[uid])
-	}
-	for uid := range removedUID {
-		removed = append(removed, oldUIDToRef[uid])
-	}
-	for uid := range intersection {
-		if !reflect.DeepEqual(oldUIDToRef[uid], newUIDToRef[uid]) {
-			changed = append(changed, ownerRefPair{oldRef: oldUIDToRef[uid], newRef: newUIDToRef[uid]})
+		newUID := string(value.UID)
+		if oldUIDSet.Has(newUID) {
+			if !reflect.DeepEqual(oldUIDToRef[newUID], value) {
+				changed = append(changed, ownerRefPair{oldRef: oldUIDToRef[newUID], newRef: value})
+			}
+			oldUIDSet.Delete(newUID)
+		} else {
+			added = append(added, value)
 		}
 	}
+	for oldUID := range oldUIDSet {
+		removed = append(removed, oldUIDToRef[oldUID])
+	}
+
 	return added, removed, changed
 }