From 71c3593f85b9af515714dfb8ab5e41bd0318817f Mon Sep 17 00:00:00 2001
From: amewayne <hqweiyu@gmail.com>
Date: Wed, 10 Jan 2024 19:30:41 +0800
Subject: [PATCH] support nodeAnnotationsChanged event to trigger rescheduling

---
 pkg/scheduler/eventhandlers.go                  | 17 ++++++++++++-----
 pkg/scheduler/eventhandlers_test.go             |  6 ++++++
 pkg/scheduler/framework/types.go                |  5 +++--
 pkg/scheduler/internal/queue/events.go          |  2 ++
 .../internal/queue/scheduling_queue_test.go     |  1 +
 5 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/pkg/scheduler/eventhandlers.go b/pkg/scheduler/eventhandlers.go
index 1909577a293..d5ed3018222 100644
--- a/pkg/scheduler/eventhandlers.go
+++ b/pkg/scheduler/eventhandlers.go
@@ -19,12 +19,12 @@ package scheduler
 import (
 	"context"
 	"fmt"
-	"reflect"
 	"strings"
 	"time"
 
 	v1 "k8s.io/api/core/v1"
 	storagev1 "k8s.io/api/storage/v1"
+	"k8s.io/apimachinery/pkg/api/equality"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -540,20 +540,23 @@ func nodeSchedulingPropertiesChange(newNode *v1.Node, oldNode *v1.Node) []framew
 	if nodeConditionsChanged(newNode, oldNode) {
 		events = append(events, queue.NodeConditionChange)
 	}
+	if nodeAnnotationsChanged(newNode, oldNode) {
+		events = append(events, queue.NodeAnnotationChange)
+	}
 
 	return events
 }
 
 func nodeAllocatableChanged(newNode *v1.Node, oldNode *v1.Node) bool {
-	return !reflect.DeepEqual(oldNode.Status.Allocatable, newNode.Status.Allocatable)
+	return !equality.Semantic.DeepEqual(oldNode.Status.Allocatable, newNode.Status.Allocatable)
 }
 
 func nodeLabelsChanged(newNode *v1.Node, oldNode *v1.Node) bool {
-	return !reflect.DeepEqual(oldNode.GetLabels(), newNode.GetLabels())
+	return !equality.Semantic.DeepEqual(oldNode.GetLabels(), newNode.GetLabels())
 }
 
 func nodeTaintsChanged(newNode *v1.Node, oldNode *v1.Node) bool {
-	return !reflect.DeepEqual(newNode.Spec.Taints, oldNode.Spec.Taints)
+	return !equality.Semantic.DeepEqual(newNode.Spec.Taints, oldNode.Spec.Taints)
 }
 
 func nodeConditionsChanged(newNode *v1.Node, oldNode *v1.Node) bool {
@@ -564,13 +567,17 @@ func nodeConditionsChanged(newNode *v1.Node, oldNode *v1.Node) bool {
 		}
 		return conditionStatuses
 	}
-	return !reflect.DeepEqual(strip(oldNode.Status.Conditions), strip(newNode.Status.Conditions))
+	return !equality.Semantic.DeepEqual(strip(oldNode.Status.Conditions), strip(newNode.Status.Conditions))
 }
 
 func nodeSpecUnschedulableChanged(newNode *v1.Node, oldNode *v1.Node) bool {
 	return newNode.Spec.Unschedulable != oldNode.Spec.Unschedulable && !newNode.Spec.Unschedulable
 }
 
+func nodeAnnotationsChanged(newNode *v1.Node, oldNode *v1.Node) bool {
+	return !equality.Semantic.DeepEqual(oldNode.GetAnnotations(), newNode.GetAnnotations())
+}
+
 func preCheckForNode(nodeInfo *framework.NodeInfo) queue.PreEnqueueCheck {
 	// Note: the following checks doesn't take preemption into considerations, in very rare
 	// cases (e.g., node resizing), "pod" may still fail a check but preemption helps. We deliberately
diff --git a/pkg/scheduler/eventhandlers_test.go b/pkg/scheduler/eventhandlers_test.go
index e81a0fdd938..f0254df4095 100644
--- a/pkg/scheduler/eventhandlers_test.go
+++ b/pkg/scheduler/eventhandlers_test.go
@@ -574,6 +574,12 @@ func TestNodeSchedulingPropertiesChange(t *testing.T) {
 			}).Obj(),
 			wantEvents: []framework.ClusterEvent{queue.NodeTaintChange},
 		},
+		{
+			name:       "only node annotation changed",
+			newNode:    st.MakeNode().Annotation("foo", "bar").Obj(),
+			oldNode:    st.MakeNode().Annotation("foo", "fuz").Obj(),
+			wantEvents: []framework.ClusterEvent{queue.NodeAnnotationChange},
+		},
 		{
 			name:    "only node condition changed",
 			newNode: st.MakeNode().Obj(),
diff --git a/pkg/scheduler/framework/types.go b/pkg/scheduler/framework/types.go
index 7001ff9fd5a..188f5aa2f24 100644
--- a/pkg/scheduler/framework/types.go
+++ b/pkg/scheduler/framework/types.go
@@ -53,11 +53,12 @@ const (
 	UpdateNodeLabel       // 1000
 	UpdateNodeTaint       // 10000
 	UpdateNodeCondition   // 100000
+	UpdateNodeAnnotation  // 1000000
 
-	All ActionType = 1<<iota - 1 // 111111
+	All ActionType = 1<<iota - 1 // 1111111
 
 	// Use the general Update type if you don't either know or care the specific sub-Update type to use.
-	Update = UpdateNodeAllocatable | UpdateNodeLabel | UpdateNodeTaint | UpdateNodeCondition
+	Update = UpdateNodeAllocatable | UpdateNodeLabel | UpdateNodeTaint | UpdateNodeCondition | UpdateNodeAnnotation
 )
 
 // GVK is short for group/version/kind, which can uniquely represent a particular API resource.
diff --git a/pkg/scheduler/internal/queue/events.go b/pkg/scheduler/internal/queue/events.go
index 44f8922968e..1d6e8a907e3 100644
--- a/pkg/scheduler/internal/queue/events.go
+++ b/pkg/scheduler/internal/queue/events.go
@@ -52,6 +52,8 @@ var (
 	NodeAllocatableChange = framework.ClusterEvent{Resource: framework.Node, ActionType: framework.UpdateNodeAllocatable, Label: "NodeAllocatableChange"}
 	// NodeLabelChange is the event when node label is changed.
 	NodeLabelChange = framework.ClusterEvent{Resource: framework.Node, ActionType: framework.UpdateNodeLabel, Label: "NodeLabelChange"}
+	// NodeAnnotationChange is the event when node annotation is changed.
+	NodeAnnotationChange = framework.ClusterEvent{Resource: framework.Node, ActionType: framework.UpdateNodeAnnotation, Label: "NodeAnnotationChange"}
 	// NodeTaintChange is the event when node taint is changed.
 	NodeTaintChange = framework.ClusterEvent{Resource: framework.Node, ActionType: framework.UpdateNodeTaint, Label: "NodeTaintChange"}
 	// NodeConditionChange is the event when node condition is changed.
diff --git a/pkg/scheduler/internal/queue/scheduling_queue_test.go b/pkg/scheduler/internal/queue/scheduling_queue_test.go
index e097d617073..25e998677aa 100644
--- a/pkg/scheduler/internal/queue/scheduling_queue_test.go
+++ b/pkg/scheduler/internal/queue/scheduling_queue_test.go
@@ -1217,6 +1217,7 @@ func BenchmarkMoveAllToActiveOrBackoffQueue(b *testing.B) {
 		NodeAllocatableChange,
 		NodeConditionChange,
 		NodeLabelChange,
+		NodeAnnotationChange,
 		PvcAdd,
 		PvcUpdate,
 		PvAdd,