diff --git a/pkg/scheduler/framework/plugins/serviceaffinity/service_affinity.go b/pkg/scheduler/framework/plugins/serviceaffinity/service_affinity.go
index bf82913ffbe..2bff04bcafe 100644
--- a/pkg/scheduler/framework/plugins/serviceaffinity/service_affinity.go
+++ b/pkg/scheduler/framework/plugins/serviceaffinity/service_affinity.go
@@ -127,10 +127,13 @@ func (pl *ServiceAffinity) createPreFilterState(pod *v1.Pod) (*preFilterState, e
 
 // PreFilter invoked at the prefilter extension point.
 func (pl *ServiceAffinity) PreFilter(ctx context.Context, cycleState *framework.CycleState, pod *v1.Pod) *framework.Status {
+	if len(pl.args.AffinityLabels) == 0 {
+		return nil
+	}
+
 	s, err := pl.createPreFilterState(pod)
 	if err != nil {
 		return framework.NewStatus(framework.Error, fmt.Sprintf("could not create preFilterState: %v", err))
-
 	}
 	cycleState.Write(preFilterStateKey, s)
 	return nil
@@ -138,6 +141,9 @@ func (pl *ServiceAffinity) PreFilter(ctx context.Context, cycleState *framework.
 
 // PreFilterExtensions returns prefilter extensions, pod add and remove.
 func (pl *ServiceAffinity) PreFilterExtensions() framework.PreFilterExtensions {
+	if len(pl.args.AffinityLabels) == 0 {
+		return nil
+	}
 	return pl
 }
 
diff --git a/pkg/scheduler/framework/plugins/serviceaffinity/service_affinity_test.go b/pkg/scheduler/framework/plugins/serviceaffinity/service_affinity_test.go
index efae481fedf..6801ee7805d 100644
--- a/pkg/scheduler/framework/plugins/serviceaffinity/service_affinity_test.go
+++ b/pkg/scheduler/framework/plugins/serviceaffinity/service_affinity_test.go
@@ -500,6 +500,9 @@ func TestPreFilterStateAddRemovePod(t *testing.T) {
 				p := &ServiceAffinity{
 					sharedLister:  snapshot,
 					serviceLister: fakeframework.ServiceLister(test.services),
+					args: config.ServiceAffinityArgs{
+						AffinityLabels: []string{"region", "zone"},
+					},
 				}
 				cycleState := framework.NewCycleState()
 				preFilterStatus := p.PreFilter(context.Background(), cycleState, test.pendingPod)