diff --git a/pkg/server/server.go b/pkg/server/server.go
index a7d81ebbd..6163539b8 100644
--- a/pkg/server/server.go
+++ b/pkg/server/server.go
@@ -51,7 +51,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
-	"k8s.io/apimachinery/pkg/util/sets"
 	utilwait "k8s.io/apimachinery/pkg/util/wait"
 	informerfactory "k8s.io/client-go/informers"
 	v1coreinformers "k8s.io/client-go/informers/core/v1"
@@ -479,9 +478,16 @@ func (s *Server) handleDelegateRequest(r *http.Request) ([]byte, error) {
 }
 
 func overrideCNIConfigWithServerConfig(cniConf []byte, overrideConf []byte, ignoreReadinessIndicator bool) ([]byte, error) {
-	if len(overrideConf) == 0 {
+	// If there is no server-side override config AND we don't need to strip any keys,
+	// return the client config unchanged.
+	if len(overrideConf) == 0 && !ignoreReadinessIndicator {
 		return cniConf, nil
 	}
+	// Treat a missing server config as an empty object so the key-stripping logic below
+	// still runs when ignoreReadinessIndicator is true.
+	if len(overrideConf) == 0 {
+		overrideConf = []byte("{}")
+	}
 
 	var cni map[string]interface{}
 	if err := json.Unmarshal(cniConf, &cni); err != nil {
@@ -493,14 +499,13 @@ func overrideCNIConfigWithServerConfig(cniConf []byte, overrideConf []byte, igno
 		return nil, fmt.Errorf("failed to unmarshall CNI override config: %w", err)
 	}
 
-	// Copy each key of the override config into the CNI config except for
-	// a few specific keys
-	ignoreKeys := sets.NewString()
+	// Remove keys from the client config that the server wants to ignore, then
+	// overlay the server-side overrides (also skipping those same keys).
 	if ignoreReadinessIndicator {
-		ignoreKeys.Insert("readinessindicatorfile")
+		delete(cni, "readinessindicatorfile")
 	}
 	for overrideKey, overrideVal := range override {
-		if !ignoreKeys.Has(overrideKey) {
+		if !ignoreReadinessIndicator || overrideKey != "readinessindicatorfile" {
 			cni[overrideKey] = overrideVal
 		}
 	}