diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/equality.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/equality.go
index b130c8d7812..e68e9e02ac1 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/equality.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/equality.go
@@ -86,7 +86,8 @@ func equalIgnoringValueAtPath(a, b any, path []string) bool {
 	aMap, aOk := a.(map[string]any)
 	bMap, bOk := b.(map[string]any)
 	if !aOk || !bOk {
-		return !avoidTimestampEqualities.DeepEqual(a, b)
+		// Can't traverse into non-maps, ignore
+		return true
 	}
 	if len(aMap) != len(bMap) {
 		return false
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/equality_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/equality_test.go
index 04c328f641a..68b786f85b6 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/equality_test.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/equality_test.go
@@ -1,5 +1,5 @@
 /*
-Copyright 20214The Kubernetes Authors.
+Copyright 2024 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -91,7 +91,7 @@ func TestEqualIgnoringFieldValueAtPath(t *testing.T) {
 			want: false,
 		},
 		{
-			name: "extra spec fields in object a",
+			name: "extra spec field",
 			a: map[string]any{
 				"metadata": map[string]any{
 					"labels":        map[string]any{"env": "dev"},
@@ -114,7 +114,7 @@ func TestEqualIgnoringFieldValueAtPath(t *testing.T) {
 			want: false,
 		},
 		{
-			name: "different spec field in object b",
+			name: "different spec field",
 			a: map[string]any{
 				"metadata": map[string]any{
 					"labels":        map[string]any{"env": "dev"},
@@ -158,6 +158,26 @@ func TestEqualIgnoringFieldValueAtPath(t *testing.T) {
 			},
 			want: true,
 		},
+		{
+			name: "wrong type for metadata result in differences being ignored",
+			a: map[string]any{
+				"metadata": []any{
+					"something",
+				},
+				"spec": map[string]any{
+					"field": "value",
+				},
+			},
+			b: map[string]any{
+				"metadata": []any{
+					"something else",
+				},
+				"spec": map[string]any{
+					"field": "value",
+				},
+			},
+			want: true,
+		},
 	}
 
 	path := []string{"metadata", "managedFields"}
@@ -168,6 +188,11 @@ func TestEqualIgnoringFieldValueAtPath(t *testing.T) {
 			if actual != c.want {
 				t.Error("Expected equality check to return ", c.want, ", but got ", actual)
 			}
+			// Check that equality is commutative
+			actual = equalIgnoringValueAtPath(c.b, c.a, path)
+			if actual != c.want {
+				t.Error("Expected equality check to return ", c.want, ", but got ", actual)
+			}
 		})
 	}
 }
diff --git a/test/integration/apiserver/apply/apply_crd_test.go b/test/integration/apiserver/apply/apply_crd_test.go
index 6ecaa9febcb..4ac048e7974 100644
--- a/test/integration/apiserver/apply/apply_crd_test.go
+++ b/test/integration/apiserver/apply/apply_crd_test.go
@@ -737,6 +737,7 @@ spec:
 }
 
 func TestNoOpApplyWithDefaultsSameResourceVersionCRD(t *testing.T) {
+	// https://github.com/kubernetes/kubernetes/issues/124605
 	server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), nil, framework.SharedEtcd())
 	if err != nil {
 		t.Fatal(err)