diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go
index 1c47416f9a0..6b58cf6b3a7 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go
@@ -60,6 +60,17 @@ func NewFieldManager(models openapiproto.Models, objectConverter runtime.ObjectC
 // use-case), and simply updates the managed fields in the output
 // object.
 func (f *FieldManager) Update(liveObj, newObj runtime.Object, manager string) (runtime.Object, error) {
+	managed, err := internal.DecodeObjectManagedFields(newObj)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode managed fields: %v", err)
+	}
+	if err := internal.RemoveObjectManagedFields(newObj); err != nil {
+		return nil, fmt.Errorf("failed to remove managed fields from new obj: %v", err)
+	}
+	if err := internal.RemoveObjectManagedFields(liveObj); err != nil {
+		return nil, fmt.Errorf("failed to remove managed fields from live obj: %v", err)
+	}
+
 	newObjVersioned, err := f.toVersioned(newObj)
 	if err != nil {
 		return nil, fmt.Errorf("failed to convert new object to proper version: %v", err)
@@ -69,11 +80,6 @@ func (f *FieldManager) Update(liveObj, newObj runtime.Object, manager string) (r
 		return nil, fmt.Errorf("failed to convert live object to proper version: %v", err)
 	}
 
-	managed, err := internal.DecodeObjectManagedFields(newObj)
-	if err != nil {
-		return nil, fmt.Errorf("failed to decode managed fields: %v", err)
-	}
-
 	newObjTyped, err := f.typeConverter.ObjectToTyped(newObjVersioned)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create typed new object: %v", err)
@@ -98,6 +104,14 @@ func (f *FieldManager) Update(liveObj, newObj runtime.Object, manager string) (r
 // Apply is used when server-side apply is called, as it merges the
 // object and update the managed fields.
 func (f *FieldManager) Apply(liveObj runtime.Object, patch []byte, force bool) (runtime.Object, error) {
+	managed, err := internal.DecodeObjectManagedFields(liveObj)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode managed fields: %v", err)
+	}
+	if err := internal.RemoveObjectManagedFields(liveObj); err != nil {
+		return nil, fmt.Errorf("failed to remove managed fields from live obj: %v", err)
+	}
+
 	// We can assume that patchObj is already on the proper version:
 	// it shouldn't have to be converted so that it's not defaulted.
 	liveObjVersioned, err := f.toVersioned(liveObj)
@@ -105,11 +119,6 @@ func (f *FieldManager) Apply(liveObj runtime.Object, patch []byte, force bool) (
 		return nil, fmt.Errorf("failed to convert live object to proper version: %v", err)
 	}
 
-	managed, err := internal.DecodeObjectManagedFields(liveObj)
-	if err != nil {
-		return nil, fmt.Errorf("failed to decode managed fields: %v", err)
-	}
-
 	patchObjTyped, err := f.typeConverter.YAMLToTyped(patch)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create typed patch object: %v", err)
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/managedfields.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/managedfields.go
index 3318d971f8b..c9b5da0c71a 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/managedfields.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/managedfields.go
@@ -22,10 +22,21 @@ import (
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
-
 	"sigs.k8s.io/structured-merge-diff/fieldpath"
 )
 
+// RemoveObjectManagedFields removes the ManagedFields from the object
+// before we merge so that it doesn't appear in the ManagedFields
+// recursively.
+func RemoveObjectManagedFields(obj runtime.Object) error {
+	accessor, err := meta.Accessor(obj)
+	if err != nil {
+		return fmt.Errorf("couldn't get accessor: %v", err)
+	}
+	accessor.SetManagedFields(nil)
+	return nil
+}
+
 // DecodeObjectManagedFields extracts and converts the objects ManagedFields into a fieldpath.ManagedFields.
 func DecodeObjectManagedFields(from runtime.Object) (fieldpath.ManagedFields, error) {
 	if from == nil {