diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go
index d3e303f3feb..4b093e70db0 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go
@@ -72,13 +72,6 @@ func (f *FieldManager) Update(liveObj, newObj runtime.Object, manager string) (r
 			return nil, fmt.Errorf("failed to decode managed fields: %v", err)
 		}
 	}
-	if err := internal.RemoveObjectManagedFields(newObj); err != nil {
-		return nil, fmt.Errorf("failed to remove managed fields from new obj: %v", err)
-	}
-	if err := internal.RemoveObjectManagedFields(liveObj); err != nil {
-		return nil, fmt.Errorf("failed to remove managed fields from live obj: %v", err)
-	}
-
 	newObjVersioned, err := f.toVersioned(newObj)
 	if err != nil {
 		return nil, fmt.Errorf("failed to convert new object to proper version: %v", err)
@@ -87,6 +80,12 @@ func (f *FieldManager) Update(liveObj, newObj runtime.Object, manager string) (r
 	if err != nil {
 		return nil, fmt.Errorf("failed to convert live object to proper version: %v", err)
 	}
+	if err := internal.RemoveObjectManagedFields(liveObjVersioned); err != nil {
+		return nil, fmt.Errorf("failed to remove managed fields from live obj: %v", err)
+	}
+	if err := internal.RemoveObjectManagedFields(newObjVersioned); err != nil {
+		return nil, fmt.Errorf("failed to remove managed fields from new obj: %v", err)
+	}
 
 	newObjTyped, err := f.typeConverter.ObjectToTyped(newObjVersioned)
 	if err != nil {
@@ -116,16 +115,15 @@ func (f *FieldManager) Apply(liveObj runtime.Object, patch []byte, force bool) (
 	if err != nil {
 		return nil, fmt.Errorf("failed to decode managed fields: %v", err)
 	}
-	if err := internal.RemoveObjectManagedFields(liveObj); err != nil {
-		return nil, fmt.Errorf("failed to remove managed fields from live obj: %v", err)
-	}
-
 	// We can assume that patchObj is already on the proper version:
 	// it shouldn't have to be converted so that it's not defaulted.
 	liveObjVersioned, err := f.toVersioned(liveObj)
 	if err != nil {
 		return nil, fmt.Errorf("failed to convert live object to proper version: %v", err)
 	}
+	if err := internal.RemoveObjectManagedFields(liveObjVersioned); err != nil {
+		return nil, fmt.Errorf("failed to remove managed fields from live obj: %v", err)
+	}
 
 	patchObjTyped, err := f.typeConverter.YAMLToTyped(patch)
 	if err != nil {