diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go
index 239ae161940..e385a4f5417 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go
@@ -149,11 +149,9 @@ func (f *FieldManager) Update(liveObj, newObj runtime.Object, manager string) (r
 // Apply is used when server-side apply is called, as it merges the
 // object and update the managed fields.
 func (f *FieldManager) Apply(liveObj runtime.Object, patch []byte, fieldManager string, force bool) (runtime.Object, error) {
-	// If the object doesn't have metadata or managed fields is not empty, apply isn't allowed.
-	if objMeta, err := meta.Accessor(liveObj); err != nil {
+	// If the object doesn't have metadata, apply isn't allowed.
+	if _, err := meta.Accessor(liveObj); err != nil {
 		return nil, fmt.Errorf("couldn't get accessor: %v", err)
-	} else if objMeta.GetManagedFields() != nil && len(objMeta.GetManagedFields()) != 0 {
-		return nil, fmt.Errorf("apply is not allowed with managed fields set but was: %v", objMeta.GetManagedFields())
 	}
 
 	managed, err := internal.DecodeObjectManagedFields(liveObj)
@@ -166,6 +164,11 @@ func (f *FieldManager) Apply(liveObj runtime.Object, patch []byte, fieldManager
 	if err := yaml.Unmarshal(patch, &patchObj.Object); err != nil {
 		return nil, fmt.Errorf("error decoding YAML: %v", err)
 	}
+
+	if patchObj.GetManagedFields() != nil {
+		return nil, fmt.Errorf("managed fields must be nil but was %v", patchObj.GetManagedFields())
+	}
+
 	if patchObj.GetAPIVersion() != f.groupVersion.String() {
 		return nil,
 			errors.NewBadRequest(
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager_test.go
index 6698792e354..cdabd502c61 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager_test.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager_test.go
@@ -25,7 +25,6 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -444,23 +443,15 @@ func BenchmarkRepeatedUpdate(b *testing.B) {
 func TestApplyFailsWithManagedFields(t *testing.T) {
 	f := NewTestFieldManager()
 
-	obj := &corev1.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			ManagedFields: []metav1.ManagedFieldsEntry{
-				{
-					Manager: "test",
-				},
-			},
-		},
-	}
-
-	_, err := f.Apply(obj, []byte(`{
+	_, err := f.Apply(&corev1.Pod{}, []byte(`{
 		"apiVersion": "apps/v1",
 		"kind": "Pod",
 		"metadata": {
-			"labels": {
-				"a": "b"
-			},
+			"managedFields": [
+				{
+				  "manager": "test",
+				}
+			]
 		}
 	}`), "fieldmanager_test", false)
 
@@ -469,36 +460,10 @@ func TestApplyFailsWithManagedFields(t *testing.T) {
 	}
 }
 
-func TestApplySuccessWithEmptyManagedFields(t *testing.T) {
+func TestApplySuccessWithNoManagedFields(t *testing.T) {
 	f := NewTestFieldManager()
 
-	obj := &corev1.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			ManagedFields: []metav1.ManagedFieldsEntry{},
-		},
-	}
-
-	_, err := f.Apply(obj, []byte(`{
-		"apiVersion": "apps/v1",
-		"kind": "Pod",
-		"metadata": {
-			"labels": {
-				"a": "b"
-			},
-		}
-	}`), "fieldmanager_test", false)
-
-	if err != nil {
-		t.Fatalf("failed to apply object: %v", err)
-	}
-}
-
-func TestApplySuccessWithNilManagedFields(t *testing.T) {
-	f := NewTestFieldManager()
-
-	obj := &corev1.Pod{}
-
-	_, err := f.Apply(obj, []byte(`{
+	_, err := f.Apply(&corev1.Pod{}, []byte(`{
 		"apiVersion": "apps/v1",
 		"kind": "Pod",
 		"metadata": {