diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go
index d6d21d450d1..239ae161940 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go
@@ -149,9 +149,11 @@ func (f *FieldManager) Update(liveObj, newObj runtime.Object, manager string) (r
 // Apply is used when server-side apply is called, as it merges the
 // object and update the managed fields.
 func (f *FieldManager) Apply(liveObj runtime.Object, patch []byte, fieldManager string, force bool) (runtime.Object, error) {
-	// If the object doesn't have metadata, apply isn't allowed.
-	if _, err := meta.Accessor(liveObj); err != nil {
+	// If the object doesn't have metadata or managed fields is not empty, apply isn't allowed.
+	if objMeta, err := meta.Accessor(liveObj); err != nil {
 		return nil, fmt.Errorf("couldn't get accessor: %v", err)
+	} else if objMeta.GetManagedFields() != nil && len(objMeta.GetManagedFields()) != 0 {
+		return nil, fmt.Errorf("apply is not allowed with managed fields set but was: %v", objMeta.GetManagedFields())
 	}
 
 	managed, err := internal.DecodeObjectManagedFields(liveObj)
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager_test.go
index bb347bee825..6698792e354 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager_test.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager_test.go
@@ -25,6 +25,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -439,3 +440,75 @@ func BenchmarkRepeatedUpdate(b *testing.B) {
 		}
 	}
 }
+
+func TestApplyFailsWithManagedFields(t *testing.T) {
+	f := NewTestFieldManager()
+
+	obj := &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			ManagedFields: []metav1.ManagedFieldsEntry{
+				{
+					Manager: "test",
+				},
+			},
+		},
+	}
+
+	_, err := f.Apply(obj, []byte(`{
+		"apiVersion": "apps/v1",
+		"kind": "Pod",
+		"metadata": {
+			"labels": {
+				"a": "b"
+			},
+		}
+	}`), "fieldmanager_test", false)
+
+	if err == nil {
+		t.Fatalf("successfully applied with set managed fields")
+	}
+}
+
+func TestApplySuccessWithEmptyManagedFields(t *testing.T) {
+	f := NewTestFieldManager()
+
+	obj := &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			ManagedFields: []metav1.ManagedFieldsEntry{},
+		},
+	}
+
+	_, err := f.Apply(obj, []byte(`{
+		"apiVersion": "apps/v1",
+		"kind": "Pod",
+		"metadata": {
+			"labels": {
+				"a": "b"
+			},
+		}
+	}`), "fieldmanager_test", false)
+
+	if err != nil {
+		t.Fatalf("failed to apply object: %v", err)
+	}
+}
+
+func TestApplySuccessWithNilManagedFields(t *testing.T) {
+	f := NewTestFieldManager()
+
+	obj := &corev1.Pod{}
+
+	_, err := f.Apply(obj, []byte(`{
+		"apiVersion": "apps/v1",
+		"kind": "Pod",
+		"metadata": {
+			"labels": {
+				"a": "b"
+			},
+		}
+	}`), "fieldmanager_test", false)
+
+	if err != nil {
+		t.Fatalf("failed to apply object: %v", err)
+	}
+}