diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go
index 298b3afa7e1..c6447e62cbd 100644
--- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go
+++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go
@@ -679,7 +679,7 @@ func (r *crdHandler) getOrCreateServingInfoFor(uid types.UID, name string) (*crd
 		openAPIModels = nil
 	}
 
-	var typeConverter fieldmanager.TypeConverter = fieldmanager.DeducedTypeConverter{}
+	var typeConverter fieldmanager.TypeConverter = fieldmanager.NewDeducedTypeConverter()
 	if openAPIModels != nil {
 		typeConverter, err = fieldmanager.NewTypeConverter(openAPIModels, crd.Spec.PreserveUnknownFields)
 		if err != nil {
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/admission.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/admission.go
index 26d264fe833..5521f312851 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/admission.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/admission.go
@@ -22,6 +22,7 @@ import (
 
 	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apiserver/pkg/admission"
+	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
 	"k8s.io/apiserver/pkg/warning"
 )
 
@@ -70,7 +71,7 @@ func (admit *managedFieldsValidatingAdmissionController) Admit(ctx context.Conte
 		return err
 	}
 	managedFieldsAfterAdmission := objectMeta.GetManagedFields()
-	if _, err := DecodeManagedFields(managedFieldsAfterAdmission); err != nil {
+	if _, err := internal.DecodeManagedFields(managedFieldsAfterAdmission); err != nil {
 		objectMeta.SetManagedFields(managedFieldsBeforeAdmission)
 		warning.AddWarning(ctx, "",
 			fmt.Sprintf(InvalidManagedFieldsAfterMutatingAdmissionWarningFormat,
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go
index 6c3d2ce8326..668f7688261 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go
@@ -18,247 +18,40 @@ package fieldmanager
 
 import (
 	"fmt"
-	"reflect"
-	"time"
 
-	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
-	"k8s.io/klog/v2"
 	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
-	"sigs.k8s.io/structured-merge-diff/v4/merge"
 )
 
-// DefaultMaxUpdateManagers defines the default maximum retained number of managedFields entries from updates
-// if the number of update managers exceeds this, the oldest entries will be merged until the number is below the maximum.
-// TODO(jennybuckley): Determine if this is really the best value. Ideally we wouldn't unnecessarily merge too many entries.
-const DefaultMaxUpdateManagers int = 10
-
-// DefaultTrackOnCreateProbability defines the default probability that the field management of an object
-// starts being tracked from the object's creation, instead of from the first time the object is applied to.
-const DefaultTrackOnCreateProbability float32 = 1
-
-var atMostEverySecond = internal.NewAtMostEvery(time.Second)
-
-// Managed groups a fieldpath.ManagedFields together with the timestamps associated with each operation.
-type Managed interface {
-	// Fields gets the fieldpath.ManagedFields.
-	Fields() fieldpath.ManagedFields
-
-	// Times gets the timestamps associated with each operation.
-	Times() map[string]*metav1.Time
-}
-
-// Manager updates the managed fields and merges applied configurations.
-type Manager interface {
-	// Update is used when the object has already been merged (non-apply
-	// use-case), and simply updates the managed fields in the output
-	// object.
-	//  * `liveObj` is not mutated by this function
-	//  * `newObj` may be mutated by this function
-	// Returns the new object with managedFields removed, and the object's new
-	// proposed managedFields separately.
-	Update(liveObj, newObj runtime.Object, managed Managed, manager string) (runtime.Object, Managed, error)
-
-	// Apply is used when server-side apply is called, as it merges the
-	// object and updates the managed fields.
-	//  * `liveObj` is not mutated by this function
-	//  * `newObj` may be mutated by this function
-	// Returns the new object with managedFields removed, and the object's new
-	// proposed managedFields separately.
-	Apply(liveObj, appliedObj runtime.Object, managed Managed, fieldManager string, force bool) (runtime.Object, Managed, error)
-}
-
-// FieldManager updates the managed fields and merge applied
+// FieldManager updates the managed fields and merges applied
 // configurations.
-type FieldManager struct {
-	fieldManager Manager
-	subresource  string
-}
-
-// NewFieldManager creates a new FieldManager that decodes, manages, then re-encodes managedFields
-// on update and apply requests.
-func NewFieldManager(f Manager, subresource string) *FieldManager {
-	return &FieldManager{fieldManager: f, subresource: subresource}
-}
+type FieldManager = internal.FieldManager
 
 // NewDefaultFieldManager creates a new FieldManager that merges apply requests
 // and update managed fields for other types of requests.
 func NewDefaultFieldManager(typeConverter TypeConverter, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind, hub schema.GroupVersion, subresource string, resetFields map[fieldpath.APIVersion]*fieldpath.Set) (*FieldManager, error) {
-	f, err := NewStructuredMergeManager(typeConverter, objectConverter, objectDefaulter, kind.GroupVersion(), hub, resetFields)
+	f, err := internal.NewStructuredMergeManager(typeConverter, objectConverter, objectDefaulter, kind.GroupVersion(), hub, resetFields)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create field manager: %v", err)
 	}
-	return newDefaultFieldManager(f, typeConverter, objectConverter, objectCreater, kind, subresource), nil
+	return internal.NewDefaultFieldManager(f, typeConverter, objectConverter, objectCreater, kind, subresource), nil
 }
 
 // NewDefaultCRDFieldManager creates a new FieldManager specifically for
 // CRDs. This allows for the possibility of fields which are not defined
 // in models, as well as having no models defined at all.
 func NewDefaultCRDFieldManager(typeConverter TypeConverter, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind, hub schema.GroupVersion, subresource string, resetFields map[fieldpath.APIVersion]*fieldpath.Set) (_ *FieldManager, err error) {
-	f, err := NewCRDStructuredMergeManager(typeConverter, objectConverter, objectDefaulter, kind.GroupVersion(), hub, resetFields)
+	f, err := internal.NewCRDStructuredMergeManager(typeConverter, objectConverter, objectDefaulter, kind.GroupVersion(), hub, resetFields)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create field manager: %v", err)
 	}
-	return newDefaultFieldManager(f, typeConverter, objectConverter, objectCreater, kind, subresource), nil
+	return internal.NewDefaultFieldManager(f, typeConverter, objectConverter, objectCreater, kind, subresource), nil
 }
 
-// newDefaultFieldManager is a helper function which wraps a Manager with certain default logic.
-func newDefaultFieldManager(f Manager, typeConverter TypeConverter, objectConverter runtime.ObjectConvertor, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind, subresource string) *FieldManager {
-	return NewFieldManager(
-		NewLastAppliedUpdater(
-			NewLastAppliedManager(
-				NewProbabilisticSkipNonAppliedManager(
-					NewCapManagersManager(
-						NewBuildManagerInfoManager(
-							NewManagedFieldsUpdater(
-								NewStripMetaManager(f),
-							), kind.GroupVersion(), subresource,
-						), DefaultMaxUpdateManagers,
-					), objectCreater, kind, DefaultTrackOnCreateProbability,
-				), typeConverter, objectConverter, kind.GroupVersion()),
-		), subresource,
-	)
-}
-
-// DecodeManagedFields converts ManagedFields from the wire format (api format)
-// to the format used by sigs.k8s.io/structured-merge-diff
-func DecodeManagedFields(encodedManagedFields []metav1.ManagedFieldsEntry) (Managed, error) {
-	return internal.DecodeManagedFields(encodedManagedFields)
-}
-
-func decodeLiveOrNew(liveObj, newObj runtime.Object, ignoreManagedFieldsFromRequestObject bool) (Managed, error) {
-	liveAccessor, err := meta.Accessor(liveObj)
-	if err != nil {
-		return nil, err
-	}
-
-	// We take the managedFields of the live object in case the request tries to
-	// manually set managedFields via a subresource.
-	if ignoreManagedFieldsFromRequestObject {
-		return emptyManagedFieldsOnErr(DecodeManagedFields(liveAccessor.GetManagedFields()))
-	}
-
-	// If the object doesn't have metadata, we should just return without trying to
-	// set the managedFields at all, so creates/updates/patches will work normally.
-	newAccessor, err := meta.Accessor(newObj)
-	if err != nil {
-		return nil, err
-	}
-
-	if isResetManagedFields(newAccessor.GetManagedFields()) {
-		return internal.NewEmptyManaged(), nil
-	}
-
-	// If the managed field is empty or we failed to decode it,
-	// let's try the live object. This is to prevent clients who
-	// don't understand managedFields from deleting it accidentally.
-	managed, err := DecodeManagedFields(newAccessor.GetManagedFields())
-	if err != nil || len(managed.Fields()) == 0 {
-		return emptyManagedFieldsOnErr(DecodeManagedFields(liveAccessor.GetManagedFields()))
-	}
-	return managed, nil
-}
-
-func emptyManagedFieldsOnErr(managed Managed, err error) (Managed, error) {
-	if err != nil {
-		return internal.NewEmptyManaged(), nil
-	}
-	return managed, nil
-}
-
-// Update is used when the object has already been merged (non-apply
-// use-case), and simply updates the managed fields in the output
-// object.
-func (f *FieldManager) Update(liveObj, newObj runtime.Object, manager string) (object runtime.Object, err error) {
-	// First try to decode the managed fields provided in the update,
-	// This is necessary to allow directly updating managed fields.
-	isSubresource := f.subresource != ""
-	managed, err := decodeLiveOrNew(liveObj, newObj, isSubresource)
-	if err != nil {
-		return newObj, nil
-	}
-
-	internal.RemoveObjectManagedFields(newObj)
-
-	if object, managed, err = f.fieldManager.Update(liveObj, newObj, managed, manager); err != nil {
-		return nil, err
-	}
-
-	if err = internal.EncodeObjectManagedFields(object, managed); err != nil {
-		return nil, fmt.Errorf("failed to encode managed fields: %v", err)
-	}
-
-	return object, nil
-}
-
-// UpdateNoErrors is the same as Update, but it will not return
-// errors. If an error happens, the object is returned with
-// managedFields cleared.
-func (f *FieldManager) UpdateNoErrors(liveObj, newObj runtime.Object, manager string) runtime.Object {
-	obj, err := f.Update(liveObj, newObj, manager)
-	if err != nil {
-		atMostEverySecond.Do(func() {
-			ns, name := "unknown", "unknown"
-			if accessor, err := meta.Accessor(newObj); err == nil {
-				ns = accessor.GetNamespace()
-				name = accessor.GetName()
-			}
-
-			klog.ErrorS(err, "[SHOULD NOT HAPPEN] failed to update managedFields", "VersionKind",
-				newObj.GetObjectKind().GroupVersionKind(), "namespace", ns, "name", name)
-		})
-		// Explicitly remove managedFields on failure, so that
-		// we can't have garbage in it.
-		internal.RemoveObjectManagedFields(newObj)
-		return newObj
-	}
-	return obj
-}
-
-// Returns true if the managedFields indicate that the user is trying to
-// reset the managedFields, i.e. if the list is non-nil but empty, or if
-// the list has one empty item.
-func isResetManagedFields(managedFields []metav1.ManagedFieldsEntry) bool {
-	if len(managedFields) == 0 {
-		return managedFields != nil
-	}
-
-	if len(managedFields) == 1 {
-		return reflect.DeepEqual(managedFields[0], metav1.ManagedFieldsEntry{})
-	}
-
-	return false
-}
-
-// Apply is used when server-side apply is called, as it merges the
-// object and updates the managed fields.
-func (f *FieldManager) Apply(liveObj, appliedObj runtime.Object, manager string, force bool) (object runtime.Object, err error) {
-	// If the object doesn't have metadata, apply isn't allowed.
-	accessor, err := meta.Accessor(liveObj)
-	if err != nil {
-		return nil, fmt.Errorf("couldn't get accessor: %v", err)
-	}
-
-	// Decode the managed fields in the live object, since it isn't allowed in the patch.
-	managed, err := DecodeManagedFields(accessor.GetManagedFields())
-	if err != nil {
-		return nil, fmt.Errorf("failed to decode managed fields: %v", err)
-	}
-
-	object, managed, err = f.fieldManager.Apply(liveObj, appliedObj, managed, manager, force)
-	if err != nil {
-		if conflicts, ok := err.(merge.Conflicts); ok {
-			return nil, internal.NewConflictError(conflicts)
-		}
-		return nil, err
-	}
-
-	if err = internal.EncodeObjectManagedFields(object, managed); err != nil {
-		return nil, fmt.Errorf("failed to encode managed fields: %v", err)
-	}
-
-	return object, nil
+func ValidateManagedFields(encodedManagedFields []metav1.ManagedFieldsEntry) error {
+	_, err := internal.DecodeManagedFields(encodedManagedFields)
+	return err
 }
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanagertest/testfieldmanager.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanagertest/testfieldmanager.go
index 53d0154f01d..71eb9e6ee32 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanagertest/testfieldmanager.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanagertest/testfieldmanager.go
@@ -26,6 +26,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
+	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
 	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
 	"sigs.k8s.io/structured-merge-diff/v4/merge"
 	"sigs.k8s.io/structured-merge-diff/v4/typed"
@@ -95,7 +96,7 @@ func (f *fakeObjectCreater) New(gvk schema.GroupVersionKind) (runtime.Object, er
 // to specify either a sub-resource, or a set of modified Manager to
 // test them specifically.
 type TestFieldManager struct {
-	fieldManager *fieldmanager.FieldManager
+	fieldManager *internal.FieldManager
 	apiVersion   string
 	emptyObj     runtime.Object
 	liveObj      runtime.Object
@@ -108,10 +109,10 @@ func NewDefaultTestFieldManager(typeConverter fieldmanager.TypeConverter, gvk sc
 }
 
 // NewTestFieldManager creates a new manager for the given GVK.
-func NewTestFieldManager(typeConverter fieldmanager.TypeConverter, gvk schema.GroupVersionKind, subresource string, chainFieldManager func(fieldmanager.Manager) fieldmanager.Manager) TestFieldManager {
+func NewTestFieldManager(typeConverter fieldmanager.TypeConverter, gvk schema.GroupVersionKind, subresource string, chainFieldManager func(internal.Manager) internal.Manager) TestFieldManager {
 	apiVersion := fieldpath.APIVersion(gvk.GroupVersion().String())
 	objectConverter := &fakeObjectConvertor{sameVersionConverter{}, apiVersion}
-	f, err := fieldmanager.NewStructuredMergeManager(
+	f, err := internal.NewStructuredMergeManager(
 		typeConverter,
 		objectConverter,
 		&fakeObjectDefaulter{},
@@ -125,14 +126,18 @@ func NewTestFieldManager(typeConverter fieldmanager.TypeConverter, gvk schema.Gr
 	live := &unstructured.Unstructured{}
 	live.SetKind(gvk.Kind)
 	live.SetAPIVersion(gvk.GroupVersion().String())
-	f = fieldmanager.NewLastAppliedUpdater(
-		fieldmanager.NewLastAppliedManager(
-			fieldmanager.NewProbabilisticSkipNonAppliedManager(
-				fieldmanager.NewBuildManagerInfoManager(
-					fieldmanager.NewManagedFieldsUpdater(
-						fieldmanager.NewStripMetaManager(f),
+	// This is different from `internal.NewDefaultFieldManager` because:
+	// 1. We don't want to create a `internal.FieldManager`
+	// 2. We don't want to use the CapManager that is tested separately with
+	// a smaller than the default cap.
+	f = internal.NewLastAppliedUpdater(
+		internal.NewLastAppliedManager(
+			internal.NewProbabilisticSkipNonAppliedManager(
+				internal.NewBuildManagerInfoManager(
+					internal.NewManagedFieldsUpdater(
+						internal.NewStripMetaManager(f),
 					), gvk.GroupVersion(), subresource,
-				), NewFakeObjectCreater(), gvk, fieldmanager.DefaultTrackOnCreateProbability,
+				), NewFakeObjectCreater(), gvk, internal.DefaultTrackOnCreateProbability,
 			), typeConverter, objectConverter, gvk.GroupVersion(),
 		),
 	)
@@ -140,7 +145,7 @@ func NewTestFieldManager(typeConverter fieldmanager.TypeConverter, gvk schema.Gr
 		f = chainFieldManager(f)
 	}
 	return TestFieldManager{
-		fieldManager: fieldmanager.NewFieldManager(f, subresource),
+		fieldManager: internal.NewFieldManager(f, subresource),
 		apiVersion:   gvk.GroupVersion().String(),
 		emptyObj:     live,
 		liveObj:      live.DeepCopyObject(),
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/buildmanagerinfo.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/buildmanagerinfo.go
similarity index 94%
rename from staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/buildmanagerinfo.go
rename to staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/buildmanagerinfo.go
index 58b87eb3886..fa342ca1351 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/buildmanagerinfo.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/buildmanagerinfo.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package fieldmanager
+package internal
 
 import (
 	"fmt"
@@ -22,7 +22,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
 )
 
 type buildManagerInfoManager struct {
@@ -71,5 +70,5 @@ func (f *buildManagerInfoManager) buildManagerInfo(prefix string, operation meta
 	if managerInfo.Manager == "" {
 		managerInfo.Manager = "unknown"
 	}
-	return internal.BuildManagerIdentifier(&managerInfo)
+	return BuildManagerIdentifier(&managerInfo)
 }
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/capmanagers.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/capmanagers.go
similarity index 96%
rename from staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/capmanagers.go
rename to staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/capmanagers.go
index c3184e24154..8951932ba42 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/capmanagers.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/capmanagers.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package fieldmanager
+package internal
 
 import (
 	"fmt"
@@ -22,7 +22,6 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
 	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
 )
 
@@ -100,7 +99,7 @@ func (f *capManagersManager) capUpdateManagers(managed Managed) (newManaged Mana
 
 		// Create a new manager identifier for the versioned bucket entry.
 		// The version for this manager comes from the version of the update being merged into the bucket.
-		bucket, err := internal.BuildManagerIdentifier(&metav1.ManagedFieldsEntry{
+		bucket, err := BuildManagerIdentifier(&metav1.ManagedFieldsEntry{
 			Manager:    f.oldUpdatesManagerName,
 			Operation:  metav1.ManagedFieldsOperationUpdate,
 			APIVersion: version,
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/capmanagers_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/capmanagers_test.go
similarity index 94%
rename from staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/capmanagers_test.go
rename to staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/capmanagers_test.go
index b0ee5a0259d..b98eb37dab9 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/capmanagers_test.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/capmanagers_test.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package fieldmanager_test
+package internal_test
 
 import (
 	"bytes"
@@ -29,20 +29,20 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
 	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanagertest"
+	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
 	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
 )
 
 type fakeManager struct{}
 
-var _ fieldmanager.Manager = &fakeManager{}
+var _ internal.Manager = &fakeManager{}
 
-func (*fakeManager) Update(_, newObj runtime.Object, managed fieldmanager.Managed, _ string) (runtime.Object, fieldmanager.Managed, error) {
+func (*fakeManager) Update(_, newObj runtime.Object, managed internal.Managed, _ string) (runtime.Object, internal.Managed, error) {
 	return newObj, managed, nil
 }
 
-func (*fakeManager) Apply(_, _ runtime.Object, _ fieldmanager.Managed, _ string, _ bool) (runtime.Object, fieldmanager.Managed, error) {
+func (*fakeManager) Apply(_, _ runtime.Object, _ internal.Managed, _ string, _ bool) (runtime.Object, internal.Managed, error) {
 	panic("not implemented")
 	return nil, nil, nil
 }
@@ -50,8 +50,8 @@ func (*fakeManager) Apply(_, _ runtime.Object, _ fieldmanager.Managed, _ string,
 func TestCapManagersManagerMergesEntries(t *testing.T) {
 	f := fieldmanagertest.NewTestFieldManager(fakeTypeConverter, schema.FromAPIVersionAndKind("v1", "Pod"),
 		"",
-		func(m fieldmanager.Manager) fieldmanager.Manager {
-			return fieldmanager.NewCapManagersManager(m, 3)
+		func(m internal.Manager) internal.Manager {
+			return internal.NewCapManagersManager(m, 3)
 		})
 
 	podWithLabels := func(labels ...string) runtime.Object {
@@ -116,8 +116,8 @@ func TestCapManagersManagerMergesEntries(t *testing.T) {
 func TestCapUpdateManagers(t *testing.T) {
 	f := fieldmanagertest.NewTestFieldManager(fakeTypeConverter, schema.FromAPIVersionAndKind("v1", "Pod"),
 		"",
-		func(m fieldmanager.Manager) fieldmanager.Manager {
-			return fieldmanager.NewCapManagersManager(m, 3)
+		func(m internal.Manager) internal.Manager {
+			return internal.NewCapManagersManager(m, 3)
 		})
 
 	set := func(fields ...string) *metav1.FieldsV1 {
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/fieldmanager.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/fieldmanager.go
new file mode 100644
index 00000000000..74cf4cb4f6e
--- /dev/null
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/fieldmanager.go
@@ -0,0 +1,206 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package internal
+
+import (
+	"fmt"
+	"reflect"
+	"time"
+
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/structured-merge-diff/v4/merge"
+)
+
+// DefaultMaxUpdateManagers defines the default maximum retained number of managedFields entries from updates
+// if the number of update managers exceeds this, the oldest entries will be merged until the number is below the maximum.
+// TODO(jennybuckley): Determine if this is really the best value. Ideally we wouldn't unnecessarily merge too many entries.
+const DefaultMaxUpdateManagers int = 10
+
+// DefaultTrackOnCreateProbability defines the default probability that the field management of an object
+// starts being tracked from the object's creation, instead of from the first time the object is applied to.
+const DefaultTrackOnCreateProbability float32 = 1
+
+var atMostEverySecond = NewAtMostEvery(time.Second)
+
+// FieldManager updates the managed fields and merges applied
+// configurations.
+type FieldManager struct {
+	fieldManager Manager
+	subresource  string
+}
+
+// NewFieldManager creates a new FieldManager that decodes, manages, then re-encodes managedFields
+// on update and apply requests.
+func NewFieldManager(f Manager, subresource string) *FieldManager {
+	return &FieldManager{fieldManager: f, subresource: subresource}
+}
+
+// newDefaultFieldManager is a helper function which wraps a Manager with certain default logic.
+func NewDefaultFieldManager(f Manager, typeConverter TypeConverter, objectConverter runtime.ObjectConvertor, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind, subresource string) *FieldManager {
+	return NewFieldManager(
+		NewLastAppliedUpdater(
+			NewLastAppliedManager(
+				NewProbabilisticSkipNonAppliedManager(
+					NewCapManagersManager(
+						NewBuildManagerInfoManager(
+							NewManagedFieldsUpdater(
+								NewStripMetaManager(f),
+							), kind.GroupVersion(), subresource,
+						), DefaultMaxUpdateManagers,
+					), objectCreater, kind, DefaultTrackOnCreateProbability,
+				), typeConverter, objectConverter, kind.GroupVersion()),
+		), subresource,
+	)
+}
+
+func decodeLiveOrNew(liveObj, newObj runtime.Object, ignoreManagedFieldsFromRequestObject bool) (Managed, error) {
+	liveAccessor, err := meta.Accessor(liveObj)
+	if err != nil {
+		return nil, err
+	}
+
+	// We take the managedFields of the live object in case the request tries to
+	// manually set managedFields via a subresource.
+	if ignoreManagedFieldsFromRequestObject {
+		return emptyManagedFieldsOnErr(DecodeManagedFields(liveAccessor.GetManagedFields()))
+	}
+
+	// If the object doesn't have metadata, we should just return without trying to
+	// set the managedFields at all, so creates/updates/patches will work normally.
+	newAccessor, err := meta.Accessor(newObj)
+	if err != nil {
+		return nil, err
+	}
+
+	if isResetManagedFields(newAccessor.GetManagedFields()) {
+		return NewEmptyManaged(), nil
+	}
+
+	// If the managed field is empty or we failed to decode it,
+	// let's try the live object. This is to prevent clients who
+	// don't understand managedFields from deleting it accidentally.
+	managed, err := DecodeManagedFields(newAccessor.GetManagedFields())
+	if err != nil || len(managed.Fields()) == 0 {
+		return emptyManagedFieldsOnErr(DecodeManagedFields(liveAccessor.GetManagedFields()))
+	}
+	return managed, nil
+}
+
+func emptyManagedFieldsOnErr(managed Managed, err error) (Managed, error) {
+	if err != nil {
+		return NewEmptyManaged(), nil
+	}
+	return managed, nil
+}
+
+// Update is used when the object has already been merged (non-apply
+// use-case), and simply updates the managed fields in the output
+// object.
+func (f *FieldManager) Update(liveObj, newObj runtime.Object, manager string) (object runtime.Object, err error) {
+	// First try to decode the managed fields provided in the update,
+	// This is necessary to allow directly updating managed fields.
+	isSubresource := f.subresource != ""
+	managed, err := decodeLiveOrNew(liveObj, newObj, isSubresource)
+	if err != nil {
+		return newObj, nil
+	}
+
+	RemoveObjectManagedFields(newObj)
+
+	if object, managed, err = f.fieldManager.Update(liveObj, newObj, managed, manager); err != nil {
+		return nil, err
+	}
+
+	if err = EncodeObjectManagedFields(object, managed); err != nil {
+		return nil, fmt.Errorf("failed to encode managed fields: %v", err)
+	}
+
+	return object, nil
+}
+
+// UpdateNoErrors is the same as Update, but it will not return
+// errors. If an error happens, the object is returned with
+// managedFields cleared.
+func (f *FieldManager) UpdateNoErrors(liveObj, newObj runtime.Object, manager string) runtime.Object {
+	obj, err := f.Update(liveObj, newObj, manager)
+	if err != nil {
+		atMostEverySecond.Do(func() {
+			ns, name := "unknown", "unknown"
+			if accessor, err := meta.Accessor(newObj); err == nil {
+				ns = accessor.GetNamespace()
+				name = accessor.GetName()
+			}
+
+			klog.ErrorS(err, "[SHOULD NOT HAPPEN] failed to update managedFields", "VersionKind",
+				newObj.GetObjectKind().GroupVersionKind(), "namespace", ns, "name", name)
+		})
+		// Explicitly remove managedFields on failure, so that
+		// we can't have garbage in it.
+		RemoveObjectManagedFields(newObj)
+		return newObj
+	}
+	return obj
+}
+
+// Returns true if the managedFields indicate that the user is trying to
+// reset the managedFields, i.e. if the list is non-nil but empty, or if
+// the list has one empty item.
+func isResetManagedFields(managedFields []metav1.ManagedFieldsEntry) bool {
+	if len(managedFields) == 0 {
+		return managedFields != nil
+	}
+
+	if len(managedFields) == 1 {
+		return reflect.DeepEqual(managedFields[0], metav1.ManagedFieldsEntry{})
+	}
+
+	return false
+}
+
+// Apply is used when server-side apply is called, as it merges the
+// object and updates the managed fields.
+func (f *FieldManager) Apply(liveObj, appliedObj runtime.Object, manager string, force bool) (object runtime.Object, err error) {
+	// If the object doesn't have metadata, apply isn't allowed.
+	accessor, err := meta.Accessor(liveObj)
+	if err != nil {
+		return nil, fmt.Errorf("couldn't get accessor: %v", err)
+	}
+
+	// Decode the managed fields in the live object, since it isn't allowed in the patch.
+	managed, err := DecodeManagedFields(accessor.GetManagedFields())
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode managed fields: %v", err)
+	}
+
+	object, managed, err = f.fieldManager.Apply(liveObj, appliedObj, managed, manager, force)
+	if err != nil {
+		if conflicts, ok := err.(merge.Conflicts); ok {
+			return nil, NewConflictError(conflicts)
+		}
+		return nil, err
+	}
+
+	if err = EncodeObjectManagedFields(object, managed); err != nil {
+		return nil, fmt.Errorf("failed to encode managed fields: %v", err)
+	}
+
+	return object, nil
+}
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/fieldmanager_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/fieldmanager_test.go
new file mode 100644
index 00000000000..f8edf6913dc
--- /dev/null
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/fieldmanager_test.go
@@ -0,0 +1,61 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package internal_test
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"path/filepath"
+	"strings"
+
+	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
+	"k8s.io/kube-openapi/pkg/validation/spec"
+)
+
+var fakeTypeConverter = func() internal.TypeConverter {
+	data, err := ioutil.ReadFile(filepath.Join(
+		strings.Repeat(".."+string(filepath.Separator), 9),
+		"api", "openapi-spec", "swagger.json"))
+	if err != nil {
+		panic(err)
+	}
+	spec := spec.Swagger{}
+	if err := json.Unmarshal(data, &spec); err != nil {
+		panic(err)
+	}
+	typeConverter, err := internal.NewTypeConverter(&spec, false)
+	if err != nil {
+		panic(err)
+	}
+	return typeConverter
+}()
+
+var testTypeConverter = func() internal.TypeConverter {
+	data, err := ioutil.ReadFile(filepath.Join("testdata", "swagger.json"))
+	if err != nil {
+		panic(err)
+	}
+	spec := spec.Swagger{}
+	if err := json.Unmarshal(data, &spec); err != nil {
+		panic(err)
+	}
+	typeConverter, err := internal.NewTypeConverter(&spec, false)
+	if err != nil {
+		panic(err)
+	}
+	return typeConverter
+}()
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/lastappliedmanager.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/lastappliedmanager.go
similarity index 99%
rename from staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/lastappliedmanager.go
rename to staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/lastappliedmanager.go
index 4b07d462a22..88675630aae 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/lastappliedmanager.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/lastappliedmanager.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package fieldmanager
+package internal
 
 import (
 	"encoding/json"
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/lastappliedmanager_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/lastappliedmanager_test.go
similarity index 95%
rename from staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/lastappliedmanager_test.go
rename to staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/lastappliedmanager_test.go
index cc7cf935100..61f4055e9c7 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/lastappliedmanager_test.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/lastappliedmanager_test.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package fieldmanager_test
+package internal_test
 
 import (
 	"fmt"
@@ -24,7 +24,9 @@ import (
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	yamlutil "k8s.io/apimachinery/pkg/util/yaml"
 	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanagertest"
 	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
 	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
@@ -979,3 +981,27 @@ func testConflicts(t *testing.T, f fieldmanagertest.TestFieldManager, tests []te
 		})
 	}
 }
+
+func yamlToJSON(y []byte) (string, error) {
+	obj := &unstructured.Unstructured{Object: map[string]interface{}{}}
+	if err := yaml.Unmarshal(y, &obj.Object); err != nil {
+		return "", fmt.Errorf("error decoding YAML: %v", err)
+	}
+	serialization, err := runtime.Encode(unstructured.UnstructuredJSONScheme, obj)
+	if err != nil {
+		return "", fmt.Errorf("error encoding object: %v", err)
+	}
+	json, err := yamlutil.ToJSON(serialization)
+	if err != nil {
+		return "", fmt.Errorf("error converting to json: %v", err)
+	}
+	return string(json), nil
+}
+
+func setLastAppliedFromEncoded(obj runtime.Object, lastApplied []byte) error {
+	lastAppliedJSON, err := yamlToJSON(lastApplied)
+	if err != nil {
+		return err
+	}
+	return internal.SetLastApplied(obj, lastAppliedJSON)
+}
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/lastappliedupdater.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/lastappliedupdater.go
similarity index 95%
rename from staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/lastappliedupdater.go
rename to staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/lastappliedupdater.go
index e414e167f44..0d9320730a1 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/lastappliedupdater.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/lastappliedupdater.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package fieldmanager
+package internal
 
 import (
 	"fmt"
@@ -23,7 +23,6 @@ import (
 	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
 )
 
 type lastAppliedUpdater struct {
@@ -62,7 +61,7 @@ func (f *lastAppliedUpdater) Apply(liveObj, newObj runtime.Object, managed Manag
 		if err != nil {
 			return nil, nil, fmt.Errorf("failed to build last-applied annotation: %v", err)
 		}
-		err = internal.SetLastApplied(liveObj, lastAppliedValue)
+		err = SetLastApplied(liveObj, lastAppliedValue)
 		if err != nil {
 			return nil, nil, fmt.Errorf("failed to set last-applied annotation: %v", err)
 		}
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/lastappliedupdater_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/lastappliedupdater_test.go
similarity index 88%
rename from staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/lastappliedupdater_test.go
rename to staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/lastappliedupdater_test.go
index dc9e7ff2455..c89f03c3b72 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/lastappliedupdater_test.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/lastappliedupdater_test.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package fieldmanager_test
+package internal_test
 
 import (
 	"fmt"
@@ -25,17 +25,18 @@ import (
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
 	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanagertest"
+	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
 	"sigs.k8s.io/yaml"
 )
 
 func TestLastAppliedUpdater(t *testing.T) {
 	f := fieldmanagertest.NewTestFieldManager(fakeTypeConverter, schema.FromAPIVersionAndKind("apps/v1", "Deployment"),
 		"",
-		func(m fieldmanager.Manager) fieldmanager.Manager {
-			return fieldmanager.NewLastAppliedUpdater(m)
+		func(m internal.Manager) internal.Manager {
+			return internal.NewLastAppliedUpdater(m)
 		})
 
 	originalLastApplied := `nonempty`
@@ -191,8 +192,8 @@ func TestLargeLastApplied(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			f := fieldmanagertest.NewTestFieldManager(fakeTypeConverter, schema.FromAPIVersionAndKind("v1", "ConfigMap"),
 				"",
-				func(m fieldmanager.Manager) fieldmanager.Manager {
-					return fieldmanager.NewLastAppliedUpdater(m)
+				func(m internal.Manager) internal.Manager {
+					return internal.NewLastAppliedUpdater(m)
 				})
 
 			if err := f.Apply(test.oldObject, "kubectl", false); err != nil {
@@ -226,3 +227,20 @@ func TestLargeLastApplied(t *testing.T) {
 		})
 	}
 }
+
+func getLastApplied(obj runtime.Object) (string, error) {
+	accessor := meta.NewAccessor()
+	annotations, err := accessor.Annotations(obj)
+	if err != nil {
+		return "", fmt.Errorf("failed to access annotations: %v", err)
+	}
+	if annotations == nil {
+		return "", fmt.Errorf("no annotations on obj: %v", obj)
+	}
+
+	lastApplied, ok := annotations[corev1.LastAppliedConfigAnnotation]
+	if !ok {
+		return "", fmt.Errorf("expected last applied annotation, but got none for object: %v", obj)
+	}
+	return lastApplied, nil
+}
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/managedfieldsupdater.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/managedfieldsupdater.go
similarity index 95%
rename from staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/managedfieldsupdater.go
rename to staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/managedfieldsupdater.go
index 412443a6c4e..376eed6b207 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/managedfieldsupdater.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/managedfieldsupdater.go
@@ -14,14 +14,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package fieldmanager
+package internal
 
 import (
 	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
 	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
 )
 
@@ -77,7 +76,7 @@ func (f *managedFieldsUpdater) Apply(liveObj, appliedObj runtime.Object, managed
 		managed.Times()[fieldManager] = &metav1.Time{Time: time.Now().UTC()}
 	} else {
 		object = liveObj.DeepCopyObject()
-		internal.RemoveObjectManagedFields(object)
+		RemoveObjectManagedFields(object)
 	}
 	return object, managed, nil
 }
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/managedfieldsupdater_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/managedfieldsupdater_test.go
similarity index 96%
rename from staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/managedfieldsupdater_test.go
rename to staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/managedfieldsupdater_test.go
index 27b5a1f23cb..9d9683b23b7 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/managedfieldsupdater_test.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/managedfieldsupdater_test.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package fieldmanager_test
+package internal_test
 
 import (
 	"fmt"
@@ -28,7 +28,6 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
 	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanagertest"
 	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
 	"sigs.k8s.io/yaml"
@@ -428,11 +427,11 @@ func TestTakingOverManagedFieldsDuringApplyDoesNotModifyPreviousManagerTime(t *t
 
 type NoopManager struct{}
 
-func (NoopManager) Apply(liveObj, appliedObj runtime.Object, managed fieldmanager.Managed, fieldManager string, force bool) (runtime.Object, fieldmanager.Managed, error) {
+func (NoopManager) Apply(liveObj, appliedObj runtime.Object, managed internal.Managed, fieldManager string, force bool) (runtime.Object, internal.Managed, error) {
 	return nil, managed, nil
 }
 
-func (NoopManager) Update(liveObj, newObj runtime.Object, managed fieldmanager.Managed, manager string) (runtime.Object, fieldmanager.Managed, error) {
+func (NoopManager) Update(liveObj, newObj runtime.Object, managed internal.Managed, manager string) (runtime.Object, internal.Managed, error) {
 	return nil, nil, nil
 }
 
@@ -498,12 +497,12 @@ func TestNilNewObjectReplacedWithDeepCopyExcludingManagedFields(t *testing.T) {
 	}
 
 	// Decode the managed fields in the live object, since it isn't allowed in the patch.
-	managed, err := fieldmanager.DecodeManagedFields(accessor.GetManagedFields())
+	managed, err := internal.DecodeManagedFields(accessor.GetManagedFields())
 	if err != nil {
 		t.Fatalf("failed to decode managed fields: %v", err)
 	}
 
-	updater := fieldmanager.NewManagedFieldsUpdater(NoopManager{})
+	updater := internal.NewManagedFieldsUpdater(NoopManager{})
 
 	newObject, _, err := updater.Apply(obj, obj.DeepCopyObject(), managed, "some_manager", false)
 	if err != nil {
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/manager.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/manager.go
new file mode 100644
index 00000000000..053936103d7
--- /dev/null
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/manager.go
@@ -0,0 +1,52 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package internal
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
+)
+
+// Managed groups a fieldpath.ManagedFields together with the timestamps associated with each operation.
+type Managed interface {
+	// Fields gets the fieldpath.ManagedFields.
+	Fields() fieldpath.ManagedFields
+
+	// Times gets the timestamps associated with each operation.
+	Times() map[string]*metav1.Time
+}
+
+// Manager updates the managed fields and merges applied configurations.
+type Manager interface {
+	// Update is used when the object has already been merged (non-apply
+	// use-case), and simply updates the managed fields in the output
+	// object.
+	//  * `liveObj` is not mutated by this function
+	//  * `newObj` may be mutated by this function
+	// Returns the new object with managedFields removed, and the object's new
+	// proposed managedFields separately.
+	Update(liveObj, newObj runtime.Object, managed Managed, manager string) (runtime.Object, Managed, error)
+
+	// Apply is used when server-side apply is called, as it merges the
+	// object and updates the managed fields.
+	//  * `liveObj` is not mutated by this function
+	//  * `newObj` may be mutated by this function
+	// Returns the new object with managedFields removed, and the object's new
+	// proposed managedFields separately.
+	Apply(liveObj, appliedObj runtime.Object, managed Managed, fieldManager string, force bool) (runtime.Object, Managed, error)
+}
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/skipnonapplied.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/skipnonapplied.go
similarity index 99%
rename from staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/skipnonapplied.go
rename to staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/skipnonapplied.go
index a8c34ad6529..6b281ec1e57 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/skipnonapplied.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/skipnonapplied.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package fieldmanager
+package internal
 
 import (
 	"fmt"
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/skipnonapplied_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/skipnonapplied_test.go
similarity index 91%
rename from staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/skipnonapplied_test.go
rename to staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/skipnonapplied_test.go
index 1373676706f..c5723472d69 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/skipnonapplied_test.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/skipnonapplied_test.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package fieldmanager_test
+package internal_test
 
 import (
 	"strings"
@@ -24,14 +24,14 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
 	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanagertest"
+	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
 	"sigs.k8s.io/yaml"
 )
 
 func TestNoUpdateBeforeFirstApply(t *testing.T) {
-	f := fieldmanagertest.NewTestFieldManager(fakeTypeConverter, schema.FromAPIVersionAndKind("v1", "Pod"), "", func(m fieldmanager.Manager) fieldmanager.Manager {
-		return fieldmanager.NewSkipNonAppliedManager(
+	f := fieldmanagertest.NewTestFieldManager(fakeTypeConverter, schema.FromAPIVersionAndKind("v1", "Pod"), "", func(m internal.Manager) internal.Manager {
+		return internal.NewSkipNonAppliedManager(
 			m,
 			fieldmanagertest.NewFakeObjectCreater(),
 			schema.FromAPIVersionAndKind("v1", "Pod"),
@@ -70,8 +70,8 @@ func TestNoUpdateBeforeFirstApply(t *testing.T) {
 }
 
 func TestUpdateBeforeFirstApply(t *testing.T) {
-	f := fieldmanagertest.NewTestFieldManager(fakeTypeConverter, schema.FromAPIVersionAndKind("v1", "Pod"), "", func(m fieldmanager.Manager) fieldmanager.Manager {
-		return fieldmanager.NewSkipNonAppliedManager(
+	f := fieldmanagertest.NewTestFieldManager(fakeTypeConverter, schema.FromAPIVersionAndKind("v1", "Pod"), "", func(m internal.Manager) internal.Manager {
+		return internal.NewSkipNonAppliedManager(
 			m,
 			fieldmanagertest.NewFakeObjectCreater(),
 			schema.FromAPIVersionAndKind("v1", "Pod"),
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/stripmeta.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/stripmeta.go
similarity index 99%
rename from staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/stripmeta.go
rename to staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/stripmeta.go
index 1460d9c8021..9b61f3a6f35 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/stripmeta.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/stripmeta.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package fieldmanager
+package internal
 
 import (
 	"fmt"
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/structuredmerge.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/structuredmerge.go
similarity index 97%
rename from staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/structuredmerge.go
rename to staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/structuredmerge.go
index 213988e23c2..eb5598ac3bf 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/structuredmerge.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/structuredmerge.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package fieldmanager
+package internal
 
 import (
 	"fmt"
@@ -23,7 +23,6 @@ import (
 	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
 	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
 	"sigs.k8s.io/structured-merge-diff/v4/merge"
 )
@@ -108,7 +107,7 @@ func (f *structuredMergeManager) Update(liveObj, newObj runtime.Object, managed
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to update ManagedFields (%v): %v", objectGVKNN(newObjVersioned), err)
 	}
-	managed = internal.NewManaged(managedFields, managed.Times())
+	managed = NewManaged(managedFields, managed.Times())
 
 	return newObj, managed, nil
 }
@@ -151,7 +150,7 @@ func (f *structuredMergeManager) Apply(liveObj, patchObj runtime.Object, managed
 	if err != nil {
 		return nil, nil, err
 	}
-	managed = internal.NewManaged(managedFields, managed.Times())
+	managed = NewManaged(managedFields, managed.Times())
 
 	if newObjTyped == nil {
 		return nil, managed, nil
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/testdata/swagger.json b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/testdata/swagger.json
similarity index 100%
rename from staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/testdata/swagger.json
rename to staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/testdata/swagger.json
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/typeconverter.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/typeconverter.go
new file mode 100644
index 00000000000..3ff7507b32c
--- /dev/null
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/typeconverter.go
@@ -0,0 +1,112 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package internal
+
+import (
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/managedfields"
+	utilopenapi "k8s.io/apiserver/pkg/util/openapi"
+	"k8s.io/kube-openapi/pkg/validation/spec"
+	"sigs.k8s.io/structured-merge-diff/v4/typed"
+	"sigs.k8s.io/structured-merge-diff/v4/value"
+)
+
+// TypeConverter allows you to convert from runtime.Object to
+// typed.TypedValue and the other way around.
+type TypeConverter interface {
+	ObjectToTyped(runtime.Object) (*typed.TypedValue, error)
+	TypedToObject(*typed.TypedValue) (runtime.Object, error)
+}
+
+type typeConverter struct {
+	parser *managedfields.GvkParser
+}
+
+var _ TypeConverter = &typeConverter{}
+
+// NewTypeConverter builds a TypeConverter from a proto.Models. This
+// will automatically find the proper version of the object, and the
+// corresponding schema information.
+func NewTypeConverter(openapiSpec *spec.Swagger, preserveUnknownFields bool) (TypeConverter, error) {
+	models, err := utilopenapi.ToProtoModels(openapiSpec)
+	if err != nil {
+		return nil, err
+	}
+	parser, err := managedfields.NewGVKParser(models, preserveUnknownFields)
+	if err != nil {
+		return nil, err
+	}
+	return &typeConverter{parser: parser}, nil
+}
+
+func (c *typeConverter) ObjectToTyped(obj runtime.Object) (*typed.TypedValue, error) {
+	gvk := obj.GetObjectKind().GroupVersionKind()
+	t := c.parser.Type(gvk)
+	if t == nil {
+		return nil, NewNoCorrespondingTypeError(gvk)
+	}
+	switch o := obj.(type) {
+	case *unstructured.Unstructured:
+		return t.FromUnstructured(o.UnstructuredContent())
+	default:
+		return t.FromStructured(obj)
+	}
+}
+
+func (c *typeConverter) TypedToObject(value *typed.TypedValue) (runtime.Object, error) {
+	return valueToObject(value.AsValue())
+}
+
+type deducedTypeConverter struct{}
+
+// DeducedTypeConverter is a TypeConverter for CRDs that don't have a
+// schema. It does implement the same interface though (and create the
+// same types of objects), so that everything can still work the same.
+// CRDs are merged with all their fields being "atomic" (lists
+// included).
+func NewDeducedTypeConverter() TypeConverter {
+	return deducedTypeConverter{}
+}
+
+// ObjectToTyped converts an object into a TypedValue with a "deduced type".
+func (deducedTypeConverter) ObjectToTyped(obj runtime.Object) (*typed.TypedValue, error) {
+	switch o := obj.(type) {
+	case *unstructured.Unstructured:
+		return typed.DeducedParseableType.FromUnstructured(o.UnstructuredContent())
+	default:
+		return typed.DeducedParseableType.FromStructured(obj)
+	}
+}
+
+// TypedToObject transforms the typed value into a runtime.Object. That
+// is not specific to deduced type.
+func (deducedTypeConverter) TypedToObject(value *typed.TypedValue) (runtime.Object, error) {
+	return valueToObject(value.AsValue())
+}
+
+func valueToObject(val value.Value) (runtime.Object, error) {
+	vu := val.Unstructured()
+	switch o := vu.(type) {
+	case map[string]interface{}:
+		return &unstructured.Unstructured{Object: o}, nil
+	default:
+		return nil, fmt.Errorf("failed to convert value to unstructured for type %T", vu)
+	}
+}
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/typeconverter_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/typeconverter_test.go
similarity index 83%
rename from staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/typeconverter_test.go
rename to staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/typeconverter_test.go
index a51abd8cc95..83197ea886a 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/typeconverter_test.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/typeconverter_test.go
@@ -14,13 +14,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package fieldmanager_test
+package internal_test
 
 import (
-	"encoding/json"
 	"fmt"
-	"io/ioutil"
-	"path/filepath"
 	"reflect"
 	"testing"
 
@@ -28,28 +25,11 @@ import (
 	"sigs.k8s.io/yaml"
 
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
-	"k8s.io/kube-openapi/pkg/validation/spec"
+	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
 )
 
-var testTypeConverter = func() fieldmanager.TypeConverter {
-	data, err := ioutil.ReadFile(filepath.Join("testdata", "swagger.json"))
-	if err != nil {
-		panic(err)
-	}
-	spec := spec.Swagger{}
-	if err := json.Unmarshal(data, &spec); err != nil {
-		panic(err)
-	}
-	typeConverter, err := fieldmanager.NewTypeConverter(&spec, false)
-	if err != nil {
-		panic(err)
-	}
-	return typeConverter
-}()
-
 func TestTypeConverter(t *testing.T) {
-	dtc := fieldmanager.DeducedTypeConverter{}
+	dtc := internal.NewDeducedTypeConverter()
 
 	testCases := []struct {
 		name string
@@ -126,7 +106,7 @@ spec:
 	}
 }
 
-func testObjectToTyped(t *testing.T, tc fieldmanager.TypeConverter, y string) {
+func testObjectToTyped(t *testing.T, tc internal.TypeConverter, y string) {
 	obj := &unstructured.Unstructured{Object: map[string]interface{}{}}
 	if err := yaml.Unmarshal([]byte(y), &obj.Object); err != nil {
 		t.Fatalf("Failed to parse yaml object: %v", err)
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/versionconverter.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/versionconverter.go
similarity index 87%
rename from staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/versionconverter.go
rename to staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/versionconverter.go
index 477e92f796e..45855fa4ca2 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/versionconverter.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/versionconverter.go
@@ -14,9 +14,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package fieldmanager
+package internal
 
 import (
+	"fmt"
+
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
@@ -99,3 +101,23 @@ func (v *versionConverter) Convert(object *typed.TypedValue, version fieldpath.A
 func (v *versionConverter) IsMissingVersionError(err error) bool {
 	return runtime.IsNotRegisteredError(err) || isNoCorrespondingTypeError(err)
 }
+
+type noCorrespondingTypeErr struct {
+	gvk schema.GroupVersionKind
+}
+
+func NewNoCorrespondingTypeError(gvk schema.GroupVersionKind) error {
+	return &noCorrespondingTypeErr{gvk: gvk}
+}
+
+func (k *noCorrespondingTypeErr) Error() string {
+	return fmt.Sprintf("no corresponding type for %v", k.gvk)
+}
+
+func isNoCorrespondingTypeError(err error) bool {
+	if err == nil {
+		return false
+	}
+	_, ok := err.(*noCorrespondingTypeErr)
+	return ok
+}
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/versionconverter_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/versionconverter_test.go
similarity index 87%
rename from staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/versionconverter_test.go
rename to staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/versionconverter_test.go
index 3eb90ccbb8b..19e4476cb02 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/versionconverter_test.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/versionconverter_test.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package fieldmanager
+package internal
 
 import (
 	"encoding/json"
@@ -31,7 +31,7 @@ import (
 	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
 )
 
-var testSchema = func() *spec.Swagger {
+var testTypeConverter = func() TypeConverter {
 	data, err := ioutil.ReadFile(filepath.Join("testdata", "swagger.json"))
 	if err != nil {
 		panic(err)
@@ -40,22 +40,22 @@ var testSchema = func() *spec.Swagger {
 	if err := json.Unmarshal(data, &spec); err != nil {
 		panic(err)
 	}
-	return &spec
+	typeConverter, err := NewTypeConverter(&spec, false)
+	if err != nil {
+		panic(err)
+	}
+	return typeConverter
 }()
 
 // TestVersionConverter tests the version converter
 func TestVersionConverter(t *testing.T) {
-	tc, err := NewTypeConverter(testSchema, false)
-	if err != nil {
-		t.Fatalf("Failed to build TypeConverter: %v", err)
-	}
 	oc := fakeObjectConvertorForTestSchema{
 		gvkForVersion("v1beta1"): objForGroupVersion("apps/v1beta1"),
 		gvkForVersion("v1"):      objForGroupVersion("apps/v1"),
 	}
-	vc := newVersionConverter(tc, oc, schema.GroupVersion{Group: "apps", Version: runtime.APIVersionInternal})
+	vc := newVersionConverter(testTypeConverter, oc, schema.GroupVersion{Group: "apps", Version: runtime.APIVersionInternal})
 
-	input, err := tc.ObjectToTyped(objForGroupVersion("apps/v1beta1"))
+	input, err := testTypeConverter.ObjectToTyped(objForGroupVersion("apps/v1beta1"))
 	if err != nil {
 		t.Fatalf("error creating converting input object to a typed value: %v", err)
 	}
@@ -64,7 +64,7 @@ func TestVersionConverter(t *testing.T) {
 	if err != nil {
 		t.Fatalf("expected err to be nil but got %v", err)
 	}
-	actual, err := tc.TypedToObject(output)
+	actual, err := testTypeConverter.TypedToObject(output)
 	if err != nil {
 		t.Fatalf("error converting output typed value to an object %v", err)
 	}
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/scalehandler.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/scalehandler.go
index d9844990c29..655bf91d1e1 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/scalehandler.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/scalehandler.go
@@ -60,7 +60,7 @@ func NewScaleHandler(parentEntries []metav1.ManagedFieldsEntry, groupVersion sch
 //  2. Replicas path of the main resource is transformed to the replicas path of
 //     the scale subresource
 func (h *ScaleHandler) ToSubresource() ([]metav1.ManagedFieldsEntry, error) {
-	managed, err := DecodeManagedFields(h.parentEntries)
+	managed, err := internal.DecodeManagedFields(h.parentEntries)
 	if err != nil {
 		return nil, err
 	}
@@ -92,13 +92,13 @@ func (h *ScaleHandler) ToSubresource() ([]metav1.ManagedFieldsEntry, error) {
 // ToParent merges `scaleEntries` with the entries of the main resource and
 // transforms them accordingly
 func (h *ScaleHandler) ToParent(scaleEntries []metav1.ManagedFieldsEntry) ([]metav1.ManagedFieldsEntry, error) {
-	decodedParentEntries, err := DecodeManagedFields(h.parentEntries)
+	decodedParentEntries, err := internal.DecodeManagedFields(h.parentEntries)
 	if err != nil {
 		return nil, err
 	}
 	parentFields := decodedParentEntries.Fields()
 
-	decodedScaleEntries, err := DecodeManagedFields(scaleEntries)
+	decodedScaleEntries, err := internal.DecodeManagedFields(scaleEntries)
 	if err != nil {
 		return nil, err
 	}
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/typeconverter.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/typeconverter.go
index 9dc42161493..0aa97f7e790 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/typeconverter.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/typeconverter.go
@@ -17,119 +17,26 @@ limitations under the License.
 package fieldmanager
 
 import (
-	"fmt"
-
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apimachinery/pkg/util/managedfields"
-	utilopenapi "k8s.io/apiserver/pkg/util/openapi"
+	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
 	"k8s.io/kube-openapi/pkg/validation/spec"
-	"sigs.k8s.io/structured-merge-diff/v4/typed"
-	"sigs.k8s.io/structured-merge-diff/v4/value"
 )
 
 // TypeConverter allows you to convert from runtime.Object to
 // typed.TypedValue and the other way around.
-type TypeConverter interface {
-	ObjectToTyped(runtime.Object) (*typed.TypedValue, error)
-	TypedToObject(*typed.TypedValue) (runtime.Object, error)
-}
+type TypeConverter = internal.TypeConverter
 
-// DeducedTypeConverter is a TypeConverter for CRDs that don't have a
-// schema. It does implement the same interface though (and create the
-// same types of objects), so that everything can still work the same.
-// CRDs are merged with all their fields being "atomic" (lists
+// NewDeducedTypeConverter creates a TypeConverter for CRDs that don't
+// have a schema. It does implement the same interface though (and
+// create the same types of objects), so that everything can still work
+// the same. CRDs are merged with all their fields being "atomic" (lists
 // included).
-//
-// Note that this is not going to be sufficient for converting to/from
-// CRDs that have a schema defined (we don't support that schema yet).
-// TODO(jennybuckley): Use the schema provided by a CRD if it exists.
-type DeducedTypeConverter struct{}
-
-var _ TypeConverter = DeducedTypeConverter{}
-
-// ObjectToTyped converts an object into a TypedValue with a "deduced type".
-func (DeducedTypeConverter) ObjectToTyped(obj runtime.Object) (*typed.TypedValue, error) {
-	switch o := obj.(type) {
-	case *unstructured.Unstructured:
-		return typed.DeducedParseableType.FromUnstructured(o.UnstructuredContent())
-	default:
-		return typed.DeducedParseableType.FromStructured(obj)
-	}
+func NewDeducedTypeConverter() TypeConverter {
+	return internal.NewDeducedTypeConverter()
 }
 
-// TypedToObject transforms the typed value into a runtime.Object. That
-// is not specific to deduced type.
-func (DeducedTypeConverter) TypedToObject(value *typed.TypedValue) (runtime.Object, error) {
-	return valueToObject(value.AsValue())
-}
-
-type typeConverter struct {
-	parser *managedfields.GvkParser
-}
-
-var _ TypeConverter = &typeConverter{}
-
-// NewTypeConverter builds a TypeConverter from a spec.Swagger. This
+// NewTypeConverter builds a TypeConverter from a proto.Models. This
 // will automatically find the proper version of the object, and the
 // corresponding schema information.
 func NewTypeConverter(openapiSpec *spec.Swagger, preserveUnknownFields bool) (TypeConverter, error) {
-	models, err := utilopenapi.ToProtoModels(openapiSpec)
-	if err != nil {
-		return nil, err
-	}
-	parser, err := managedfields.NewGVKParser(models, preserveUnknownFields)
-	if err != nil {
-		return nil, err
-	}
-	return &typeConverter{parser: parser}, nil
-}
-
-func (c *typeConverter) ObjectToTyped(obj runtime.Object) (*typed.TypedValue, error) {
-	gvk := obj.GetObjectKind().GroupVersionKind()
-	t := c.parser.Type(gvk)
-	if t == nil {
-		return nil, newNoCorrespondingTypeError(gvk)
-	}
-	switch o := obj.(type) {
-	case *unstructured.Unstructured:
-		return t.FromUnstructured(o.UnstructuredContent())
-	default:
-		return t.FromStructured(obj)
-	}
-}
-
-func (c *typeConverter) TypedToObject(value *typed.TypedValue) (runtime.Object, error) {
-	return valueToObject(value.AsValue())
-}
-
-func valueToObject(val value.Value) (runtime.Object, error) {
-	vu := val.Unstructured()
-	switch o := vu.(type) {
-	case map[string]interface{}:
-		return &unstructured.Unstructured{Object: o}, nil
-	default:
-		return nil, fmt.Errorf("failed to convert value to unstructured for type %T", vu)
-	}
-}
-
-type noCorrespondingTypeErr struct {
-	gvk schema.GroupVersionKind
-}
-
-func newNoCorrespondingTypeError(gvk schema.GroupVersionKind) error {
-	return &noCorrespondingTypeErr{gvk: gvk}
-}
-
-func (k *noCorrespondingTypeErr) Error() string {
-	return fmt.Sprintf("no corresponding type for %v", k.gvk)
-}
-
-func isNoCorrespondingTypeError(err error) bool {
-	if err == nil {
-		return false
-	}
-	_, ok := err.(*noCorrespondingTypeErr)
-	return ok
+	return internal.NewTypeConverter(openapiSpec, preserveUnknownFields)
 }
diff --git a/test/integration/apiserver/admissionwebhook/invalid_managedFields_test.go b/test/integration/apiserver/admissionwebhook/invalid_managedFields_test.go
index 476eae690b1..684805ac55c 100644
--- a/test/integration/apiserver/admissionwebhook/invalid_managedFields_test.go
+++ b/test/integration/apiserver/admissionwebhook/invalid_managedFields_test.go
@@ -169,8 +169,9 @@ func TestMutatingWebhookResetsInvalidManagedFields(t *testing.T) {
 // validate against both decoding and validation to make sure we use the hardest rule between the both to reset
 // with decoding being as strict as it gets, only using it should be enough in admission
 func validateManagedFieldsAndDecode(managedFields []metav1.ManagedFieldsEntry) error {
-	if _, err := fieldmanager.DecodeManagedFields(managedFields); err != nil {
+	if err := fieldmanager.ValidateManagedFields(managedFields); err != nil {
 		return err
+
 	}
 	validationErrs := v1validation.ValidateManagedFields(managedFields, field.NewPath("metadata").Child("managedFields"))
 	return validationErrs.ToAggregate()
diff --git a/test/integration/apiserver/timestamp_transformer_test.go b/test/integration/apiserver/timestamp_transformer_test.go
index bdc564e7da8..345c9158129 100644
--- a/test/integration/apiserver/timestamp_transformer_test.go
+++ b/test/integration/apiserver/timestamp_transformer_test.go
@@ -34,7 +34,7 @@ import (
 )
 
 func convertToUnstructured(b *testing.B, obj runtime.Object) runtime.Object {
-	converter := fieldmanager.DeducedTypeConverter{}
+	converter := fieldmanager.NewDeducedTypeConverter()
 	typed, err := converter.ObjectToTyped(obj)
 	require.NoError(b, err)
 	res, err := converter.TypedToObject(typed)