diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/BUILD b/staging/src/k8s.io/apiserver/pkg/endpoints/BUILD
index e354ceb64f8..1b0d7be70b1 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/BUILD
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/BUILD
@@ -82,7 +82,7 @@ go_library(
         "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/endpoints/discovery:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/endpoints/handlers:go_default_library",
-        "//staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/endpoints/handlers/negotiation:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/endpoints/metrics:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/features:go_default_library",
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/BUILD b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/BUILD
index 127d6471cd8..3ab3338977e 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/BUILD
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/BUILD
@@ -74,7 +74,7 @@ go_library(
         "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/audit:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library",
-        "//staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/endpoints/handlers/negotiation:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/endpoints/metrics:go_default_library",
@@ -103,7 +103,7 @@ filegroup(
     name = "all-srcs",
     srcs = [
         ":package-srcs",
-        "//staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply:all-srcs",
+        "//staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager:all-srcs",
         "//staging/src/k8s.io/apiserver/pkg/endpoints/handlers/negotiation:all-srcs",
         "//staging/src/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters:all-srcs",
     ],
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/decode.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/decode.go
deleted file mode 100644
index 73af5d40326..00000000000
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/decode.go
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package apply
-
-import (
-	"fmt"
-	"reflect"
-	"strconv"
-
-	"sigs.k8s.io/structured-merge-diff/fieldpath"
-	"sigs.k8s.io/structured-merge-diff/value"
-
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-// DecodeManagedFields converts ManagedFields from the wire format (api format)
-// to the format used by sigs.k8s.io/structured-merge-diff
-func DecodeManagedFields(encodedManagedFields map[string]metav1.VersionedFieldSet) (managedFields fieldpath.ManagedFields, err error) {
-	managedFields = make(map[string]*fieldpath.VersionedSet, len(encodedManagedFields))
-	for manager, encodedVersionedSet := range encodedManagedFields {
-		managedFields[manager], err = decodeVersionedSet(&encodedVersionedSet)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding versioned set for %v: %v", manager, err)
-		}
-	}
-	return managedFields, nil
-}
-
-func decodeVersionedSet(encodedVersionedSet *metav1.VersionedFieldSet) (versionedSet *fieldpath.VersionedSet, err error) {
-	versionedSet = &fieldpath.VersionedSet{}
-	versionedSet.APIVersion = fieldpath.APIVersion(encodedVersionedSet.APIVersion)
-	versionedSet.Set = &fieldpath.Set{}
-	err = decodeSetInto(&encodedVersionedSet.Fields, versionedSet.Set)
-	if err != nil {
-		return nil, fmt.Errorf("error decoding set: %v", err)
-	}
-	return versionedSet, nil
-}
-
-func decodeSetInto(encodedSet *metav1.FieldSet, set *fieldpath.Set) (err error) {
-	m, err := decodePathElementSet(encodedSet.Members)
-	if err != nil {
-		return fmt.Errorf("error decoding path element set: %v", err)
-	}
-	set.Members = *m
-	c, err := decodeSetNodeMap(encodedSet.Children)
-	if err != nil {
-		return fmt.Errorf("error decoding children: %v", err)
-	}
-	set.Children = *c
-	return nil
-}
-
-func decodePathElementSet(encodedPathElementSet []metav1.FieldPathElement) (pathElementSet *fieldpath.PathElementSet, err error) {
-	pathElementSet = &fieldpath.PathElementSet{}
-	for _, encodedPathElement := range encodedPathElementSet {
-		pathElement, err := decodePathElement(&encodedPathElement)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding path element: %v", err)
-		}
-		pathElementSet.Insert(*pathElement)
-	}
-	return pathElementSet, nil
-}
-
-func decodeSetNodeMap(encodedSetNodeMap []metav1.FieldSetNode) (setNodeMap *fieldpath.SetNodeMap, err error) {
-	setNodeMap = &fieldpath.SetNodeMap{}
-	for _, encodedSetNode := range encodedSetNodeMap {
-		pathElement, err := decodePathElement(&encodedSetNode.PathElement)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding path element: %v", err)
-		}
-		set := setNodeMap.Descend(*pathElement)
-		decodeSetInto(encodedSetNode.Set, set)
-	}
-	return setNodeMap, nil
-}
-
-func decodePathElement(encodedPathElement *metav1.FieldPathElement) (pathElement *fieldpath.PathElement, err error) {
-	if encodedPathElement == nil {
-		return nil, nil
-	}
-	if !validateOneOf(encodedPathElement.FieldName, encodedPathElement.Key, encodedPathElement.Value, encodedPathElement.Index) {
-		return nil, fmt.Errorf("too many fields set: %v", encodedPathElement)
-	}
-
-	pathElement = &fieldpath.PathElement{}
-	pathElement.FieldName = encodedPathElement.FieldName
-	if encodedPathElement.Key != nil {
-		pathElement.Key = make([]value.Field, len(encodedPathElement.Key))
-		for i, encodedField := range encodedPathElement.Key {
-			pathElement.Key[i], err = decodeField(&encodedField)
-			if err != nil {
-				return nil, fmt.Errorf("error decoding field: %v", err)
-			}
-		}
-	}
-	pathElement.Value, err = decodeValue(encodedPathElement.Value)
-	if err != nil {
-		return nil, fmt.Errorf("error decoding value: %v", err)
-	}
-	if encodedPathElement.Index != nil {
-		v := int(*encodedPathElement.Index)
-		pathElement.Index = &v
-	}
-	return pathElement, nil
-}
-
-func decodeField(encodedField *metav1.FieldNameValuePair) (field value.Field, err error) {
-	field = value.Field{}
-	if encodedField == nil {
-		return field, fmt.Errorf("unexpected nil pointer")
-	}
-	field.Name = encodedField.Name
-	v, err := decodeValue(&encodedField.Value)
-	if err != nil {
-		return field, fmt.Errorf("error decoding value: %v", err)
-	}
-	if v != nil {
-		field.Value = *v
-	}
-	return field, nil
-}
-
-func decodeValue(encodedValue *metav1.FieldValue) (v *value.Value, err error) {
-	if encodedValue == nil {
-		return nil, nil
-	}
-	if !validateOneOf(encodedValue.FloatValue, encodedValue.IntValue, encodedValue.StringValue, encodedValue.BooleanValue) {
-		return nil, fmt.Errorf("too many fields set: %v", encodedValue)
-	}
-
-	v = &value.Value{}
-	if encodedValue.FloatValue != nil {
-		f, err := strconv.ParseFloat(*encodedValue.FloatValue, 64)
-		if err != nil {
-			return nil, fmt.Errorf("error parsing float: %v", err)
-		}
-		v.FloatValue = (*value.Float)(&f)
-	}
-	if encodedValue.IntValue != nil {
-		i := int64(*encodedValue.IntValue)
-		v.IntValue = (*value.Int)(&i)
-	}
-	v.StringValue = (*value.String)(encodedValue.StringValue)
-	v.BooleanValue = (*value.Boolean)(encodedValue.BooleanValue)
-	v.Null = encodedValue.Null
-	return v, nil
-}
-
-func validateOneOf(fields ...interface{}) bool {
-	nonNilCount := 0
-	for _, field := range fields {
-		if !reflect.ValueOf(field).IsNil() {
-			nonNilCount++
-		}
-	}
-	return nonNilCount <= 1
-}
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/decode_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/decode_test.go
deleted file mode 100644
index 3b069c59ac1..00000000000
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/decode_test.go
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package apply
-
-import (
-	"testing"
-)
-
-// TestValidateOneOf tests that validateOneOf returns true if <= 1
-// non-nil fields are passed to it are and false otherwise
-func TestValidateOneOf(t *testing.T) {
-	var nilValue *string
-	nonNilValue := strPtr("test")
-
-	tests := []struct {
-		fields   []interface{}
-		expected bool
-	}{
-		{
-			fields:   []interface{}{},
-			expected: true,
-		}, {
-			fields:   []interface{}{nilValue},
-			expected: true,
-		}, {
-			fields:   []interface{}{nonNilValue},
-			expected: true,
-		}, {
-			fields:   []interface{}{nilValue, nonNilValue, nilValue},
-			expected: true,
-		}, {
-			fields:   []interface{}{nonNilValue, nonNilValue},
-			expected: false,
-		}, {
-			fields:   []interface{}{nilValue, nonNilValue, nonNilValue, nilValue},
-			expected: false,
-		},
-	}
-
-	for i, tc := range tests {
-		actual := validateOneOf(tc.fields...)
-		if actual != tc.expected {
-			t.Errorf("[%v]expected validateOneOf(%v) to return:\n\t%+v\nbut got:\n\t%+v", i, tc.fields, tc.expected, actual)
-		}
-	}
-}
-
-func strPtr(s string) *string {
-	return &s
-}
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/managedfields.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/managedfields.go
deleted file mode 100644
index d71325ad9f8..00000000000
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/managedfields.go
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package apply
-
-import (
-	"fmt"
-
-	"k8s.io/apimachinery/pkg/api/meta"
-	"k8s.io/apimachinery/pkg/runtime"
-	"sigs.k8s.io/structured-merge-diff/fieldpath"
-)
-
-// DecodeObjectManagedFields extracts and converts the objects ManagedFields into a fieldpath.ManagedFields.
-func DecodeObjectManagedFields(from runtime.Object) (fieldpath.ManagedFields, error) {
-	if from == nil {
-		return make(map[string]*fieldpath.VersionedSet), nil
-	}
-	accessor, err := meta.Accessor(from)
-	if err != nil {
-		return nil, fmt.Errorf("couldn't get accessor: %v", err)
-	}
-	accessor.SetManagedFields(nil)
-
-	managed, err := DecodeManagedFields(accessor.GetManagedFields())
-	if err != nil {
-		return nil, fmt.Errorf("failed to convert managed fields from API: %v", err)
-	}
-	return managed, err
-}
-
-// EncodeObjectManagedFields converts and stores the fieldpathManagedFields into the objects ManagedFields
-func EncodeObjectManagedFields(obj runtime.Object, fields fieldpath.ManagedFields) error {
-	accessor, err := meta.Accessor(obj)
-	if err != nil {
-		return fmt.Errorf("couldn't get accessor: %v", err)
-	}
-
-	managed, err := EncodeManagedFields(fields)
-	if err != nil {
-		return fmt.Errorf("failed to convert back managed fields to API: %v", err)
-	}
-	accessor.SetManagedFields(managed)
-
-	return nil
-}
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/BUILD b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/BUILD
new file mode 100644
index 00000000000..c9ad22f1b95
--- /dev/null
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/BUILD
@@ -0,0 +1,33 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["fieldmanager.go"],
+    importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager",
+    importpath = "k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal:go_default_library",
+        "//vendor/k8s.io/kube-openapi/pkg/util/proto:go_default_library",
+        "//vendor/sigs.k8s.io/structured-merge-diff/merge:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/fieldmanager.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go
similarity index 90%
rename from staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/fieldmanager.go
rename to staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go
index cf168b17b73..1c47416f9a0 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/fieldmanager.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go
@@ -14,13 +14,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package apply
+package fieldmanager
 
 import (
 	"fmt"
 
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
 	openapiproto "k8s.io/kube-openapi/pkg/util/proto"
 	"sigs.k8s.io/structured-merge-diff/merge"
 )
@@ -30,7 +31,7 @@ const applyManager = "apply"
 // FieldManager updates the managed fields and merge applied
 // configurations.
 type FieldManager struct {
-	typeConverter   TypeConverter
+	typeConverter   internal.TypeConverter
 	objectConverter runtime.ObjectConvertor
 	objectDefaulter runtime.ObjectDefaulter
 	groupVersion    schema.GroupVersion
@@ -40,7 +41,7 @@ type FieldManager struct {
 // NewFieldManager creates a new FieldManager that merges apply requests
 // and update managed fields for other types of requests.
 func NewFieldManager(models openapiproto.Models, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, gv schema.GroupVersion) (*FieldManager, error) {
-	typeConverter, err := NewTypeConverter(models)
+	typeConverter, err := internal.NewTypeConverter(models)
 	if err != nil {
 		return nil, err
 	}
@@ -50,7 +51,7 @@ func NewFieldManager(models openapiproto.Models, objectConverter runtime.ObjectC
 		objectDefaulter: objectDefaulter,
 		groupVersion:    gv,
 		updater: merge.Updater{
-			Converter: NewVersionConverter(typeConverter, objectConverter),
+			Converter: internal.NewVersionConverter(typeConverter, objectConverter),
 		},
 	}, nil
 }
@@ -68,7 +69,7 @@ func (f *FieldManager) Update(liveObj, newObj runtime.Object, manager string) (r
 		return nil, fmt.Errorf("failed to convert live object to proper version: %v", err)
 	}
 
-	managed, err := DecodeObjectManagedFields(newObj)
+	managed, err := internal.DecodeObjectManagedFields(newObj)
 	if err != nil {
 		return nil, fmt.Errorf("failed to decode managed fields: %v", err)
 	}
@@ -87,7 +88,7 @@ func (f *FieldManager) Update(liveObj, newObj runtime.Object, manager string) (r
 		return nil, fmt.Errorf("failed to update ManagedFields: %v", err)
 	}
 
-	if err := EncodeObjectManagedFields(newObj, managed); err != nil {
+	if err := internal.EncodeObjectManagedFields(newObj, managed); err != nil {
 		return nil, fmt.Errorf("failed to encode managed fields: %v", err)
 	}
 
@@ -104,7 +105,7 @@ func (f *FieldManager) Apply(liveObj runtime.Object, patch []byte, force bool) (
 		return nil, fmt.Errorf("failed to convert live object to proper version: %v", err)
 	}
 
-	managed, err := DecodeObjectManagedFields(liveObj)
+	managed, err := internal.DecodeObjectManagedFields(liveObj)
 	if err != nil {
 		return nil, fmt.Errorf("failed to decode managed fields: %v", err)
 	}
@@ -128,7 +129,7 @@ func (f *FieldManager) Apply(liveObj runtime.Object, patch []byte, force bool) (
 		return nil, fmt.Errorf("failed to convert new typed object to object: %v", err)
 	}
 
-	if err := EncodeObjectManagedFields(newObj, managed); err != nil {
+	if err := internal.EncodeObjectManagedFields(newObj, managed); err != nil {
 		return nil, fmt.Errorf("failed to encode managed fields: %v", err)
 	}
 
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/BUILD b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/BUILD
similarity index 91%
rename from staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/BUILD
rename to staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/BUILD
index 23d8ae4f37d..5052fafa876 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/BUILD
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/BUILD
@@ -3,16 +3,13 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 go_library(
     name = "go_default_library",
     srcs = [
-        "decode.go",
-        "encode.go",
-        "fieldmanager.go",
         "gvkparser.go",
         "managedfields.go",
         "typeconverter.go",
         "versionconverter.go",
     ],
-    importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/handlers/apply",
-    importpath = "k8s.io/apiserver/pkg/endpoints/handlers/apply",
+    importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal",
+    importpath = "k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal",
     visibility = ["//visibility:public"],
     deps = [
         "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library",
@@ -33,8 +30,7 @@ go_library(
 go_test(
     name = "go_default_test",
     srcs = [
-        "decode_test.go",
-        "roundtrip_test.go",
+        "managedfields_test.go",
         "typeconverter_test.go",
         "versionconverter_test.go",
     ],
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/gvkparser.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/gvkparser.go
similarity index 99%
rename from staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/gvkparser.go
rename to staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/gvkparser.go
index 6aef2762da7..ce9a0492445 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/gvkparser.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/gvkparser.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package apply
+package internal
 
 import (
 	"fmt"
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/encode.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/managedfields.go
similarity index 50%
rename from staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/encode.go
rename to staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/managedfields.go
index 3d82b47ce30..f12c343a296 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/encode.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/managedfields.go
@@ -14,22 +14,204 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package apply
+package internal
 
 import (
 	"fmt"
+	"reflect"
 	"sort"
 	"strconv"
 
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+
 	"sigs.k8s.io/structured-merge-diff/fieldpath"
 	"sigs.k8s.io/structured-merge-diff/value"
-
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-// EncodeManagedFields converts ManagedFields from the the format used by
+// DecodeObjectManagedFields extracts and converts the objects ManagedFields into a fieldpath.ManagedFields.
+func DecodeObjectManagedFields(from runtime.Object) (fieldpath.ManagedFields, error) {
+	if from == nil {
+		return make(map[string]*fieldpath.VersionedSet), nil
+	}
+	accessor, err := meta.Accessor(from)
+	if err != nil {
+		return nil, fmt.Errorf("couldn't get accessor: %v", err)
+	}
+	accessor.SetManagedFields(nil)
+
+	managed, err := decodeManagedFields(accessor.GetManagedFields())
+	if err != nil {
+		return nil, fmt.Errorf("failed to convert managed fields from API: %v", err)
+	}
+	return managed, err
+}
+
+// EncodeObjectManagedFields converts and stores the fieldpathManagedFields into the objects ManagedFields
+func EncodeObjectManagedFields(obj runtime.Object, fields fieldpath.ManagedFields) error {
+	accessor, err := meta.Accessor(obj)
+	if err != nil {
+		return fmt.Errorf("couldn't get accessor: %v", err)
+	}
+
+	managed, err := encodeManagedFields(fields)
+	if err != nil {
+		return fmt.Errorf("failed to convert back managed fields to API: %v", err)
+	}
+	accessor.SetManagedFields(managed)
+
+	return nil
+}
+
+// decodeManagedFields converts ManagedFields from the wire format (api format)
+// to the format used by sigs.k8s.io/structured-merge-diff
+func decodeManagedFields(encodedManagedFields map[string]metav1.VersionedFieldSet) (managedFields fieldpath.ManagedFields, err error) {
+	managedFields = make(map[string]*fieldpath.VersionedSet, len(encodedManagedFields))
+	for manager, encodedVersionedSet := range encodedManagedFields {
+		managedFields[manager], err = decodeVersionedSet(&encodedVersionedSet)
+		if err != nil {
+			return nil, fmt.Errorf("error decoding versioned set for %v: %v", manager, err)
+		}
+	}
+	return managedFields, nil
+}
+
+func decodeVersionedSet(encodedVersionedSet *metav1.VersionedFieldSet) (versionedSet *fieldpath.VersionedSet, err error) {
+	versionedSet = &fieldpath.VersionedSet{}
+	versionedSet.APIVersion = fieldpath.APIVersion(encodedVersionedSet.APIVersion)
+	versionedSet.Set = &fieldpath.Set{}
+	err = decodeSetInto(&encodedVersionedSet.Fields, versionedSet.Set)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding set: %v", err)
+	}
+	return versionedSet, nil
+}
+
+func decodeSetInto(encodedSet *metav1.FieldSet, set *fieldpath.Set) (err error) {
+	m, err := decodePathElementSet(encodedSet.Members)
+	if err != nil {
+		return fmt.Errorf("error decoding path element set: %v", err)
+	}
+	set.Members = *m
+	c, err := decodeSetNodeMap(encodedSet.Children)
+	if err != nil {
+		return fmt.Errorf("error decoding children: %v", err)
+	}
+	set.Children = *c
+	return nil
+}
+
+func decodePathElementSet(encodedPathElementSet []metav1.FieldPathElement) (pathElementSet *fieldpath.PathElementSet, err error) {
+	pathElementSet = &fieldpath.PathElementSet{}
+	for _, encodedPathElement := range encodedPathElementSet {
+		pathElement, err := decodePathElement(&encodedPathElement)
+		if err != nil {
+			return nil, fmt.Errorf("error decoding path element: %v", err)
+		}
+		pathElementSet.Insert(*pathElement)
+	}
+	return pathElementSet, nil
+}
+
+func decodeSetNodeMap(encodedSetNodeMap []metav1.FieldSetNode) (setNodeMap *fieldpath.SetNodeMap, err error) {
+	setNodeMap = &fieldpath.SetNodeMap{}
+	for _, encodedSetNode := range encodedSetNodeMap {
+		pathElement, err := decodePathElement(&encodedSetNode.PathElement)
+		if err != nil {
+			return nil, fmt.Errorf("error decoding path element: %v", err)
+		}
+		set := setNodeMap.Descend(*pathElement)
+		decodeSetInto(encodedSetNode.Set, set)
+	}
+	return setNodeMap, nil
+}
+
+func decodePathElement(encodedPathElement *metav1.FieldPathElement) (pathElement *fieldpath.PathElement, err error) {
+	if encodedPathElement == nil {
+		return nil, nil
+	}
+	if !validateOneOf(encodedPathElement.FieldName, encodedPathElement.Key, encodedPathElement.Value, encodedPathElement.Index) {
+		return nil, fmt.Errorf("too many fields set: %v", encodedPathElement)
+	}
+
+	pathElement = &fieldpath.PathElement{}
+	pathElement.FieldName = encodedPathElement.FieldName
+	if encodedPathElement.Key != nil {
+		pathElement.Key = make([]value.Field, len(encodedPathElement.Key))
+		for i, encodedField := range encodedPathElement.Key {
+			pathElement.Key[i], err = decodeField(&encodedField)
+			if err != nil {
+				return nil, fmt.Errorf("error decoding field: %v", err)
+			}
+		}
+	}
+	pathElement.Value, err = decodeValue(encodedPathElement.Value)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding value: %v", err)
+	}
+	if encodedPathElement.Index != nil {
+		v := int(*encodedPathElement.Index)
+		pathElement.Index = &v
+	}
+	return pathElement, nil
+}
+
+func decodeField(encodedField *metav1.FieldNameValuePair) (field value.Field, err error) {
+	field = value.Field{}
+	if encodedField == nil {
+		return field, fmt.Errorf("unexpected nil pointer")
+	}
+	field.Name = encodedField.Name
+	v, err := decodeValue(&encodedField.Value)
+	if err != nil {
+		return field, fmt.Errorf("error decoding value: %v", err)
+	}
+	if v != nil {
+		field.Value = *v
+	}
+	return field, nil
+}
+
+func decodeValue(encodedValue *metav1.FieldValue) (v *value.Value, err error) {
+	if encodedValue == nil {
+		return nil, nil
+	}
+	if !validateOneOf(encodedValue.FloatValue, encodedValue.IntValue, encodedValue.StringValue, encodedValue.BooleanValue) {
+		return nil, fmt.Errorf("too many fields set: %v", encodedValue)
+	}
+
+	v = &value.Value{}
+	if encodedValue.FloatValue != nil {
+		f, err := strconv.ParseFloat(*encodedValue.FloatValue, 64)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing float: %v", err)
+		}
+		v.FloatValue = (*value.Float)(&f)
+	}
+	if encodedValue.IntValue != nil {
+		i := int64(*encodedValue.IntValue)
+		v.IntValue = (*value.Int)(&i)
+	}
+	v.StringValue = (*value.String)(encodedValue.StringValue)
+	v.BooleanValue = (*value.Boolean)(encodedValue.BooleanValue)
+	v.Null = encodedValue.Null
+	return v, nil
+}
+
+func validateOneOf(fields ...interface{}) bool {
+	nonNilCount := 0
+	for _, field := range fields {
+		if !reflect.ValueOf(field).IsNil() {
+			nonNilCount++
+		}
+	}
+	return nonNilCount <= 1
+}
+
+// encodeManagedFields converts ManagedFields from the the format used by
 // sigs.k8s.io/structured-merge-diff to the the wire format (api format)
-func EncodeManagedFields(managedFields fieldpath.ManagedFields) (encodedManagedFields map[string]metav1.VersionedFieldSet, err error) {
+func encodeManagedFields(managedFields fieldpath.ManagedFields) (encodedManagedFields map[string]metav1.VersionedFieldSet, err error) {
 	encodedManagedFields = make(map[string]metav1.VersionedFieldSet, len(managedFields))
 	for manager, versionedSet := range managedFields {
 		v, err := encodeVersionedSet(versionedSet)
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/roundtrip_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/managedfields_test.go
similarity index 85%
rename from staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/roundtrip_test.go
rename to staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/managedfields_test.go
index a8c6dcd7292..94e4e3ae1e8 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/roundtrip_test.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/managedfields_test.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package apply
+package internal
 
 import (
 	"reflect"
@@ -240,7 +240,7 @@ func TestRoundTripManagedFields(t *testing.T) {
 			continue
 		}
 
-		decoded, err := DecodeManagedFields(original)
+		decoded, err := decodeManagedFields(original)
 		if err == nil && len(tc.errString) > 0 {
 			t.Errorf("[%v]expected error but got none.", i)
 			continue
@@ -253,7 +253,7 @@ func TestRoundTripManagedFields(t *testing.T) {
 			t.Errorf("[%v]expected error with %q but got: %v", i, tc.errString, err)
 			continue
 		}
-		encoded, err := EncodeManagedFields(decoded)
+		encoded, err := encodeManagedFields(decoded)
 		if err != nil {
 			t.Errorf("[%v]did not expect round trip error but got: %v", i, err)
 			continue
@@ -263,3 +263,46 @@ func TestRoundTripManagedFields(t *testing.T) {
 		}
 	}
 }
+
+// TestValidateOneOf tests that validateOneOf returns true if <= 1
+// non-nil fields are passed to it are and false otherwise
+func TestValidateOneOf(t *testing.T) {
+	var nilValue *string
+	nonNilValue := strPtr("test")
+
+	tests := []struct {
+		fields   []interface{}
+		expected bool
+	}{
+		{
+			fields:   []interface{}{},
+			expected: true,
+		}, {
+			fields:   []interface{}{nilValue},
+			expected: true,
+		}, {
+			fields:   []interface{}{nonNilValue},
+			expected: true,
+		}, {
+			fields:   []interface{}{nilValue, nonNilValue, nilValue},
+			expected: true,
+		}, {
+			fields:   []interface{}{nonNilValue, nonNilValue},
+			expected: false,
+		}, {
+			fields:   []interface{}{nilValue, nonNilValue, nonNilValue, nilValue},
+			expected: false,
+		},
+	}
+
+	for i, tc := range tests {
+		actual := validateOneOf(tc.fields...)
+		if actual != tc.expected {
+			t.Errorf("[%v]expected validateOneOf(%v) to return:\n\t%+v\nbut got:\n\t%+v", i, tc.fields, tc.expected, actual)
+		}
+	}
+}
+
+func strPtr(s string) *string {
+	return &s
+}
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/typeconverter.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/typeconverter.go
similarity index 99%
rename from staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/typeconverter.go
rename to staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/typeconverter.go
index 1ed05be1698..79f3017cb43 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/typeconverter.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/typeconverter.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package apply
+package internal
 
 import (
 	"fmt"
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/typeconverter_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/typeconverter_test.go
similarity index 93%
rename from staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/typeconverter_test.go
rename to staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/typeconverter_test.go
index d9b879bc88a..bced6d6328f 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/typeconverter_test.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/typeconverter_test.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package apply_test
+package internal_test
 
 import (
 	"path/filepath"
@@ -23,14 +23,14 @@ import (
 
 	"github.com/ghodss/yaml"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	"k8s.io/apiserver/pkg/endpoints/handlers/apply"
+	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
 	"k8s.io/kube-openapi/pkg/util/proto"
 	prototesting "k8s.io/kube-openapi/pkg/util/proto/testing"
 )
 
 var fakeSchema = prototesting.Fake{
 	Path: filepath.Join(
-		"..", "..", "..", "..", "..", "..", "..", "..",
+		"..", "..", "..", "..", "..", "..", "..", "..", "..",
 		"api", "openapi-spec", "swagger.json"),
 }
 
@@ -44,7 +44,7 @@ func TestTypeConverter(t *testing.T) {
 		t.Fatalf("Failed to build OpenAPI models: %v", err)
 	}
 
-	tc, err := apply.NewTypeConverter(m)
+	tc, err := internal.NewTypeConverter(m)
 	if err != nil {
 		t.Fatalf("Failed to build TypeConverter: %v", err)
 	}
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/versionconverter.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/versionconverter.go
similarity index 99%
rename from staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/versionconverter.go
rename to staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/versionconverter.go
index 72e83bfce2a..b1ad9567003 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/versionconverter.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/versionconverter.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package apply
+package internal
 
 import (
 	"k8s.io/apimachinery/pkg/runtime"
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/versionconverter_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/versionconverter_test.go
similarity index 90%
rename from staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/versionconverter_test.go
rename to staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/versionconverter_test.go
index 0d5a92e711f..f710f7ec349 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/apply/versionconverter_test.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/versionconverter_test.go
@@ -14,29 +14,22 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package apply
+package internal_test
 
 import (
 	"fmt"
-	"path/filepath"
 	"reflect"
 	"testing"
 
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
 
 	"k8s.io/kube-openapi/pkg/util/proto"
-	prototesting "k8s.io/kube-openapi/pkg/util/proto/testing"
 	"sigs.k8s.io/structured-merge-diff/fieldpath"
 )
 
-var fakeSchema = prototesting.Fake{
-	Path: filepath.Join(
-		"..", "..", "..", "..", "..", "..", "..", "..",
-		"api", "openapi-spec", "swagger.json"),
-}
-
 // TestVersionConverter tests the version converter
 func TestVersionConverter(t *testing.T) {
 	d, err := fakeSchema.OpenAPISchema()
@@ -47,7 +40,7 @@ func TestVersionConverter(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Failed to build OpenAPI models: %v", err)
 	}
-	tc, err := NewTypeConverter(m)
+	tc, err := internal.NewTypeConverter(m)
 	if err != nil {
 		t.Fatalf("Failed to build TypeConverter: %v", err)
 	}
@@ -55,7 +48,7 @@ func TestVersionConverter(t *testing.T) {
 		gvkForVersion("v1beta1"): objForGroupVersion("apps/v1beta1"),
 		gvkForVersion("v1"):      objForGroupVersion("apps/v1"),
 	}
-	vc := NewVersionConverter(tc, oc)
+	vc := internal.NewVersionConverter(tc, oc)
 
 	input, err := tc.ObjectToTyped(objForGroupVersion("apps/v1beta1"))
 	if err != nil {
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/patch.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/patch.go
index a38814c47a7..7f90677fc09 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/patch.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/patch.go
@@ -39,7 +39,7 @@ import (
 	"k8s.io/apiserver/pkg/admission"
 	"k8s.io/apiserver/pkg/audit"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
-	"k8s.io/apiserver/pkg/endpoints/handlers/apply"
+	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
 	"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
 	"k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/features"
@@ -280,7 +280,7 @@ type patchMechanism interface {
 type jsonPatcher struct {
 	*patcher
 
-	fieldManager *apply.FieldManager
+	fieldManager *fieldmanager.FieldManager
 }
 
 func (p *jsonPatcher) applyPatchToCurrentObject(currentObject runtime.Object) (runtime.Object, error) {
@@ -339,7 +339,7 @@ type smpPatcher struct {
 
 	// Schema
 	schemaReferenceObj runtime.Object
-	fieldManager       *apply.FieldManager
+	fieldManager       *fieldmanager.FieldManager
 }
 
 func (p *smpPatcher) applyPatchToCurrentObject(currentObject runtime.Object) (runtime.Object, error) {
@@ -377,7 +377,7 @@ type applyPatcher struct {
 	options      *metav1.PatchOptions
 	creater      runtime.ObjectCreater
 	kind         schema.GroupVersionKind
-	fieldManager *apply.FieldManager
+	fieldManager *fieldmanager.FieldManager
 }
 
 func (p *applyPatcher) applyPatchToCurrentObject(obj runtime.Object) (runtime.Object, error) {
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest.go
index a37f5fc30fb..f058e24dcfb 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest.go
@@ -35,7 +35,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apiserver/pkg/admission"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
-	"k8s.io/apiserver/pkg/endpoints/handlers/apply"
+	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
 	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
 	"k8s.io/apiserver/pkg/endpoints/metrics"
 	"k8s.io/apiserver/pkg/endpoints/request"
@@ -60,7 +60,7 @@ type RequestScope struct {
 	Trace           *utiltrace.Trace
 
 	TableConvertor rest.TableConvertor
-	FieldManager   *apply.FieldManager
+	FieldManager   *fieldmanager.FieldManager
 
 	Resource    schema.GroupVersionResource
 	Kind        schema.GroupVersionKind
diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go b/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go
index d810b38571a..c9bcbd4628e 100644
--- a/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go
+++ b/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go
@@ -34,7 +34,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apiserver/pkg/admission"
 	"k8s.io/apiserver/pkg/endpoints/handlers"
-	"k8s.io/apiserver/pkg/endpoints/handlers/apply"
+	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
 	"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
 	"k8s.io/apiserver/pkg/endpoints/metrics"
 	"k8s.io/apiserver/pkg/features"
@@ -516,7 +516,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 		reqScope.MetaGroupVersion = *a.group.MetaGroupVersion
 	}
 	if a.group.OpenAPIModels != nil && utilfeature.DefaultFeatureGate.Enabled(features.ServerSideApply) {
-		fm, err := apply.NewFieldManager(
+		fm, err := fieldmanager.NewFieldManager(
 			a.group.OpenAPIModels,
 			a.group.UnsafeConvertor,
 			a.group.Defaulter,