Merge pull request #118990 from alexzielenski/apiserver/apiextensions/crd-validation-ratcheting

CRD Validation Ratcheting alpha implementation

pkg/features/kube_features.go

@@ -17,6 +17,7 @@ limitations under the License.
 package features
 
 import (
+	apiextensionsfeatures "k8s.io/apiextensions-apiserver/pkg/features"
 	"k8s.io/apimachinery/pkg/util/runtime"
 	genericfeatures "k8s.io/apiserver/pkg/features"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
@@ -1191,6 +1192,11 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
 
 	genericfeatures.ServerSideFieldValidation: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.29
 
+	// inherited features from apiextensions-apiserver, relisted here to get a conflict if it is changed
+	// unintentionally on either side:
+
+	apiextensionsfeatures.CRDValidationRatcheting: {Default: false, PreRelease: featuregate.Alpha},
+
 	// features that enable backwards compatibility but are scheduled to be removed
 	// ...
 	HPAScaleToZero: {Default: false, PreRelease: featuregate.Alpha},

staging/src/k8s.io/apiextensions-apiserver/go.mod

@@ -6,6 +6,7 @@ go 1.20
 
 require (
 	github.com/emicklei/go-restful/v3 v3.9.0
+	github.com/evanphx/json-patch v5.6.0+incompatible
 	github.com/gogo/protobuf v1.3.2
 	github.com/google/cel-go v0.16.0
 	github.com/google/gnostic-models v0.6.8
@@ -49,7 +50,6 @@ require (
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
-	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/felixge/httpsnoop v1.0.3 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect

staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation.go

@@ -825,7 +825,7 @@ func validateCustomResourceDefinitionValidation(ctx context.Context, customResou
 
 	// if validation passed otherwise, make sure we can actually construct a schema validator from this custom resource validation.
 	if len(allErrs) == 0 {
-		if _, _, err := apiservervalidation.NewSchemaValidator(customResourceValidation); err != nil {
+		if _, _, err := apiservervalidation.NewSchemaValidator(customResourceValidation.OpenAPIV3Schema); err != nil {
 			allErrs = append(allErrs, field.Invalid(fldPath, "", fmt.Sprintf("error building validator: %v", err)))
 		}
 	}

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go

@@ -79,8 +79,6 @@ import (
 	"k8s.io/klog/v2"
 	"k8s.io/kube-openapi/pkg/spec3"
 	"k8s.io/kube-openapi/pkg/validation/spec"
-	"k8s.io/kube-openapi/pkg/validation/strfmt"
-	"k8s.io/kube-openapi/pkg/validation/validate"
 )
 
 // crdHandler serves the `/apis` endpoint.
@@ -739,20 +737,22 @@ func (r *crdHandler) getOrCreateServingInfoFor(uid types.UID, name string) (*crd
 			utilruntime.HandleError(err)
 			return nil, fmt.Errorf("the server could not properly serve the CR schema")
 		}
+		var internalSchemaProps *apiextensionsinternal.JSONSchemaProps
 		var internalValidationSchema *apiextensionsinternal.CustomResourceValidation
 		if validationSchema != nil {
 			internalValidationSchema = &apiextensionsinternal.CustomResourceValidation{}
 			if err := apiextensionsv1.Convert_v1_CustomResourceValidation_To_apiextensions_CustomResourceValidation(validationSchema, internalValidationSchema, nil); err != nil {
 				return nil, fmt.Errorf("failed to convert CRD validation to internal version: %v", err)
 			}
+			internalSchemaProps = internalValidationSchema.OpenAPIV3Schema
 		}
-		validator, _, err := apiservervalidation.NewSchemaValidator(internalValidationSchema)
+		validator, _, err := apiservervalidation.NewSchemaValidator(internalSchemaProps)
 		if err != nil {
 			return nil, err
 		}
 
 		var statusSpec *apiextensionsinternal.CustomResourceSubresourceStatus
-		var statusValidator *validate.SchemaValidator
+		var statusValidator apiservervalidation.SchemaValidator
 		subresources, err := apiextensionshelpers.GetSubresourcesForVersion(crd, v.Name)
 		if err != nil {
 			utilruntime.HandleError(err)
@@ -767,11 +767,10 @@ func (r *crdHandler) getOrCreateServingInfoFor(uid types.UID, name string) (*crd
 			// for the status subresource, validate only against the status schema
 			if internalValidationSchema != nil && internalValidationSchema.OpenAPIV3Schema != nil && internalValidationSchema.OpenAPIV3Schema.Properties != nil {
 				if statusSchema, ok := internalValidationSchema.OpenAPIV3Schema.Properties["status"]; ok {
-					openapiSchema := &spec.Schema{}
+					statusValidator, _, err = apiservervalidation.NewSchemaValidator(&statusSchema)
-					if err := apiservervalidation.ConvertJSONSchemaPropsWithPostProcess(&statusSchema, openapiSchema, apiservervalidation.StripUnsupportedFormatsPostProcess); err != nil {
+					if err != nil {
 						return nil, err
 					}
-					statusValidator = validate.NewSchemaValidator(openapiSchema, nil, "", strfmt.Default)
 				}
 			}
 		}

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation/ratcheting.go

@@ -0,0 +1,412 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package validation
+
+import (
+	"reflect"
+
+	"k8s.io/apiserver/pkg/cel/common"
+	celopenapi "k8s.io/apiserver/pkg/cel/openapi"
+	"k8s.io/kube-openapi/pkg/validation/spec"
+	"k8s.io/kube-openapi/pkg/validation/strfmt"
+	"k8s.io/kube-openapi/pkg/validation/validate"
+)
+
+// schemaArgs are the arguments to constructor for OpenAPI schema validator,
+// NewSchemaValidator
+type schemaArgs struct {
+	schema       *spec.Schema
+	root         interface{}
+	path         string
+	knownFormats strfmt.Registry
+	options      []validate.Option
+}
+
+// RatchetingSchemaValidator wraps kube-openapis SchemaValidator to provide a
+// ValidateUpdate function which allows ratcheting
+type RatchetingSchemaValidator struct {
+	schemaArgs
+}
+
+func NewRatchetingSchemaValidator(schema *spec.Schema, rootSchema interface{}, root string, formats strfmt.Registry, options ...validate.Option) *RatchetingSchemaValidator {
+	return &RatchetingSchemaValidator{
+		schemaArgs: schemaArgs{
+			schema:       schema,
+			root:         rootSchema,
+			path:         root,
+			knownFormats: formats,
+			options:      options,
+		},
+	}
+}
+
+func (r *RatchetingSchemaValidator) Validate(new interface{}) *validate.Result {
+	sv := validate.NewSchemaValidator(r.schema, r.root, r.path, r.knownFormats, r.options...)
+	return sv.Validate(new)
+}
+
+func (r *RatchetingSchemaValidator) ValidateUpdate(new, old interface{}) *validate.Result {
+	return newRatchetingValueValidator(new, old, r.schemaArgs).Validate()
+}
+
+// ratchetingValueValidator represents an invocation of SchemaValidator.ValidateUpdate
+// for specific arguments for `old` and `new`
+//
+// It follows the openapi SchemaValidator down its traversal of the new value
+// by injecting validate.Option into each recursive invocation.
+//
+// A ratchetingValueValidator will be constructed and added to the tree for
+// each explored sub-index and sub-property during validation.
+//
+// It's main job is to keep the old/new values correlated as the traversal
+// continues, and postprocess errors according to our ratcheting policy.
+//
+// ratchetingValueValidator is not thread safe.
+type ratchetingValueValidator struct {
+	// schemaArgs provides the arguments to use in the temporary SchemaValidator
+	// that is created during a call to Validate.
+	schemaArgs
+
+	// Currently correlated old value during traversal of the schema/object
+	oldValue interface{}
+
+	// Value being validated
+	value interface{}
+
+	// Scratch space below, may change during validation
+
+	// Cached comparison result of DeepEqual of `value` and `thunk.oldValue`
+	comparisonResult *bool
+
+	// Cached map representation of a map-type list, or nil if not map-type list
+	mapList common.MapList
+
+	// Children spawned by a call to `Validate` on this object
+	// key is either a string or an index, depending upon whether `value` is
+	// a map or a list, respectively.
+	//
+	// The list of children may be incomplete depending upon if the internal
+	// logic of kube-openapi's SchemaValidator short-circuited before
+	// reaching all of the children.
+	//
+	// It should be expected to have an entry for either all of the children, or
+	// none of them.
+	children map[interface{}]*ratchetingValueValidator
+}
+
+func newRatchetingValueValidator(newValue, oldValue interface{}, args schemaArgs) *ratchetingValueValidator {
+	return &ratchetingValueValidator{
+		oldValue:   oldValue,
+		value:      newValue,
+		schemaArgs: args,
+		children:   map[interface{}]*ratchetingValueValidator{},
+	}
+}
+
+// getValidateOption provides a kube-openapi validate.Option for SchemaValidator
+// that injects a ratchetingValueValidator to be used for all subkeys and subindices
+func (r *ratchetingValueValidator) getValidateOption() validate.Option {
+	return func(svo *validate.SchemaValidatorOptions) {
+		svo.NewValidatorForField = func(field string, schema *spec.Schema, rootSchema interface{}, root string, formats strfmt.Registry, opts ...validate.Option) validate.ValueValidator {
+			return r.SubPropertyValidator(field, schema, rootSchema, root, formats, opts...)
+		}
+		svo.NewValidatorForIndex = func(index int, schema *spec.Schema, rootSchema interface{}, root string, formats strfmt.Registry, opts ...validate.Option) validate.ValueValidator {
+			return r.SubIndexValidator(index, schema, rootSchema, root, formats, opts...)
+		}
+	}
+}
+
+// Validate validates the update from r.oldValue to r.value
+//
+// During evaluation, a temporary tree of ratchetingValueValidator is built for all
+// traversed field paths. It is necessary to build the tree to take advantage of
+// DeepEqual checks performed by lower levels of the object during validation without
+// greatly modifying `kube-openapi`'s implementation.
+//
+// The tree, and all cache storage/scratch space for the validation of a single
+// call to `Validate` is thrown away at the end of the top-level call
+// to `Validate`.
+//
+// `Validate` will create a node in the tree to for each of the explored children.
+// The node's main purpose is to store a lazily computed DeepEqual check between
+// the oldValue and the currently passed value. If the check is performed, it
+// will be stored in the node to be re-used by a parent node during a DeepEqual
+// comparison, if necessary.
+//
+// This call has a side-effect of populating it's `children` variable with
+// the explored nodes of the object tree.
+func (r *ratchetingValueValidator) Validate() *validate.Result {
+	opts := append([]validate.Option{
+		r.getValidateOption(),
+	}, r.options...)
+
+	s := validate.NewSchemaValidator(r.schema, r.root, r.path, r.knownFormats, opts...)
+
+	res := s.Validate(r.value)
+
+	if res.IsValid() {
+		return res
+	}
+
+	// Current ratcheting rule is to ratchet errors if DeepEqual(old, new) is true.
+	if r.CachedDeepEqual() {
+		newRes := &validate.Result{}
+		newRes.MergeAsWarnings(res)
+		return newRes
+	}
+
+	return res
+}
+
+// SubPropertyValidator overrides the standard validator constructor for sub-properties by
+// returning our special ratcheting variant.
+//
+// If we can correlate an old value, we return a ratcheting validator to
+// use for the child.
+//
+// If the old value cannot be correlated, then default validation is used.
+func (r *ratchetingValueValidator) SubPropertyValidator(field string, schema *spec.Schema, rootSchema interface{}, root string, formats strfmt.Registry, options ...validate.Option) validate.ValueValidator {
+	// Find correlated old value
+	asMap, ok := r.oldValue.(map[string]interface{})
+	if !ok {
+		return validate.NewSchemaValidator(schema, rootSchema, root, formats, options...)
+	}
+
+	oldValueForField, ok := asMap[field]
+	if !ok {
+		return validate.NewSchemaValidator(schema, rootSchema, root, formats, options...)
+	}
+
+	return inlineValidator(func(new interface{}) *validate.Result {
+		childNode := newRatchetingValueValidator(new, oldValueForField, schemaArgs{
+			schema:       schema,
+			root:         rootSchema,
+			path:         root,
+			knownFormats: formats,
+			options:      options,
+		})
+
+		r.children[field] = childNode
+		return childNode.Validate()
+	})
+
+}
+
+// SubIndexValidator overrides the standard validator constructor for sub-indicies by
+// returning our special ratcheting variant.
+//
+// If we can correlate an old value, we return a ratcheting validator to
+// use for the child.
+//
+// If the old value cannot be correlated, then default validation is used.
+func (r *ratchetingValueValidator) SubIndexValidator(index int, schema *spec.Schema, rootSchema interface{}, root string, formats strfmt.Registry, options ...validate.Option) validate.ValueValidator {
+	oldValueForIndex := r.correlateOldValueForChildAtNewIndex(index)
+	if oldValueForIndex == nil {
+		// If correlation fails, default to non-ratcheting logic
+		return validate.NewSchemaValidator(schema, rootSchema, root, formats, options...)
+	}
+
+	return inlineValidator(func(new interface{}) *validate.Result {
+		childNode := newRatchetingValueValidator(new, oldValueForIndex, schemaArgs{
+			schema:       schema,
+			root:         rootSchema,
+			path:         root,
+			knownFormats: formats,
+			options:      options,
+		})
+
+		r.children[index] = childNode
+		return childNode.Validate()
+	})
+}
+
+// If oldValue is not a list, returns nil
+// If oldValue is a list takes mapType into account and attempts to find the
+// old value with the same index or key, depending upon the mapType.
+//
+// If listType is map, creates a map representation of the list using the designated
+// map-keys and caches it for future calls.
+func (r *ratchetingValueValidator) correlateOldValueForChildAtNewIndex(index int) any {
+	oldAsList, ok := r.oldValue.([]interface{})
+	if !ok {
+		return nil
+	}
+
+	asList, ok := r.value.([]interface{})
+	if !ok {
+		return nil
+	} else if len(asList) <= index {
+		// Cannot correlate out of bounds index
+		return nil
+	}
+
+	listType, _ := r.schema.Extensions.GetString("x-kubernetes-list-type")
+	switch listType {
+	case "map":
+		// Look up keys for this index in current object
+		currentElement := asList[index]
+
+		oldList := r.mapList
+		if oldList == nil {
+			oldList = celopenapi.MakeMapList(r.schema, oldAsList)
+			r.mapList = oldList
+		}
+		return oldList.Get(currentElement)
+
+	case "set":
+		// Are sets correlatable? Only if the old value equals the current value.
+		// We might be able to support this, but do not currently see a lot
+		// of value
+		// (would allow you to add/remove items from sets with ratcheting but not change them)
+		return nil
+	case "atomic":
+		// Atomic lists are not correlatable by item
+		// Ratcheting is not available on a per-index basis
+		return nil
+	default:
+		// Correlate by-index by default.
+		//
+		// Cannot correlate an out-of-bounds index
+		if len(oldAsList) <= index {
+			return nil
+		}
+
+		return oldAsList[index]
+	}
+}
+
+// CachedDeepEqual is equivalent to reflect.DeepEqual, but caches the
+// results in the tree of ratchetInvocationScratch objects on the way:
+//
+// For objects and arrays, this function will make a best effort to make
+// use of past DeepEqual checks performed by this Node's children, if available.
+//
+// If a lazy computation could not be found for all children possibly due
+// to validation logic short circuiting and skipping the children, then
+// this function simply defers to reflect.DeepEqual.
+func (r *ratchetingValueValidator) CachedDeepEqual() (res bool) {
+	if r.comparisonResult != nil {
+		return *r.comparisonResult
+	}
+
+	defer func() {
+		r.comparisonResult = &res
+	}()
+
+	if r.value == nil && r.oldValue == nil {
+		return true
+	} else if r.value == nil || r.oldValue == nil {
+		return false
+	}
+
+	oldAsArray, oldIsArray := r.oldValue.([]interface{})
+	newAsArray, newIsArray := r.value.([]interface{})
+
+	if oldIsArray != newIsArray {
+		return false
+	} else if oldIsArray {
+		if len(oldAsArray) != len(newAsArray) {
+			return false
+		} else if len(r.children) != len(oldAsArray) {
+			// kube-openapi validator is written to always visit all
+			// children of a slice, so this case is only possible if
+			// one of the children could not be correlated. In that case,
+			// we know the objects are not equal.
+			//
+			return false
+		}
+
+		// Correctly considers map-type lists due to fact that index here
+		// is only used for numbering. The correlation is stored in the
+		// childInvocation itself
+		//
+		// NOTE: This does not consider sets, since we don't correlate them.
+		for i := range newAsArray {
+			// Query for child
+			child, ok := r.children[i]
+			if !ok {
+				// This should not happen
+				return false
+			} else if !child.CachedDeepEqual() {
+				// If one child is not equal the entire object is not equal
+				return false
+			}
+		}
+
+		return true
+	}
+
+	oldAsMap, oldIsMap := r.oldValue.(map[string]interface{})
+	newAsMap, newIsMap := r.value.(map[string]interface{})
+
+	if oldIsMap != newIsMap {
+		return false
+	} else if oldIsMap {
+		if len(oldAsMap) != len(newAsMap) {
+			return false
+		} else if len(oldAsMap) == 0 && len(newAsMap) == 0 {
+			// Both empty
+			return true
+		} else if len(r.children) != len(oldAsMap) {
+			// If we are missing a key it is because the old value could not
+			// be correlated to the new, so the objects are not equal.
+			//
+			return false
+		}
+
+		for k := range oldAsMap {
+			// Check to see if this child was explored during validation
+			child, ok := r.children[k]
+			if !ok {
+				// Child from old missing in new due to key change
+				// Objects are not equal.
+				return false
+			} else if !child.CachedDeepEqual() {
+				// If one child is not equal the entire object is not equal
+				return false
+			}
+		}
+
+		return true
+	}
+
+	return reflect.DeepEqual(r.oldValue, r.value)
+}
+
+// A validator which just calls a validate function, and advertises that it
+// validates anything
+//
+// In the future kube-openapi's ValueValidator interface can be simplified
+// to be closer to `currentValidator.Options.NewValidator(value, ...).Validate()`
+// so that a tree of "validation nodes" can be more formally encoded in the API.
+// In that case this class would not be necessary.
+type inlineValidator func(new interface{}) *validate.Result
+
+var _ validate.ValueValidator = inlineValidator(nil)
+
+func (f inlineValidator) Validate(new interface{}) *validate.Result {
+	return f(new)
+}
+
+func (f inlineValidator) SetPath(path string) {
+	// Do nothing
+	// Unused by kube-openapi
+}
+
+func (f inlineValidator) Applies(source interface{}, valueKind reflect.Kind) bool {
+	return true
+}

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation/ratcheting_test.go

@@ -0,0 +1,136 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package validation_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"k8s.io/apiextensions-apiserver/pkg/apiserver/validation"
+	"k8s.io/kube-openapi/pkg/validation/spec"
+	"k8s.io/kube-openapi/pkg/validation/strfmt"
+)
+
+var zeroIntSchema *spec.Schema = &spec.Schema{
+	SchemaProps: spec.SchemaProps{
+		Type:    spec.StringOrArray{"number"},
+		Minimum: ptr(float64(0)),
+		Maximum: ptr(float64(0)),
+	},
+}
+
+var smallIntSchema *spec.Schema = &spec.Schema{
+	SchemaProps: spec.SchemaProps{
+		Type:    spec.StringOrArray{"number"},
+		Maximum: ptr(float64(50)),
+	},
+}
+
+var mediumIntSchema *spec.Schema = &spec.Schema{
+	SchemaProps: spec.SchemaProps{
+		Type:    spec.StringOrArray{"number"},
+		Minimum: ptr(float64(50)),
+		Maximum: ptr(float64(10000)),
+	},
+}
+
+var largeIntSchema *spec.Schema = &spec.Schema{
+	SchemaProps: spec.SchemaProps{
+		Type:    spec.StringOrArray{"number"},
+		Minimum: ptr(float64(10000)),
+	},
+}
+
+func TestScalarRatcheting(t *testing.T) {
+	validator := validation.NewRatchetingSchemaValidator(mediumIntSchema, nil, "", strfmt.Default)
+	require.True(t, validator.ValidateUpdate(1, 1).IsValid())
+	require.False(t, validator.ValidateUpdate(1, 2).IsValid())
+}
+
+var objectSchema *spec.Schema = &spec.Schema{
+	SchemaProps: spec.SchemaProps{
+		Type: spec.StringOrArray{"object"},
+		Properties: map[string]spec.Schema{
+			"zero":   *zeroIntSchema,
+			"small":  *smallIntSchema,
+			"medium": *mediumIntSchema,
+			"large":  *largeIntSchema,
+		},
+	},
+}
+
+var objectObjectSchema *spec.Schema = &spec.Schema{
+	SchemaProps: spec.SchemaProps{
+		Type: spec.StringOrArray{"object"},
+		Properties: map[string]spec.Schema{
+			"nested": *objectSchema,
+		},
+	},
+}
+
+// Shows scalar fields of objects can be ratcheted
+func TestObjectScalarFieldsRatcheting(t *testing.T) {
+	validator := validation.NewRatchetingSchemaValidator(objectSchema, nil, "", strfmt.Default)
+	assert.True(t, validator.ValidateUpdate(map[string]interface{}{
+		"small": 500,
+	}, map[string]interface{}{
+		"small": 500,
+	}).IsValid())
+	assert.True(t, validator.ValidateUpdate(map[string]interface{}{
+		"small": 501,
+	}, map[string]interface{}{
+		"small":  501,
+		"medium": 500,
+	}).IsValid())
+	assert.False(t, validator.ValidateUpdate(map[string]interface{}{
+		"small": 500,
+	}, map[string]interface{}{
+		"small": 501,
+	}).IsValid())
+}
+
+// Shows schemas with object fields which themselves are ratcheted can be ratcheted
+func TestObjectObjectFieldsRatcheting(t *testing.T) {
+	validator := validation.NewRatchetingSchemaValidator(objectObjectSchema, nil, "", strfmt.Default)
+	assert.True(t, validator.ValidateUpdate(map[string]interface{}{
+		"nested": map[string]interface{}{
+			"small": 500,
+		}}, map[string]interface{}{
+		"nested": map[string]interface{}{
+			"small": 500,
+		}}).IsValid())
+	assert.True(t, validator.ValidateUpdate(map[string]interface{}{
+		"nested": map[string]interface{}{
+			"small": 501,
+		}}, map[string]interface{}{
+		"nested": map[string]interface{}{
+			"small":  501,
+			"medium": 500,
+		}}).IsValid())
+	assert.False(t, validator.ValidateUpdate(map[string]interface{}{
+		"nested": map[string]interface{}{
+			"small": 500,
+		}}, map[string]interface{}{
+		"nested": map[string]interface{}{
+			"small": 501,
+		}}).IsValid())
+}
+
+func ptr[T any](v T) *T {
+	return &v
+}

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation/validation.go

@@ -22,29 +22,83 @@ import (
 
 	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apiextensions-apiserver/pkg/features"
 	"k8s.io/apimachinery/pkg/util/validation/field"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	openapierrors "k8s.io/kube-openapi/pkg/validation/errors"
 	"k8s.io/kube-openapi/pkg/validation/spec"
 	"k8s.io/kube-openapi/pkg/validation/strfmt"
 	"k8s.io/kube-openapi/pkg/validation/validate"
 )
 
+type SchemaValidator interface {
+	SchemaCreateValidator
+	ValidateUpdate(new, old interface{}) *validate.Result
+}
+
+type SchemaCreateValidator interface {
+	Validate(value interface{}) *validate.Result
+}
+
+// basicSchemaValidator wraps a kube-openapi SchemaCreateValidator to
+// support ValidateUpdate. It implements ValidateUpdate by simply validating
+// the new value via kube-openapi, ignoring the old value.
+type basicSchemaValidator struct {
+	*validate.SchemaValidator
+}
+
+func (s basicSchemaValidator) ValidateUpdate(new, old interface{}) *validate.Result {
+	return s.Validate(new)
+}
+
 // NewSchemaValidator creates an openapi schema validator for the given CRD validation.
-func NewSchemaValidator(customResourceValidation *apiextensions.CustomResourceValidation) (*validate.SchemaValidator, *spec.Schema, error) {
+//
+// If feature `CRDValidationRatcheting` is disabled, this returns validator which
+// validates all `Update`s and `Create`s as a `Create` - without considering old value.
+//
+// If feature `CRDValidationRatcheting` is enabled - the validator returned
+// will support ratcheting unchanged correlatable fields across an update.
+func NewSchemaValidator(customResourceValidation *apiextensions.JSONSchemaProps) (SchemaValidator, *spec.Schema, error) {
 	// Convert CRD schema to openapi schema
 	openapiSchema := &spec.Schema{}
 	if customResourceValidation != nil {
 		// TODO: replace with NewStructural(...).ToGoOpenAPI
-		if err := ConvertJSONSchemaPropsWithPostProcess(customResourceValidation.OpenAPIV3Schema, openapiSchema, StripUnsupportedFormatsPostProcess); err != nil {
+		if err := ConvertJSONSchemaPropsWithPostProcess(customResourceValidation, openapiSchema, StripUnsupportedFormatsPostProcess); err != nil {
 			return nil, nil, err
 		}
 	}
-	return validate.NewSchemaValidator(openapiSchema, nil, "", strfmt.Default), openapiSchema, nil
+
+	if utilfeature.DefaultFeatureGate.Enabled(features.CRDValidationRatcheting) {
+		return NewRatchetingSchemaValidator(openapiSchema, nil, "", strfmt.Default), openapiSchema, nil
+	}
+	return basicSchemaValidator{validate.NewSchemaValidator(openapiSchema, nil, "", strfmt.Default)}, openapiSchema, nil
+}
+
+// ValidateCustomResourceUpdate validates the transition of Custom Resource from
+// `old` to `new` against the schema in the CustomResourceDefinition.
+// Both customResource and old represent a JSON data structures.
+//
+// If feature `CRDValidationRatcheting` is disabled, this behaves identically to
+// ValidateCustomResource(customResource).
+func ValidateCustomResourceUpdate(fldPath *field.Path, customResource, old interface{}, validator SchemaValidator) field.ErrorList {
+	// Additional feature gate check for sanity
+	if !utilfeature.DefaultFeatureGate.Enabled(features.CRDValidationRatcheting) {
+		return ValidateCustomResource(nil, customResource, validator)
+	} else if validator == nil {
+		return nil
+	}
+
+	result := validator.ValidateUpdate(customResource, old)
+	if result.IsValid() {
+		return nil
+	}
+
+	return kubeOpenAPIResultToFieldErrors(fldPath, result)
 }
 
 // ValidateCustomResource validates the Custom Resource against the schema in the CustomResourceDefinition.
 // CustomResource is a JSON data structure.
-func ValidateCustomResource(fldPath *field.Path, customResource interface{}, validator *validate.SchemaValidator) field.ErrorList {
+func ValidateCustomResource(fldPath *field.Path, customResource interface{}, validator SchemaCreateValidator) field.ErrorList {
 	if validator == nil {
 		return nil
 	}
@@ -53,6 +107,11 @@ func ValidateCustomResource(fldPath *field.Path, customResource interface{}, val
 	if result.IsValid() {
 		return nil
 	}
+
+	return kubeOpenAPIResultToFieldErrors(fldPath, result)
+}
+
+func kubeOpenAPIResultToFieldErrors(fldPath *field.Path, result *validate.Result) field.ErrorList {
 	var allErrs field.ErrorList
 	for _, err := range result.Errors {
 		switch err := err.(type) {
@@ -287,7 +346,7 @@ func ConvertJSONSchemaPropsWithPostProcess(in *apiextensions.JSONSchemaProps, ou
 		out.VendorExtensible.AddExtension("x-kubernetes-embedded-resource", true)
 	}
 	if len(in.XListMapKeys) != 0 {
-		out.VendorExtensible.AddExtension("x-kubernetes-list-map-keys", in.XListMapKeys)
+		out.VendorExtensible.AddExtension("x-kubernetes-list-map-keys", convertSliceToInterfaceSlice(in.XListMapKeys))
 	}
 	if in.XListType != nil {
 		out.VendorExtensible.AddExtension("x-kubernetes-list-type", *in.XListType)
@@ -300,11 +359,19 @@ func ConvertJSONSchemaPropsWithPostProcess(in *apiextensions.JSONSchemaProps, ou
 		if err := apiextensionsv1.Convert_apiextensions_ValidationRules_To_v1_ValidationRules(&in.XValidations, &serializationValidationRules, nil); err != nil {
 			return err
 		}
-		out.VendorExtensible.AddExtension("x-kubernetes-validations", serializationValidationRules)
+		out.VendorExtensible.AddExtension("x-kubernetes-validations", convertSliceToInterfaceSlice(serializationValidationRules))
 	}
 	return nil
 }
 
+func convertSliceToInterfaceSlice[T any](in []T) []interface{} {
+	var res []interface{}
+	for _, v := range in {
+		res = append(res, v)
+	}
+	return res
+}
+
 func convertSliceOfJSONSchemaProps(in *[]apiextensions.JSONSchemaProps, out *[]spec.Schema, postProcess PostProcessFunc) error {
 	if in != nil {
 		for _, jsonSchemaProps := range *in {

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation/validation_test.go

@@ -601,7 +601,7 @@ func TestValidateCustomResource(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			validator, _, err := NewSchemaValidator(&apiextensions.CustomResourceValidation{OpenAPIV3Schema: &tt.schema})
+			validator, _, err := NewSchemaValidator(&tt.schema)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -689,7 +689,7 @@ func TestItemsProperty(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			validator, _, err := NewSchemaValidator(&apiextensions.CustomResourceValidation{OpenAPIV3Schema: &tt.args.schema})
+			validator, _, err := NewSchemaValidator(&tt.args.schema)
 			if err != nil {
 				t.Fatal(err)
 			}

staging/src/k8s.io/apiextensions-apiserver/pkg/features/kube_features.go

@@ -22,11 +22,18 @@ import (
 )
 
 const (
-// Every feature gate should add method here following this template:
+	// Every feature gate should add method here following this template:
-//
+	//
-// // owner: @username
+	// // owner: @username
-// // alpha: v1.4
+	// // alpha: v1.4
-// MyFeature() bool
+	// MyFeature() bool
+
+	// owner: @alexzielenski
+	// alpha: v1.28
+	//
+	// Ignores errors raised on unchanged fields of Custom Resources
+	// across UPDATE/PATCH requests.
+	CRDValidationRatcheting featuregate.Feature = "CRDValidationRatcheting"
 )
 
 func init() {
@@ -36,4 +43,6 @@ func init() {
 // defaultKubernetesFeatureGates consists of all known Kubernetes-specific feature keys.
 // To add a new feature, define a key for it above and add it here. The features will be
 // available throughout Kubernetes binaries.
-var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{}
+var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{
+	CRDValidationRatcheting: {Default: false, PreRelease: featuregate.Alpha},
+}

staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/strategy.go

@@ -19,13 +19,12 @@ package customresource
 import (
 	"context"
 
-	"k8s.io/kube-openapi/pkg/validation/validate"
-
 	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
 	structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema"
 	"k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel"
 	structurallisttype "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/listtype"
 	schemaobjectmeta "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/objectmeta"
+	"k8s.io/apiextensions-apiserver/pkg/apiserver/validation"
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -58,7 +57,7 @@ type customResourceStrategy struct {
 	kind              schema.GroupVersionKind
 }
 
-func NewStrategy(typer runtime.ObjectTyper, namespaceScoped bool, kind schema.GroupVersionKind, schemaValidator, statusSchemaValidator *validate.SchemaValidator, structuralSchemas map[string]*structuralschema.Structural, status *apiextensions.CustomResourceSubresourceStatus, scale *apiextensions.CustomResourceSubresourceScale) customResourceStrategy {
+func NewStrategy(typer runtime.ObjectTyper, namespaceScoped bool, kind schema.GroupVersionKind, schemaValidator, statusSchemaValidator validation.SchemaValidator, structuralSchemas map[string]*structuralschema.Structural, status *apiextensions.CustomResourceSubresourceStatus, scale *apiextensions.CustomResourceSubresourceScale) customResourceStrategy {
 	celValidators := map[string]*cel.Validator{}
 	if utilfeature.DefaultFeatureGate.Enabled(features.CustomResourceValidationExpressions) {
 		for name, s := range structuralSchemas {

staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/validator.go

@@ -28,17 +28,16 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/validation/field"
-	"k8s.io/kube-openapi/pkg/validation/validate"
 
 	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
-	apiservervalidation "k8s.io/apiextensions-apiserver/pkg/apiserver/validation"
+	apiextensionsvalidation "k8s.io/apiextensions-apiserver/pkg/apiserver/validation"
 )
 
 type customResourceValidator struct {
 	namespaceScoped       bool
 	kind                  schema.GroupVersionKind
-	schemaValidator       *validate.SchemaValidator
+	schemaValidator       apiextensionsvalidation.SchemaValidator
-	statusSchemaValidator *validate.SchemaValidator
+	statusSchemaValidator apiextensionsvalidation.SchemaValidator
 }
 
 func (a customResourceValidator) Validate(ctx context.Context, obj runtime.Object, scale *apiextensions.CustomResourceSubresourceScale) field.ErrorList {
@@ -58,7 +57,7 @@ func (a customResourceValidator) Validate(ctx context.Context, obj runtime.Objec
 	var allErrs field.ErrorList
 
 	allErrs = append(allErrs, validation.ValidateObjectMetaAccessor(accessor, a.namespaceScoped, validation.NameIsDNSSubdomain, field.NewPath("metadata"))...)
-	allErrs = append(allErrs, apiservervalidation.ValidateCustomResource(nil, u.UnstructuredContent(), a.schemaValidator)...)
+	allErrs = append(allErrs, apiextensionsvalidation.ValidateCustomResource(nil, u.UnstructuredContent(), a.schemaValidator)...)
 	allErrs = append(allErrs, a.ValidateScaleSpec(ctx, u, scale)...)
 	allErrs = append(allErrs, a.ValidateScaleStatus(ctx, u, scale)...)
 
@@ -70,6 +69,10 @@ func (a customResourceValidator) ValidateUpdate(ctx context.Context, obj, old ru
 	if !ok {
 		return field.ErrorList{field.Invalid(field.NewPath(""), u, fmt.Sprintf("has type %T. Must be a pointer to an Unstructured type", u))}
 	}
+	oldU, ok := old.(*unstructured.Unstructured)
+	if !ok {
+		return field.ErrorList{field.Invalid(field.NewPath(""), old, fmt.Sprintf("has type %T. Must be a pointer to an Unstructured type", u))}
+	}
 	objAccessor, err := meta.Accessor(obj)
 	if err != nil {
 		return field.ErrorList{field.Invalid(field.NewPath("metadata"), nil, err.Error())}
@@ -86,7 +89,7 @@ func (a customResourceValidator) ValidateUpdate(ctx context.Context, obj, old ru
 	var allErrs field.ErrorList
 
 	allErrs = append(allErrs, validation.ValidateObjectMetaAccessorUpdate(objAccessor, oldAccessor, field.NewPath("metadata"))...)
-	allErrs = append(allErrs, apiservervalidation.ValidateCustomResource(nil, u.UnstructuredContent(), a.schemaValidator)...)
+	allErrs = append(allErrs, apiextensionsvalidation.ValidateCustomResourceUpdate(nil, u.UnstructuredContent(), oldU.UnstructuredContent(), a.schemaValidator)...)
 	allErrs = append(allErrs, a.ValidateScaleSpec(ctx, u, scale)...)
 	allErrs = append(allErrs, a.ValidateScaleStatus(ctx, u, scale)...)
 
@@ -103,6 +106,10 @@ func (a customResourceValidator) ValidateStatusUpdate(ctx context.Context, obj,
 	if !ok {
 		return field.ErrorList{field.Invalid(field.NewPath(""), u, fmt.Sprintf("has type %T. Must be a pointer to an Unstructured type", u))}
 	}
+	oldU, ok := old.(*unstructured.Unstructured)
+	if !ok {
+		return field.ErrorList{field.Invalid(field.NewPath(""), old, fmt.Sprintf("has type %T. Must be a pointer to an Unstructured type", u))}
+	}
 	objAccessor, err := meta.Accessor(obj)
 	if err != nil {
 		return field.ErrorList{field.Invalid(field.NewPath("metadata"), nil, err.Error())}
@@ -119,7 +126,7 @@ func (a customResourceValidator) ValidateStatusUpdate(ctx context.Context, obj,
 	var allErrs field.ErrorList
 
 	allErrs = append(allErrs, validation.ValidateObjectMetaAccessorUpdate(objAccessor, oldAccessor, field.NewPath("metadata"))...)
-	allErrs = append(allErrs, apiservervalidation.ValidateCustomResource(nil, u.UnstructuredContent(), a.schemaValidator)...)
+	allErrs = append(allErrs, apiextensionsvalidation.ValidateCustomResourceUpdate(nil, u.UnstructuredContent(), oldU, a.schemaValidator)...)
 	allErrs = append(allErrs, a.ValidateScaleStatus(ctx, u, scale)...)
 
 	return allErrs

vendor/modules.txt

@@ -1334,6 +1334,7 @@ k8s.io/apiextensions-apiserver/pkg/controller/openapi/v2
 k8s.io/apiextensions-apiserver/pkg/controller/openapiv3
 k8s.io/apiextensions-apiserver/pkg/controller/status
 k8s.io/apiextensions-apiserver/pkg/crdserverscheme
+k8s.io/apiextensions-apiserver/pkg/features
 k8s.io/apiextensions-apiserver/pkg/generated/openapi
 k8s.io/apiextensions-apiserver/pkg/registry/customresource
 k8s.io/apiextensions-apiserver/pkg/registry/customresource/tableconvertor