Add validation-gen test infrastructure

Introduces the infrastructure for testing validation-gen tags.

Co-authored-by: Tim Hockin <thockin@google.com>
Co-authored-by: Aaron Prindle <aprindle@google.com>
Co-authored-by: Yongrui Lin <yongrlin@google.com>

staging/src/k8s.io/apimachinery/pkg/api/validate/README.md

@@ -0,0 +1,64 @@
+# API validation
+
+This package holds functions which validate fields and types in the Kubernetes
+API. It may be useful beyond API validation, but this is the primary goal.
+
+Most of the public functions here have signatures which adhere to the following
+pattern, which is assumed by automation and code-generation:
+
+```
+import (
+        "context"
+        "k8s.io/apimachinery/pkg/api/operation"
+        "k8s.io/apimachinery/pkg/util/validation/field"
+)
+
+func <Name>(ctx context.Context, op operation.Operation, fldPath *field.Path, value, oldValue <ValueType>, <OtherArgs...>) field.ErrorList
+```
+
+The name of validator functions should consider that callers will generally be
+spelling out the package name and the function name, and so should aim for
+legibility.  E.g. `validate.Concept()`.
+
+The `ctx` argument is Go's usual Context.
+
+The `opCtx` argument provides information about the API operation in question.
+
+The `fldPath` argument indicates the path to the field in question, to be used
+in errors.
+
+The `value` and `oldValue` arguments are the thing(s) being validated.  For
+CREATE operations (`opCtx.Operation == operation.Create`), the `oldValue`
+argument will be nil.  Many validators functions only look at the current value
+(`value`) and disregard `oldValue`.
+
+The `value` and `oldValue` arguments are always nilable - pointers to primitive
+types, slices of any type, or maps of any type.  Validator functions should
+avoid dereferencing nil. Callers are expected to not pass a nil `value` unless the
+API field itself was nilable. `oldValue` is always nil for CREATE operations and 
+is also nil for UPDATE operations if the `value` is not correlated with an `oldValue`.
+
+Simple content-validators may have no `<OtherArgs>`, but validator functions
+may take additional arguments.  Some validator functions will be built as
+generics, e.g. to allow any integer type or to handle arbitrary slices.
+
+Examples:
+
+```
+// NonEmpty validates that a string is not empty.
+func NonEmpty(ctx context.Context, op operation.Operation, fldPath *field.Path, value, _ *string) field.ErrorList
+
+// Even validates that a slice has an even number of items.
+func Even[T any](ctx context.Context, op operation.Operation, fldPath *field.Path, value, _ []T) field.ErrorList
+
+// KeysMaxLen validates that all of the string keys in a map are under the
+// specified length.
+func KeysMaxLen[T any](ctx context.Context, op operation.Operation, fldPath *field.Path, value, _ map[string]T, maxLen int) field.ErrorList
+```
+
+Validator functions always return an `ErrorList` where each item is a distinct
+validation failure and a zero-length return value (not just nil) indicates
+success.
+
+Good validation failure messages follow the Kubernetes API conventions, for
+example using "must" instead of "should".

staging/src/k8s.io/apimachinery/pkg/api/validate/doc.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package validate holds API validation functions which are designed for use
+// with the k8s.io/code-generator/cmd/validation-gen tool.  Each validation
+// function has a similar fingerprint:
+//
+//	func <Name>(ctx context.Context,
+//	            op operation.Operation,
+//	            fldPath *field.Path,
+//	            value, oldValue <nilable type>,
+//	            <other args...>) field.ErrorList
+//
+// The value and oldValue arguments will always be a nilable type.  If the
+// original value was a string, these will be a *string.  If the original value
+// was a slice or map, these will be the same slice or map type.
+//
+// For a CREATE operation, the oldValue will always be nil.  For an UPDATE
+// operation, either value or oldValue may be nil, e.g. when adding or removing
+// a value in a list-map.  Validators which care about UPDATE operations should
+// look at the opCtx argument to know which operation is being executed.
+//
+// Tightened validation (also known as ratcheting validation) is supported by
+// defining a new validation function. For example:
+//
+//	func TightenedMaxLength(ctx context.Context, op operation.Operation, fldPath *field.Path, value, oldValue *string) field.ErrorList {
+//	  if oldValue != nil && len(MaxLength(ctx, op, fldPath, oldValue, nil)) > 0 {
+//	    // old value is not valid, so this value skips the tightened validation
+//	    return nil
+//	  }
+//	  return MaxLength(ctx, op, fldPath, value, nil)
+//	}
+//
+// In general, we cannot distinguish a non-specified slice or map from one that
+// is specified but empty.  Validators should not rely on nil values, but use
+// len() instead.
+package validate

staging/src/k8s.io/apimachinery/pkg/api/validate/testing.go

@@ -0,0 +1,35 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package validate
+
+import (
+	"context"
+
+	"k8s.io/apimachinery/pkg/api/operation"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+)
+
+// FixedResult asserts a fixed boolean result.  This is mostly useful for
+// testing.
+func FixedResult[T any](_ context.Context, op operation.Operation, fldPath *field.Path, value, _ T, result bool, arg string) field.ErrorList {
+	if result {
+		return nil
+	}
+	return field.ErrorList{
+		field.Invalid(fldPath, value, "forced failure: "+arg),
+	}
+}

staging/src/k8s.io/apimachinery/pkg/api/validate/testing_test.go

@@ -0,0 +1,144 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package validate
+
+import (
+	"context"
+	"testing"
+
+	"k8s.io/apimachinery/pkg/api/operation"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+	"k8s.io/utils/ptr"
+)
+
+func TestFixedResult(t *testing.T) {
+	cases := []struct {
+		value any
+		pass  bool
+	}{{
+		value: "",
+		pass:  false,
+	}, {
+		value: "",
+		pass:  true,
+	}, {
+		value: "nonempty",
+		pass:  false,
+	}, {
+		value: "nonempty",
+		pass:  true,
+	}, {
+		value: 0,
+		pass:  false,
+	}, {
+		value: 0,
+		pass:  true,
+	}, {
+		value: 1,
+		pass:  false,
+	}, {
+		value: 1,
+		pass:  true,
+	}, {
+		value: false,
+		pass:  false,
+	}, {
+		value: false,
+		pass:  true,
+	}, {
+		value: true,
+		pass:  false,
+	}, {
+		value: true,
+		pass:  true,
+	}, {
+		value: nil,
+		pass:  false,
+	}, {
+		value: nil,
+		pass:  true,
+	}, {
+		value: ptr.To(""),
+		pass:  false,
+	}, {
+		value: ptr.To(""),
+		pass:  true,
+	}, {
+		value: ptr.To("nonempty"),
+		pass:  false,
+	}, {
+		value: ptr.To("nonempty"),
+		pass:  true,
+	}, {
+		value: []string(nil),
+		pass:  false,
+	}, {
+		value: []string(nil),
+		pass:  true,
+	}, {
+		value: []string{},
+		pass:  false,
+	}, {
+		value: []string{},
+		pass:  true,
+	}, {
+		value: []string{"s"},
+		pass:  false,
+	}, {
+		value: []string{"s"},
+		pass:  true,
+	}, {
+		value: map[string]string(nil),
+		pass:  false,
+	}, {
+		value: map[string]string(nil),
+		pass:  true,
+	}, {
+		value: map[string]string{},
+		pass:  false,
+	}, {
+		value: map[string]string{},
+		pass:  true,
+	}, {
+		value: map[string]string{"k": "v"},
+		pass:  false,
+	}, {
+		value: map[string]string{"k": "v"},
+		pass:  true,
+	}}
+
+	for i, tc := range cases {
+		result := FixedResult(context.Background(), operation.Operation{}, field.NewPath("fldpath"), tc.value, nil, tc.pass, "detail string")
+		if len(result) != 0 && tc.pass {
+			t.Errorf("case %d: unexpected failure: %v", i, fmtErrs(result))
+			continue
+		}
+		if len(result) == 0 && !tc.pass {
+			t.Errorf("case %d: unexpected success", i)
+			continue
+		}
+		if len(result) > 0 {
+			if len(result) > 1 {
+				t.Errorf("case %d: unexepected multi-error: %v", i, fmtErrs(result))
+				continue
+			}
+			if want, got := "forced failure: detail string", result[0].Detail; got != want {
+				t.Errorf("case %d: wrong error, expected: %q, got: %q", i, want, got)
+			}
+		}
+	}
+}

staging/src/k8s.io/code-generator/cmd/validation-gen/output_tests/tags/README.md

@@ -0,0 +1,7 @@
+# Tag tests
+
+Tests in this directory are intended to validate specific tags, rather than
+general behavior of the code generator. Some tags are deeply integrated into
+the code-generation and will end up with similar tests elsewhere.
+
+These test cases should be as focused as possible.

staging/src/k8s.io/code-generator/cmd/validation-gen/output_tests/tags/validate_false/doc.go

@@ -0,0 +1,32 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// +k8s:validation-gen=TypeMeta
+// +k8s:validation-gen-scheme-registry=k8s.io/code-generator/cmd/validation-gen/testscheme.Scheme
+
+// This is a test package.
+package validatefalse
+
+import "k8s.io/code-generator/cmd/validation-gen/testscheme"
+
+var localSchemeBuilder = testscheme.New()
+
+type Struct struct {
+	TypeMeta int
+
+	// +k8s:validateFalse="field Struct.StringField"
+	StringField string `json:"stringField"`
+}

staging/src/k8s.io/code-generator/cmd/validation-gen/output_tests/tags/validate_false/doc_test.go

@@ -0,0 +1,37 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package validatefalse
+
+import (
+	"testing"
+)
+
+func Test(t *testing.T) {
+	st := localSchemeBuilder.Test(t)
+
+	st.Value(&Struct{
+		// All zero-values.
+	}).ExpectValidateFalseByPath(map[string][]string{
+		"stringField": {"field Struct.StringField"},
+	})
+
+	st.Value(&Struct{
+		StringField: "abc",
+	}).ExpectValidateFalseByPath(map[string][]string{
+		"stringField": {"field Struct.StringField"},
+	})
+}

staging/src/k8s.io/code-generator/cmd/validation-gen/output_tests/tags/validate_false/zz_generated.validations.go

@@ -0,0 +1,60 @@
+//go:build !ignore_autogenerated
+// +build !ignore_autogenerated
+
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by validation-gen. DO NOT EDIT.
+
+package validatefalse
+
+import (
+	context "context"
+	fmt "fmt"
+
+	operation "k8s.io/apimachinery/pkg/api/operation"
+	safe "k8s.io/apimachinery/pkg/api/safe"
+	validate "k8s.io/apimachinery/pkg/api/validate"
+	field "k8s.io/apimachinery/pkg/util/validation/field"
+	testscheme "k8s.io/code-generator/cmd/validation-gen/testscheme"
+)
+
+func init() { localSchemeBuilder.Register(RegisterValidations) }
+
+// RegisterValidations adds validation functions to the given scheme.
+// Public to allow building arbitrary schemes.
+func RegisterValidations(scheme *testscheme.Scheme) error {
+	scheme.AddValidationFunc((*Struct)(nil), func(ctx context.Context, op operation.Operation, obj, oldObj interface{}, subresources ...string) field.ErrorList {
+		if len(subresources) == 0 {
+			return Validate_Struct(ctx, op, nil /* fldPath */, obj.(*Struct), safe.Cast[*Struct](oldObj))
+		}
+		return field.ErrorList{field.InternalError(nil, fmt.Errorf("no validation found for %T, subresources: %v", obj, subresources))}
+	})
+	return nil
+}
+
+func Validate_Struct(ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *Struct) (errs field.ErrorList) {
+	// field Struct.TypeMeta has no validation
+
+	// field Struct.StringField
+	errs = append(errs,
+		func(fldPath *field.Path, obj, oldObj *string) (errs field.ErrorList) {
+			errs = append(errs, validate.FixedResult(ctx, op, fldPath, obj, oldObj, false, "field Struct.StringField")...)
+			return
+		}(fldPath.Child("stringField"), &obj.StringField, safe.Field(oldObj, func(oldObj *Struct) *string { return &oldObj.StringField }))...)
+
+	return errs
+}

staging/src/k8s.io/code-generator/cmd/validation-gen/testscheme/doc.go

@@ -0,0 +1,105 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package testscheme provides a scheme implementation and test utilities
+// useful for writing output_tests for validation-gen.
+//
+// For an output test to use this scheme, it should be located in a dedicated go package.
+// The go package should have validation-gen and this test scheme enabled and must declare a
+// 'localSchemeBuilder' using the 'testscheme.New()' function. For example:
+//
+//	// +k8s:validation-gen-scheme-registry=k8s.io/code-generator/cmd/validation-gen/testscheme.Scheme
+//	// +k8s:validation-gen
+//	package example
+//	import "k8s.io/code-generator/cmd/validation-gen/testscheme"
+//	var localSchemeBuilder = testscheme.New()
+//
+// This is sufficient for validation-gen to generate a `zz_generated.validations.go` for the types
+// in the package that compile.
+//
+// With the scheme enabled. An output test may be tested either by handwritten test code or
+// by generated test fixtures.
+//
+// For example, to test by hand. The testschema provides utilities to create a value and assert
+// that the expected errors are returned when the value is validated:
+//
+//	 func Test(t *testing.T) {
+//		  st := localSchemeBuilder.Test(t)
+//		  st.Value(&T1{
+//			E0:  "x",
+//			PE0: pointer.To(E0("y")),
+//		  }).ExpectInvalid(
+//			field.NotSupported(field.NewPath("e0"), "x", []string{EnumValue1, EnumValue2}),
+//			field.NotSupported(field.NewPath("pe0"), "y", []string{EnumValue1, EnumValue2}))}
+//	 }
+//
+// Tests fixtures can also be enabled. For example:
+//
+//	// ...
+//	// +k8s:validation-gen-test-fixture=validateFalse
+//	// package example
+//
+// When a test fixture is enabled, a `zz_generated.validations_test.go` file will be generated
+// test for all the registered types in the package according to the behavior of the named test
+// fixture(s).
+//
+// Test Fixtures:
+//
+// `validateFalse` - This test fixture executes validation of each registered type and accumulates
+// all `validateFalse` validation errors. For example:
+//
+//	type T1 struct {
+//		// +k8s:validateFalse="field T1.S"
+//		S string `json:"s"`
+//		// +k8s:validateFalse="field T1.T"
+//		T T2 `json:"t"`
+//	}
+//
+// The above `example.T1` test type has two validated fields: `s` and 't'. The fields are named
+// according to the Go "json" field tag, and each has a validation error identifier
+// provided by `+k8s:validateFalse=<identifier>`.
+//
+// The `validateFalse` test fixture will validate an instance of `example.T1`, generated using fuzzed
+// data, and then group the validation errors by field name. Represented in JSON like:
+//
+//	{
+//	  "*example.T1": {
+//	    "s": [
+//	      "field T1.S"
+//	    ],
+//	    "t": [
+//	      "field T1.T"
+//	    ]
+//	  }
+//	}
+//
+// This validation error data contains an object for each registered type, keyed by type name.
+// For each registered type, the validation errors from `+k8s:validateFalse` tags are grouped
+// by field path with all validation error identifiers collected into a list.
+//
+// This data is compared with the expected validation results that are defined in
+// a `testdata/validate-false.json` file in the same package, and any differences are
+// reported as test errors.
+//
+// `testdata/validate-false.json` can be generated automatically by setting the
+// `UPDATE_VALIDATION_GEN_FIXTURE_DATA=true` environment variable to true when running the tests.
+//
+// Test authors that generated `testdata/validate-false.json` are expected to ensure that file
+// is correct before checking it in to source control.
+//
+// The fuzzed data is generated pseudo-randomly with a consistent seed, with all nilable fields se
+// to a value, and with a single entry for each map  and a single element for each slice.
+package testscheme

staging/src/k8s.io/code-generator/cmd/validation-gen/testscheme/testscheme.go

@@ -0,0 +1,537 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testscheme
+
+import (
+	"bytes"
+	stdcmp "cmp"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"math/rand"
+	"os"
+	"path"
+	"reflect"
+	"regexp"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"         // nolint:depguard // this package provides test utilities
+	"github.com/google/go-cmp/cmp/cmpopts" // nolint:depguard // this package provides test utilities
+	fuzz "github.com/google/gofuzz"
+
+	"k8s.io/apimachinery/pkg/api/operation"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+)
+
+// Scheme is similar to runtime.Scheme, but for validation testing purposes. Scheme only supports validation,
+// supports registration of any type (not just runtime.Object) and implements Register directly, allowing it
+// to also be used as a scheme builder.
+// Must only be used with tests that perform all registration before calls to validate.
+type Scheme struct {
+	validationFuncs    map[reflect.Type]func(ctx context.Context, op operation.Operation, object, oldObject interface{}, subresources ...string) field.ErrorList
+	registrationErrors field.ErrorList
+}
+
+// New creates a new Scheme.
+func New() *Scheme {
+	return &Scheme{validationFuncs: map[reflect.Type]func(ctx context.Context, op operation.Operation, object interface{}, oldObject interface{}, subresources ...string) field.ErrorList{}}
+}
+
+// AddValidationFunc registers a validation function.
+// Last writer wins.
+func (s *Scheme) AddValidationFunc(srcType any, fn func(ctx context.Context, op operation.Operation, object, oldObject interface{}, subresources ...string) field.ErrorList) {
+	s.validationFuncs[reflect.TypeOf(srcType)] = fn
+}
+
+// Validate validates an object using the registered validation function.
+func (s *Scheme) Validate(ctx context.Context, opts sets.Set[string], object any, subresources ...string) field.ErrorList {
+	if len(s.registrationErrors) > 0 {
+		return s.registrationErrors // short circuit with registration errors if any are present
+	}
+	if fn, ok := s.validationFuncs[reflect.TypeOf(object)]; ok {
+		return fn(ctx, operation.Operation{Type: operation.Create, Options: opts}, object, nil, subresources...)
+	}
+	return nil
+}
+
+// ValidateUpdate validates an update to an object using the registered validation function.
+func (s *Scheme) ValidateUpdate(ctx context.Context, opts sets.Set[string], object, oldObject any, subresources ...string) field.ErrorList {
+	if len(s.registrationErrors) > 0 {
+		return s.registrationErrors // short circuit with registration errors if any are present
+	}
+	if fn, ok := s.validationFuncs[reflect.TypeOf(object)]; ok {
+		return fn(ctx, operation.Operation{Type: operation.Update}, object, oldObject, subresources...)
+	}
+	return nil
+}
+
+// Register adds a scheme setup function to the list.
+func (s *Scheme) Register(funcs ...func(*Scheme) error) {
+	for _, f := range funcs {
+		err := f(s)
+		if err != nil {
+			s.registrationErrors = append(s.registrationErrors, toRegistrationError(err))
+		}
+	}
+}
+
+func toRegistrationError(err error) *field.Error {
+	return field.InternalError(nil, fmt.Errorf("registration error: %w", err))
+}
+
+// Test returns a ValidationTestBuilder for this scheme.
+func (s *Scheme) Test(t *testing.T) *ValidationTestBuilder {
+	return &ValidationTestBuilder{t, s}
+}
+
+// ValidationTestBuilder provides convenience functions to build
+// validation tests.
+type ValidationTestBuilder struct {
+	*testing.T
+	s *Scheme
+}
+
+const fixtureEnvVar = "UPDATE_VALIDATION_GEN_FIXTURE_DATA"
+
+// ValidateFixtures ensures that the validation errors of all registered types match what is expected by the test fixture files.
+// For each registered type, a value is created for the type, and populated by fuzzing the value, before validating the type.
+// See ValueFuzzed for details.
+//
+// If the UPDATE_VALIDATION_GEN_FIXTURE_DATA=true environment variable is set, test fixture files are created or overridden.
+//
+// Fixtures:
+//   - validate-false.json: defines a map of registered type to a map of field path to  +validateFalse validations args
+//     that are expected to be returned as errors when the type is validated.
+func (s *ValidationTestBuilder) ValidateFixtures() {
+	s.T.Helper()
+
+	flag := os.Getenv(fixtureEnvVar)
+	// Run validation
+	got := map[string]map[string][]string{}
+	for t := range s.s.validationFuncs {
+		var v any
+		// TODO: this should handle maps and slices
+		if t.Kind() == reflect.Ptr {
+			v = reflect.New(t.Elem()).Interface()
+		} else {
+			v = reflect.Indirect(reflect.New(t)).Interface()
+		}
+		if reflect.TypeOf(v).Kind() != reflect.Ptr {
+			v = &v
+		}
+		s.ValueFuzzed(v)
+		vt := &ValidationTester{ValidationTestBuilder: s, value: v}
+		byPath := vt.ValidateFalseArgsByPath()
+		got[t.String()] = byPath
+	}
+
+	testdataFilename := "testdata/validate-false.json"
+	if flag == "true" {
+		// Generate fixture file
+		if err := os.MkdirAll(path.Dir(testdataFilename), os.FileMode(0755)); err != nil {
+			s.Fatal("error making directory", err)
+		}
+		data, err := json.MarshalIndent(got, "  ", "  ")
+		if err != nil {
+			s.Fatal(err)
+		}
+		err = os.WriteFile(testdataFilename, data, os.FileMode(0644))
+		if err != nil {
+			s.Fatal(err)
+		}
+	} else {
+		// Load fixture file
+		testdataFile, err := os.Open(testdataFilename)
+		if errors.Is(err, os.ErrNotExist) {
+			s.Fatalf("%s test fixture data not found. Run go test with the environment variable %s=true to create test fixture data.",
+				testdataFilename, fixtureEnvVar)
+		} else if err != nil {
+			s.Fatal(err)
+		}
+		defer func() {
+			err := testdataFile.Close()
+			if err != nil {
+				s.Fatal(err)
+			}
+		}()
+
+		byteValue, err := io.ReadAll(testdataFile)
+		if err != nil {
+			s.Fatal(err)
+		}
+		testdata := map[string]map[string][]string{}
+		err = json.Unmarshal(byteValue, &testdata)
+		if err != nil {
+			s.Fatal(err)
+		}
+		// Compare fixture with validation results
+		expectedKeys := sets.New[string]()
+		gotKeys := sets.New[string]()
+		for k := range got {
+			gotKeys.Insert(k)
+		}
+		hasErrors := false
+		for k, expectedForType := range testdata {
+			expectedKeys.Insert(k)
+			gotForType, ok := got[k]
+			s.T.Run(k, func(t *testing.T) {
+				t.Helper()
+
+				if !ok {
+					t.Errorf("%q has expected validateFalse args in %s but got no validation errors.", k, testdataFilename)
+					hasErrors = true
+				} else if !cmp.Equal(gotForType, expectedForType) {
+					t.Errorf("validateFalse args, grouped by field path, differed from %s:\n%s\n",
+						testdataFilename, cmp.Diff(gotForType, expectedForType, cmpopts.SortMaps(stdcmp.Less[string])))
+					hasErrors = true
+				}
+			})
+		}
+		for unexpectedType := range gotKeys.Difference(expectedKeys) {
+			s.T.Run(unexpectedType, func(t *testing.T) {
+				t.Helper()
+
+				t.Errorf("%q got unexpected validateFalse args, grouped by field path:\n%s\n",
+					unexpectedType, cmp.Diff(nil, got[unexpectedType], cmpopts.SortMaps(stdcmp.Less[string])))
+				hasErrors = true
+			})
+		}
+		if hasErrors {
+			s.T.Logf("If the test expectations have changed, run go test with the environment variable %s=true", fixtureEnvVar)
+		}
+	}
+}
+
+func fuzzer() *fuzz.Fuzzer {
+	// Ensure that lists and maps are not empty and use a deterministic seed.
+	return fuzz.New().NilChance(0.0).NumElements(2, 2).RandSource(rand.NewSource(0))
+}
+
+// ValueFuzzed automatically populates the given value using a deterministic fuzzer.
+// The fuzzer sets pointers to values and always includes a two map keys and slice elements.
+func (s *ValidationTestBuilder) ValueFuzzed(value any) *ValidationTester {
+	fuzzer().Fuzz(value)
+	return &ValidationTester{ValidationTestBuilder: s, value: value}
+}
+
+// Value returns a ValidationTester for the given value. The value
+// must be a registered with the scheme for validation.
+func (s *ValidationTestBuilder) Value(value any) *ValidationTester {
+	return &ValidationTester{ValidationTestBuilder: s, value: value}
+}
+
+// ValidationTester provides convenience functions to define validation
+// tests for a validatable value.
+type ValidationTester struct {
+	*ValidationTestBuilder
+	value    any
+	oldValue any
+	opts     sets.Set[string]
+}
+
+// OldValue sets the oldValue for this ValidationTester. When oldValue is set to
+// a non-nil value, update validation will be used to test validation.
+// oldValue must be the same type as value.
+// Returns ValidationTester to support call chaining.
+func (v *ValidationTester) OldValue(oldValue any) *ValidationTester {
+	v.oldValue = oldValue
+	return v
+}
+
+// OldValueFuzzed automatically populates the given value using a deterministic fuzzer.
+// The fuzzer sets pointers to values and always includes a two map keys and slice elements.
+func (v *ValidationTester) OldValueFuzzed(oldValue any) *ValidationTester {
+	fuzzer().Fuzz(oldValue)
+	v.oldValue = oldValue
+	return v
+}
+
+// Opts sets the ValidationOpts to use.
+func (v *ValidationTester) Opts(opts sets.Set[string]) *ValidationTester {
+	v.opts = opts
+	return v
+}
+
+func multiline(errs field.ErrorList) string {
+	if len(errs) == 0 {
+		return "<no errors>"
+	}
+	if len(errs) == 1 {
+		return errs[0].Error()
+	}
+
+	var buf bytes.Buffer
+	for _, err := range errs {
+		buf.WriteString("\n")
+		buf.WriteString(err.Error())
+	}
+	return buf.String()
+}
+
+// ExpectValid validates the value and calls t.Errorf if any validation errors are returned.
+// Returns ValidationTester to support call chaining.
+func (v *ValidationTester) ExpectValid() *ValidationTester {
+	v.T.Helper()
+
+	v.T.Run(fmt.Sprintf("%T", v.value), func(t *testing.T) {
+		t.Helper()
+
+		errs := v.validate()
+		if len(errs) > 0 {
+			t.Errorf("want no errors, got: %v", multiline(errs))
+		}
+	})
+	return v
+}
+
+// ExpectValidAt validates the value and calls t.Errorf for any validation errors at the given path.
+// Returns ValidationTester to support call chaining.
+func (v *ValidationTester) ExpectValidAt(fldPath *field.Path) *ValidationTester {
+	v.T.Helper()
+
+	v.T.Run(fmt.Sprintf("%T.%v", v.value, fldPath), func(t *testing.T) {
+		t.Helper()
+
+		var got field.ErrorList
+		for _, e := range v.validate() {
+			if e.Field == fldPath.String() {
+				got = append(got, e)
+			}
+		}
+		if len(got) > 0 {
+			t.Errorf("want no errors at %v, got: %v", fldPath, got)
+		}
+	})
+	return v
+}
+
+// ExpectInvalid validates the value and calls t.Errorf if want does not match the actual errors.
+// Returns ValidationTester to support call chaining.
+func (v *ValidationTester) ExpectInvalid(want ...*field.Error) *ValidationTester {
+	v.T.Helper()
+
+	return v.expectInvalid(byFullError, want...)
+}
+
+// ExpectValidateFalse validates the value and calls t.Errorf if the actual errors do not
+// match the given validateFalseArgs.  For example, if the value to validate has a
+// single `+validateFalse="type T1"` tag, ExpectValidateFalse("type T1") will pass.
+// Returns ValidationTester to support call chaining.
+func (v *ValidationTester) ExpectValidateFalse(validateFalseArgs ...string) *ValidationTester {
+	v.T.Helper()
+
+	var want []*field.Error
+	for _, s := range validateFalseArgs {
+		want = append(want, field.Invalid(nil, "", fmt.Sprintf("forced failure: %s", s)))
+	}
+	return v.expectInvalid(byDetail, want...)
+}
+
+func (v *ValidationTester) ExpectValidateFalseByPath(validateFalseArgsByPath map[string][]string) *ValidationTester {
+	v.T.Helper()
+
+	v.T.Run(fmt.Sprintf("%T", v.value), func(t *testing.T) {
+		t.Helper()
+
+		byPath := v.ValidateFalseArgsByPath()
+		// ensure args are sorted
+		for _, args := range validateFalseArgsByPath {
+			sort.Strings(args)
+		}
+		if !cmp.Equal(validateFalseArgsByPath, byPath) {
+			t.Errorf("validateFalse args, grouped by field path, differed from expected:\n%s\n", cmp.Diff(validateFalseArgsByPath, byPath, cmpopts.SortMaps(stdcmp.Less[string])))
+		}
+
+	})
+	return v
+}
+
+func (v *ValidationTester) ValidateFalseArgsByPath() map[string][]string {
+	byPath := map[string][]string{}
+	errs := v.validate()
+	for _, e := range errs {
+		if strings.HasPrefix(e.Detail, "forced failure: ") {
+			arg := strings.TrimPrefix(e.Detail, "forced failure: ")
+			f := e.Field
+			if f == "<nil>" {
+				f = ""
+			}
+			byPath[f] = append(byPath[f], arg)
+		}
+	}
+	// ensure args are sorted
+	for _, args := range byPath {
+		sort.Strings(args)
+	}
+	return byPath
+}
+
+func (v *ValidationTester) ExpectRegexpsByPath(regexpStringsByPath map[string][]string) *ValidationTester {
+	v.T.Helper()
+
+	v.T.Run(fmt.Sprintf("%T", v.value), func(t *testing.T) {
+		t.Helper()
+
+		errorsByPath := v.getErrorsByPath()
+
+		// sanity check
+		if want, got := len(regexpStringsByPath), len(errorsByPath); got != want {
+			t.Fatalf("wrong number of error-fields: expected %d, got %d:\nwanted:\n%sgot:\n%s",
+				want, got, renderByPath(regexpStringsByPath), renderByPath(errorsByPath))
+		}
+
+		// compile regexps
+		regexpsByPath := map[string][]*regexp.Regexp{}
+		for field, strs := range regexpStringsByPath {
+			regexps := make([]*regexp.Regexp, 0, len(strs))
+			for _, str := range strs {
+				regexps = append(regexps, regexp.MustCompile(str))
+			}
+			regexpsByPath[field] = regexps
+		}
+
+		for field := range errorsByPath {
+			errors := errorsByPath[field]
+			regexps := regexpsByPath[field]
+
+			// sanity check
+			if want, got := len(regexps), len(errors); got != want {
+				t.Fatalf("field %q: wrong number of errors: expected %d, got %d:\nwanted:\n%sgot:\n%s",
+					field, want, got, renderList(regexpStringsByPath[field]), renderList(errors))
+			}
+
+			// build a set of errors and expectations, so we can track them,
+			expSet := sets.New(regexps...)
+
+			for _, err := range errors {
+				var found *regexp.Regexp
+				for _, re := range regexps {
+					if re.MatchString(err) {
+						found = re
+						break // done with regexps
+					}
+				}
+				if found != nil {
+					expSet.Delete(found)
+					continue // next error
+				}
+				t.Errorf("field %q, error %q did not match any expectation", field, err)
+			}
+			if len(expSet) != 0 {
+				t.Errorf("field %q had unsatisfied expectations: %q", field, expSet.UnsortedList())
+			}
+		}
+	})
+	return v
+}
+
+func (v *ValidationTester) getErrorsByPath() map[string][]string {
+	byPath := map[string][]string{}
+	errs := v.validate()
+	for _, e := range errs {
+		f := e.Field
+		if f == "<nil>" {
+			f = ""
+		}
+		byPath[f] = append(byPath[f], e.ErrorBody())
+	}
+	// ensure args are sorted
+	for _, args := range byPath {
+		sort.Strings(args)
+	}
+	return byPath
+}
+
+func renderByPath(byPath map[string][]string) string {
+	keys := []string{}
+	for key := range byPath {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+
+	for _, vals := range byPath {
+		sort.Strings(vals)
+	}
+
+	buf := strings.Builder{}
+	for _, key := range keys {
+		vals := byPath[key]
+		for _, val := range vals {
+			buf.WriteString(fmt.Sprintf("\t%s: %q\n", key, val))
+		}
+	}
+	return buf.String()
+}
+
+func renderList(list []string) string {
+	buf := strings.Builder{}
+	for _, item := range list {
+		buf.WriteString(fmt.Sprintf("\t%q\n", item))
+	}
+	return buf.String()
+}
+
+func (v *ValidationTester) expectInvalid(matcher matcher, errs ...*field.Error) *ValidationTester {
+	v.T.Helper()
+
+	v.T.Run(fmt.Sprintf("%T", v.value), func(t *testing.T) {
+		t.Helper()
+
+		want := sets.New[string]()
+		for _, e := range errs {
+			want.Insert(matcher(e))
+		}
+
+		got := sets.New[string]()
+		for _, e := range v.validate() {
+			got.Insert(matcher(e))
+		}
+		if !got.Equal(want) {
+			t.Errorf("validation errors differed from expected:\n%v\n", cmp.Diff(want, got, cmpopts.SortMaps(stdcmp.Less[string])))
+
+			for x := range got.Difference(want) {
+				fmt.Printf("%q,\n", strings.TrimPrefix(x, "forced failure: "))
+			}
+		}
+	})
+	return v
+}
+
+type matcher func(err *field.Error) string
+
+func byDetail(err *field.Error) string {
+	return err.Detail
+}
+
+func byFullError(err *field.Error) string {
+	return err.Error()
+}
+
+func (v *ValidationTester) validate() field.ErrorList {
+	var errs field.ErrorList
+	if v.oldValue == nil {
+		errs = v.s.Validate(context.Background(), v.opts, v.value)
+	} else {
+		errs = v.s.ValidateUpdate(context.Background(), v.opts, v.value, v.oldValue)
+	}
+	return errs
+}

staging/src/k8s.io/code-generator/cmd/validation-gen/validators/testing.go

@@ -0,0 +1,177 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package validators
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/gengo/v2/types"
+)
+
+const (
+	// These tags return a fixed pass/fail state.
+	validateTrueTagName  = "k8s:validateTrue"
+	validateFalseTagName = "k8s:validateFalse"
+
+	// This tag always returns an error from ExtractValidations.
+	validateErrorTagName = "k8s:validateError"
+)
+
+func init() {
+	RegisterTagValidator(fixedResultTagValidator{result: true})
+	RegisterTagValidator(fixedResultTagValidator{result: false})
+	RegisterTagValidator(fixedResultTagValidator{error: true})
+}
+
+type fixedResultTagValidator struct {
+	result bool
+	error  bool
+}
+
+func (fixedResultTagValidator) Init(_ Config) {}
+
+func (frtv fixedResultTagValidator) TagName() string {
+	if frtv.error {
+		return validateErrorTagName
+	} else if frtv.result {
+		return validateTrueTagName
+	}
+	return validateFalseTagName
+}
+
+var fixedResultTagValidScopes = sets.New(ScopeAny)
+
+func (fixedResultTagValidator) ValidScopes() sets.Set[Scope] {
+	return fixedResultTagValidScopes
+}
+
+func (frtv fixedResultTagValidator) GetValidations(context Context, _ []string, payload string) (Validations, error) {
+	var result Validations
+
+	if frtv.error {
+		return result, fmt.Errorf("forced error: %q", payload)
+	}
+
+	tag, err := frtv.parseTagPayload(payload)
+	if err != nil {
+		return result, fmt.Errorf("can't decode tag payload: %w", err)
+	}
+	result.AddFunction(GenericFunction(frtv.TagName(), tag.flags, fixedResultValidator, tag.typeArgs, frtv.result, tag.msg))
+
+	return result, nil
+}
+
+var (
+	fixedResultValidator = types.Name{Package: libValidationPkg, Name: "FixedResult"}
+)
+
+type fixedResultPayload struct {
+	flags    FunctionFlags
+	msg      string
+	typeArgs []types.Name
+}
+
+func (fixedResultTagValidator) parseTagPayload(in string) (fixedResultPayload, error) {
+	type payload struct {
+		Flags   []string `json:"flags"`
+		Msg     string   `json:"msg"`
+		TypeArg string   `json:"typeArg,omitempty"`
+	}
+	// We expect either a string (maybe empty) or a JSON object.
+	if len(in) == 0 {
+		return fixedResultPayload{}, nil
+	}
+	var pl payload
+	if err := json.Unmarshal([]byte(in), &pl); err != nil {
+		s := ""
+		if err := json.Unmarshal([]byte(in), &s); err != nil {
+			return fixedResultPayload{}, fmt.Errorf("error parsing JSON value: %w (%q)", err, in)
+		}
+		return fixedResultPayload{msg: s}, nil
+	}
+	// The msg field is required in JSON mode.
+	if pl.Msg == "" {
+		return fixedResultPayload{}, fmt.Errorf("JSON msg is required")
+	}
+	var flags FunctionFlags
+	for _, fl := range pl.Flags {
+		switch fl {
+		case "ShortCircuit":
+			flags |= ShortCircuit
+		case "NonError":
+			flags |= NonError
+		default:
+			return fixedResultPayload{}, fmt.Errorf("unknown flag: %q", fl)
+		}
+	}
+	var typeArgs []types.Name
+	if tn := pl.TypeArg; len(tn) > 0 {
+		if !strings.HasPrefix(tn, "*") {
+			tn = "*" + tn // We always need the pointer type.
+		}
+		typeArgs = []types.Name{{Package: "", Name: tn}}
+	}
+
+	return fixedResultPayload{flags, pl.Msg, typeArgs}, nil
+}
+
+func (frtv fixedResultTagValidator) Docs() TagDoc {
+	doc := TagDoc{
+		Tag:    frtv.TagName(),
+		Scopes: frtv.ValidScopes().UnsortedList(),
+	}
+	if frtv.error {
+		doc.Description = "Always fails code generation (useful for testing)."
+		doc.Payloads = []TagPayloadDoc{{
+			Description: "<string>",
+			Docs:        "This string will be included in the error message.",
+		}}
+	} else {
+		// True and false have the same payloads.
+		doc.Payloads = []TagPayloadDoc{{
+			Description: "<none>",
+		}, {
+			Description: "<quoted-string>",
+			Docs:        "The generated code will include this string.",
+		}, {
+			Description: "<json-object>",
+			Docs:        "",
+			Schema: []TagPayloadSchema{{
+				Key:   "flags",
+				Value: "<list-of-flag-string>",
+				Docs:  `values: ShortCircuit, NonError`,
+			}, {
+				Key:   "msg",
+				Value: "<string>",
+				Docs:  "The generated code will include this string.",
+			}, {
+				Key:   "typeArg",
+				Value: "<string>",
+				Docs:  "The type arg in generated code (must be the value-type, not pointer).",
+			}},
+		}}
+		if frtv.result {
+			doc.Description = "Always passes validation (useful for testing)."
+		} else {
+			doc.Description = "Always fails validation (useful for testing)."
+		}
+	}
+	return doc
+}