diff --git a/staging/src/k8s.io/apimachinery/pkg/api/validate/README.md b/staging/src/k8s.io/apimachinery/pkg/api/validate/README.md new file mode 100644 index 00000000000..52ca031d44e --- /dev/null +++ b/staging/src/k8s.io/apimachinery/pkg/api/validate/README.md @@ -0,0 +1,64 @@ +# API validation + +This package holds functions which validate fields and types in the Kubernetes +API. It may be useful beyond API validation, but this is the primary goal. + +Most of the public functions here have signatures which adhere to the following +pattern, which is assumed by automation and code-generation: + +``` +import ( + "context" + "k8s.io/apimachinery/pkg/api/operation" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +func (ctx context.Context, op operation.Operation, fldPath *field.Path, value, oldValue , ) field.ErrorList +``` + +The name of validator functions should consider that callers will generally be +spelling out the package name and the function name, and so should aim for +legibility. E.g. `validate.Concept()`. + +The `ctx` argument is Go's usual Context. + +The `opCtx` argument provides information about the API operation in question. + +The `fldPath` argument indicates the path to the field in question, to be used +in errors. + +The `value` and `oldValue` arguments are the thing(s) being validated. For +CREATE operations (`opCtx.Operation == operation.Create`), the `oldValue` +argument will be nil. Many validators functions only look at the current value +(`value`) and disregard `oldValue`. + +The `value` and `oldValue` arguments are always nilable - pointers to primitive +types, slices of any type, or maps of any type. Validator functions should +avoid dereferencing nil. Callers are expected to not pass a nil `value` unless the +API field itself was nilable. `oldValue` is always nil for CREATE operations and +is also nil for UPDATE operations if the `value` is not correlated with an `oldValue`. + +Simple content-validators may have no ``, but validator functions +may take additional arguments. Some validator functions will be built as +generics, e.g. to allow any integer type or to handle arbitrary slices. + +Examples: + +``` +// NonEmpty validates that a string is not empty. +func NonEmpty(ctx context.Context, op operation.Operation, fldPath *field.Path, value, _ *string) field.ErrorList + +// Even validates that a slice has an even number of items. +func Even[T any](ctx context.Context, op operation.Operation, fldPath *field.Path, value, _ []T) field.ErrorList + +// KeysMaxLen validates that all of the string keys in a map are under the +// specified length. +func KeysMaxLen[T any](ctx context.Context, op operation.Operation, fldPath *field.Path, value, _ map[string]T, maxLen int) field.ErrorList +``` + +Validator functions always return an `ErrorList` where each item is a distinct +validation failure and a zero-length return value (not just nil) indicates +success. + +Good validation failure messages follow the Kubernetes API conventions, for +example using "must" instead of "should". diff --git a/staging/src/k8s.io/apimachinery/pkg/api/validate/doc.go b/staging/src/k8s.io/apimachinery/pkg/api/validate/doc.go new file mode 100644 index 00000000000..eee13e9b38b --- /dev/null +++ b/staging/src/k8s.io/apimachinery/pkg/api/validate/doc.go @@ -0,0 +1,50 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package validate holds API validation functions which are designed for use +// with the k8s.io/code-generator/cmd/validation-gen tool. Each validation +// function has a similar fingerprint: +// +// func (ctx context.Context, +// op operation.Operation, +// fldPath *field.Path, +// value, oldValue , +// ) field.ErrorList +// +// The value and oldValue arguments will always be a nilable type. If the +// original value was a string, these will be a *string. If the original value +// was a slice or map, these will be the same slice or map type. +// +// For a CREATE operation, the oldValue will always be nil. For an UPDATE +// operation, either value or oldValue may be nil, e.g. when adding or removing +// a value in a list-map. Validators which care about UPDATE operations should +// look at the opCtx argument to know which operation is being executed. +// +// Tightened validation (also known as ratcheting validation) is supported by +// defining a new validation function. For example: +// +// func TightenedMaxLength(ctx context.Context, op operation.Operation, fldPath *field.Path, value, oldValue *string) field.ErrorList { +// if oldValue != nil && len(MaxLength(ctx, op, fldPath, oldValue, nil)) > 0 { +// // old value is not valid, so this value skips the tightened validation +// return nil +// } +// return MaxLength(ctx, op, fldPath, value, nil) +// } +// +// In general, we cannot distinguish a non-specified slice or map from one that +// is specified but empty. Validators should not rely on nil values, but use +// len() instead. +package validate diff --git a/staging/src/k8s.io/apimachinery/pkg/api/validate/testing.go b/staging/src/k8s.io/apimachinery/pkg/api/validate/testing.go new file mode 100644 index 00000000000..722cfb544cc --- /dev/null +++ b/staging/src/k8s.io/apimachinery/pkg/api/validate/testing.go @@ -0,0 +1,35 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validate + +import ( + "context" + + "k8s.io/apimachinery/pkg/api/operation" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +// FixedResult asserts a fixed boolean result. This is mostly useful for +// testing. +func FixedResult[T any](_ context.Context, op operation.Operation, fldPath *field.Path, value, _ T, result bool, arg string) field.ErrorList { + if result { + return nil + } + return field.ErrorList{ + field.Invalid(fldPath, value, "forced failure: "+arg), + } +} diff --git a/staging/src/k8s.io/apimachinery/pkg/api/validate/testing_test.go b/staging/src/k8s.io/apimachinery/pkg/api/validate/testing_test.go new file mode 100644 index 00000000000..ef8c13c8bf2 --- /dev/null +++ b/staging/src/k8s.io/apimachinery/pkg/api/validate/testing_test.go @@ -0,0 +1,144 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validate + +import ( + "context" + "testing" + + "k8s.io/apimachinery/pkg/api/operation" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/utils/ptr" +) + +func TestFixedResult(t *testing.T) { + cases := []struct { + value any + pass bool + }{{ + value: "", + pass: false, + }, { + value: "", + pass: true, + }, { + value: "nonempty", + pass: false, + }, { + value: "nonempty", + pass: true, + }, { + value: 0, + pass: false, + }, { + value: 0, + pass: true, + }, { + value: 1, + pass: false, + }, { + value: 1, + pass: true, + }, { + value: false, + pass: false, + }, { + value: false, + pass: true, + }, { + value: true, + pass: false, + }, { + value: true, + pass: true, + }, { + value: nil, + pass: false, + }, { + value: nil, + pass: true, + }, { + value: ptr.To(""), + pass: false, + }, { + value: ptr.To(""), + pass: true, + }, { + value: ptr.To("nonempty"), + pass: false, + }, { + value: ptr.To("nonempty"), + pass: true, + }, { + value: []string(nil), + pass: false, + }, { + value: []string(nil), + pass: true, + }, { + value: []string{}, + pass: false, + }, { + value: []string{}, + pass: true, + }, { + value: []string{"s"}, + pass: false, + }, { + value: []string{"s"}, + pass: true, + }, { + value: map[string]string(nil), + pass: false, + }, { + value: map[string]string(nil), + pass: true, + }, { + value: map[string]string{}, + pass: false, + }, { + value: map[string]string{}, + pass: true, + }, { + value: map[string]string{"k": "v"}, + pass: false, + }, { + value: map[string]string{"k": "v"}, + pass: true, + }} + + for i, tc := range cases { + result := FixedResult(context.Background(), operation.Operation{}, field.NewPath("fldpath"), tc.value, nil, tc.pass, "detail string") + if len(result) != 0 && tc.pass { + t.Errorf("case %d: unexpected failure: %v", i, fmtErrs(result)) + continue + } + if len(result) == 0 && !tc.pass { + t.Errorf("case %d: unexpected success", i) + continue + } + if len(result) > 0 { + if len(result) > 1 { + t.Errorf("case %d: unexepected multi-error: %v", i, fmtErrs(result)) + continue + } + if want, got := "forced failure: detail string", result[0].Detail; got != want { + t.Errorf("case %d: wrong error, expected: %q, got: %q", i, want, got) + } + } + } +} diff --git a/staging/src/k8s.io/code-generator/cmd/validation-gen/output_tests/tags/README.md b/staging/src/k8s.io/code-generator/cmd/validation-gen/output_tests/tags/README.md new file mode 100644 index 00000000000..d1b6b0a3099 --- /dev/null +++ b/staging/src/k8s.io/code-generator/cmd/validation-gen/output_tests/tags/README.md @@ -0,0 +1,7 @@ +# Tag tests + +Tests in this directory are intended to validate specific tags, rather than +general behavior of the code generator. Some tags are deeply integrated into +the code-generation and will end up with similar tests elsewhere. + +These test cases should be as focused as possible. diff --git a/staging/src/k8s.io/code-generator/cmd/validation-gen/output_tests/tags/validate_false/doc.go b/staging/src/k8s.io/code-generator/cmd/validation-gen/output_tests/tags/validate_false/doc.go new file mode 100644 index 00000000000..bba748ade51 --- /dev/null +++ b/staging/src/k8s.io/code-generator/cmd/validation-gen/output_tests/tags/validate_false/doc.go @@ -0,0 +1,32 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +k8s:validation-gen=TypeMeta +// +k8s:validation-gen-scheme-registry=k8s.io/code-generator/cmd/validation-gen/testscheme.Scheme + +// This is a test package. +package validatefalse + +import "k8s.io/code-generator/cmd/validation-gen/testscheme" + +var localSchemeBuilder = testscheme.New() + +type Struct struct { + TypeMeta int + + // +k8s:validateFalse="field Struct.StringField" + StringField string `json:"stringField"` +} diff --git a/staging/src/k8s.io/code-generator/cmd/validation-gen/output_tests/tags/validate_false/doc_test.go b/staging/src/k8s.io/code-generator/cmd/validation-gen/output_tests/tags/validate_false/doc_test.go new file mode 100644 index 00000000000..564382d84c2 --- /dev/null +++ b/staging/src/k8s.io/code-generator/cmd/validation-gen/output_tests/tags/validate_false/doc_test.go @@ -0,0 +1,37 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validatefalse + +import ( + "testing" +) + +func Test(t *testing.T) { + st := localSchemeBuilder.Test(t) + + st.Value(&Struct{ + // All zero-values. + }).ExpectValidateFalseByPath(map[string][]string{ + "stringField": {"field Struct.StringField"}, + }) + + st.Value(&Struct{ + StringField: "abc", + }).ExpectValidateFalseByPath(map[string][]string{ + "stringField": {"field Struct.StringField"}, + }) +} diff --git a/staging/src/k8s.io/code-generator/cmd/validation-gen/output_tests/tags/validate_false/zz_generated.validations.go b/staging/src/k8s.io/code-generator/cmd/validation-gen/output_tests/tags/validate_false/zz_generated.validations.go new file mode 100644 index 00000000000..493a554b96f --- /dev/null +++ b/staging/src/k8s.io/code-generator/cmd/validation-gen/output_tests/tags/validate_false/zz_generated.validations.go @@ -0,0 +1,60 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by validation-gen. DO NOT EDIT. + +package validatefalse + +import ( + context "context" + fmt "fmt" + + operation "k8s.io/apimachinery/pkg/api/operation" + safe "k8s.io/apimachinery/pkg/api/safe" + validate "k8s.io/apimachinery/pkg/api/validate" + field "k8s.io/apimachinery/pkg/util/validation/field" + testscheme "k8s.io/code-generator/cmd/validation-gen/testscheme" +) + +func init() { localSchemeBuilder.Register(RegisterValidations) } + +// RegisterValidations adds validation functions to the given scheme. +// Public to allow building arbitrary schemes. +func RegisterValidations(scheme *testscheme.Scheme) error { + scheme.AddValidationFunc((*Struct)(nil), func(ctx context.Context, op operation.Operation, obj, oldObj interface{}, subresources ...string) field.ErrorList { + if len(subresources) == 0 { + return Validate_Struct(ctx, op, nil /* fldPath */, obj.(*Struct), safe.Cast[*Struct](oldObj)) + } + return field.ErrorList{field.InternalError(nil, fmt.Errorf("no validation found for %T, subresources: %v", obj, subresources))} + }) + return nil +} + +func Validate_Struct(ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *Struct) (errs field.ErrorList) { + // field Struct.TypeMeta has no validation + + // field Struct.StringField + errs = append(errs, + func(fldPath *field.Path, obj, oldObj *string) (errs field.ErrorList) { + errs = append(errs, validate.FixedResult(ctx, op, fldPath, obj, oldObj, false, "field Struct.StringField")...) + return + }(fldPath.Child("stringField"), &obj.StringField, safe.Field(oldObj, func(oldObj *Struct) *string { return &oldObj.StringField }))...) + + return errs +} diff --git a/staging/src/k8s.io/code-generator/cmd/validation-gen/testscheme/doc.go b/staging/src/k8s.io/code-generator/cmd/validation-gen/testscheme/doc.go new file mode 100644 index 00000000000..5d0f21e1672 --- /dev/null +++ b/staging/src/k8s.io/code-generator/cmd/validation-gen/testscheme/doc.go @@ -0,0 +1,105 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package testscheme provides a scheme implementation and test utilities +// useful for writing output_tests for validation-gen. +// +// For an output test to use this scheme, it should be located in a dedicated go package. +// The go package should have validation-gen and this test scheme enabled and must declare a +// 'localSchemeBuilder' using the 'testscheme.New()' function. For example: +// +// // +k8s:validation-gen-scheme-registry=k8s.io/code-generator/cmd/validation-gen/testscheme.Scheme +// // +k8s:validation-gen +// package example +// import "k8s.io/code-generator/cmd/validation-gen/testscheme" +// var localSchemeBuilder = testscheme.New() +// +// This is sufficient for validation-gen to generate a `zz_generated.validations.go` for the types +// in the package that compile. +// +// With the scheme enabled. An output test may be tested either by handwritten test code or +// by generated test fixtures. +// +// For example, to test by hand. The testschema provides utilities to create a value and assert +// that the expected errors are returned when the value is validated: +// +// func Test(t *testing.T) { +// st := localSchemeBuilder.Test(t) +// st.Value(&T1{ +// E0: "x", +// PE0: pointer.To(E0("y")), +// }).ExpectInvalid( +// field.NotSupported(field.NewPath("e0"), "x", []string{EnumValue1, EnumValue2}), +// field.NotSupported(field.NewPath("pe0"), "y", []string{EnumValue1, EnumValue2}))} +// } +// +// Tests fixtures can also be enabled. For example: +// +// // ... +// // +k8s:validation-gen-test-fixture=validateFalse +// // package example +// +// When a test fixture is enabled, a `zz_generated.validations_test.go` file will be generated +// test for all the registered types in the package according to the behavior of the named test +// fixture(s). +// +// Test Fixtures: +// +// `validateFalse` - This test fixture executes validation of each registered type and accumulates +// all `validateFalse` validation errors. For example: +// +// type T1 struct { +// // +k8s:validateFalse="field T1.S" +// S string `json:"s"` +// // +k8s:validateFalse="field T1.T" +// T T2 `json:"t"` +// } +// +// The above `example.T1` test type has two validated fields: `s` and 't'. The fields are named +// according to the Go "json" field tag, and each has a validation error identifier +// provided by `+k8s:validateFalse=`. +// +// The `validateFalse` test fixture will validate an instance of `example.T1`, generated using fuzzed +// data, and then group the validation errors by field name. Represented in JSON like: +// +// { +// "*example.T1": { +// "s": [ +// "field T1.S" +// ], +// "t": [ +// "field T1.T" +// ] +// } +// } +// +// This validation error data contains an object for each registered type, keyed by type name. +// For each registered type, the validation errors from `+k8s:validateFalse` tags are grouped +// by field path with all validation error identifiers collected into a list. +// +// This data is compared with the expected validation results that are defined in +// a `testdata/validate-false.json` file in the same package, and any differences are +// reported as test errors. +// +// `testdata/validate-false.json` can be generated automatically by setting the +// `UPDATE_VALIDATION_GEN_FIXTURE_DATA=true` environment variable to true when running the tests. +// +// Test authors that generated `testdata/validate-false.json` are expected to ensure that file +// is correct before checking it in to source control. +// +// The fuzzed data is generated pseudo-randomly with a consistent seed, with all nilable fields se +// to a value, and with a single entry for each map and a single element for each slice. +package testscheme diff --git a/staging/src/k8s.io/code-generator/cmd/validation-gen/testscheme/testscheme.go b/staging/src/k8s.io/code-generator/cmd/validation-gen/testscheme/testscheme.go new file mode 100644 index 00000000000..b31f85a59bd --- /dev/null +++ b/staging/src/k8s.io/code-generator/cmd/validation-gen/testscheme/testscheme.go @@ -0,0 +1,537 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testscheme + +import ( + "bytes" + stdcmp "cmp" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math/rand" + "os" + "path" + "reflect" + "regexp" + "sort" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" // nolint:depguard // this package provides test utilities + "github.com/google/go-cmp/cmp/cmpopts" // nolint:depguard // this package provides test utilities + fuzz "github.com/google/gofuzz" + + "k8s.io/apimachinery/pkg/api/operation" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +// Scheme is similar to runtime.Scheme, but for validation testing purposes. Scheme only supports validation, +// supports registration of any type (not just runtime.Object) and implements Register directly, allowing it +// to also be used as a scheme builder. +// Must only be used with tests that perform all registration before calls to validate. +type Scheme struct { + validationFuncs map[reflect.Type]func(ctx context.Context, op operation.Operation, object, oldObject interface{}, subresources ...string) field.ErrorList + registrationErrors field.ErrorList +} + +// New creates a new Scheme. +func New() *Scheme { + return &Scheme{validationFuncs: map[reflect.Type]func(ctx context.Context, op operation.Operation, object interface{}, oldObject interface{}, subresources ...string) field.ErrorList{}} +} + +// AddValidationFunc registers a validation function. +// Last writer wins. +func (s *Scheme) AddValidationFunc(srcType any, fn func(ctx context.Context, op operation.Operation, object, oldObject interface{}, subresources ...string) field.ErrorList) { + s.validationFuncs[reflect.TypeOf(srcType)] = fn +} + +// Validate validates an object using the registered validation function. +func (s *Scheme) Validate(ctx context.Context, opts sets.Set[string], object any, subresources ...string) field.ErrorList { + if len(s.registrationErrors) > 0 { + return s.registrationErrors // short circuit with registration errors if any are present + } + if fn, ok := s.validationFuncs[reflect.TypeOf(object)]; ok { + return fn(ctx, operation.Operation{Type: operation.Create, Options: opts}, object, nil, subresources...) + } + return nil +} + +// ValidateUpdate validates an update to an object using the registered validation function. +func (s *Scheme) ValidateUpdate(ctx context.Context, opts sets.Set[string], object, oldObject any, subresources ...string) field.ErrorList { + if len(s.registrationErrors) > 0 { + return s.registrationErrors // short circuit with registration errors if any are present + } + if fn, ok := s.validationFuncs[reflect.TypeOf(object)]; ok { + return fn(ctx, operation.Operation{Type: operation.Update}, object, oldObject, subresources...) + } + return nil +} + +// Register adds a scheme setup function to the list. +func (s *Scheme) Register(funcs ...func(*Scheme) error) { + for _, f := range funcs { + err := f(s) + if err != nil { + s.registrationErrors = append(s.registrationErrors, toRegistrationError(err)) + } + } +} + +func toRegistrationError(err error) *field.Error { + return field.InternalError(nil, fmt.Errorf("registration error: %w", err)) +} + +// Test returns a ValidationTestBuilder for this scheme. +func (s *Scheme) Test(t *testing.T) *ValidationTestBuilder { + return &ValidationTestBuilder{t, s} +} + +// ValidationTestBuilder provides convenience functions to build +// validation tests. +type ValidationTestBuilder struct { + *testing.T + s *Scheme +} + +const fixtureEnvVar = "UPDATE_VALIDATION_GEN_FIXTURE_DATA" + +// ValidateFixtures ensures that the validation errors of all registered types match what is expected by the test fixture files. +// For each registered type, a value is created for the type, and populated by fuzzing the value, before validating the type. +// See ValueFuzzed for details. +// +// If the UPDATE_VALIDATION_GEN_FIXTURE_DATA=true environment variable is set, test fixture files are created or overridden. +// +// Fixtures: +// - validate-false.json: defines a map of registered type to a map of field path to +validateFalse validations args +// that are expected to be returned as errors when the type is validated. +func (s *ValidationTestBuilder) ValidateFixtures() { + s.T.Helper() + + flag := os.Getenv(fixtureEnvVar) + // Run validation + got := map[string]map[string][]string{} + for t := range s.s.validationFuncs { + var v any + // TODO: this should handle maps and slices + if t.Kind() == reflect.Ptr { + v = reflect.New(t.Elem()).Interface() + } else { + v = reflect.Indirect(reflect.New(t)).Interface() + } + if reflect.TypeOf(v).Kind() != reflect.Ptr { + v = &v + } + s.ValueFuzzed(v) + vt := &ValidationTester{ValidationTestBuilder: s, value: v} + byPath := vt.ValidateFalseArgsByPath() + got[t.String()] = byPath + } + + testdataFilename := "testdata/validate-false.json" + if flag == "true" { + // Generate fixture file + if err := os.MkdirAll(path.Dir(testdataFilename), os.FileMode(0755)); err != nil { + s.Fatal("error making directory", err) + } + data, err := json.MarshalIndent(got, " ", " ") + if err != nil { + s.Fatal(err) + } + err = os.WriteFile(testdataFilename, data, os.FileMode(0644)) + if err != nil { + s.Fatal(err) + } + } else { + // Load fixture file + testdataFile, err := os.Open(testdataFilename) + if errors.Is(err, os.ErrNotExist) { + s.Fatalf("%s test fixture data not found. Run go test with the environment variable %s=true to create test fixture data.", + testdataFilename, fixtureEnvVar) + } else if err != nil { + s.Fatal(err) + } + defer func() { + err := testdataFile.Close() + if err != nil { + s.Fatal(err) + } + }() + + byteValue, err := io.ReadAll(testdataFile) + if err != nil { + s.Fatal(err) + } + testdata := map[string]map[string][]string{} + err = json.Unmarshal(byteValue, &testdata) + if err != nil { + s.Fatal(err) + } + // Compare fixture with validation results + expectedKeys := sets.New[string]() + gotKeys := sets.New[string]() + for k := range got { + gotKeys.Insert(k) + } + hasErrors := false + for k, expectedForType := range testdata { + expectedKeys.Insert(k) + gotForType, ok := got[k] + s.T.Run(k, func(t *testing.T) { + t.Helper() + + if !ok { + t.Errorf("%q has expected validateFalse args in %s but got no validation errors.", k, testdataFilename) + hasErrors = true + } else if !cmp.Equal(gotForType, expectedForType) { + t.Errorf("validateFalse args, grouped by field path, differed from %s:\n%s\n", + testdataFilename, cmp.Diff(gotForType, expectedForType, cmpopts.SortMaps(stdcmp.Less[string]))) + hasErrors = true + } + }) + } + for unexpectedType := range gotKeys.Difference(expectedKeys) { + s.T.Run(unexpectedType, func(t *testing.T) { + t.Helper() + + t.Errorf("%q got unexpected validateFalse args, grouped by field path:\n%s\n", + unexpectedType, cmp.Diff(nil, got[unexpectedType], cmpopts.SortMaps(stdcmp.Less[string]))) + hasErrors = true + }) + } + if hasErrors { + s.T.Logf("If the test expectations have changed, run go test with the environment variable %s=true", fixtureEnvVar) + } + } +} + +func fuzzer() *fuzz.Fuzzer { + // Ensure that lists and maps are not empty and use a deterministic seed. + return fuzz.New().NilChance(0.0).NumElements(2, 2).RandSource(rand.NewSource(0)) +} + +// ValueFuzzed automatically populates the given value using a deterministic fuzzer. +// The fuzzer sets pointers to values and always includes a two map keys and slice elements. +func (s *ValidationTestBuilder) ValueFuzzed(value any) *ValidationTester { + fuzzer().Fuzz(value) + return &ValidationTester{ValidationTestBuilder: s, value: value} +} + +// Value returns a ValidationTester for the given value. The value +// must be a registered with the scheme for validation. +func (s *ValidationTestBuilder) Value(value any) *ValidationTester { + return &ValidationTester{ValidationTestBuilder: s, value: value} +} + +// ValidationTester provides convenience functions to define validation +// tests for a validatable value. +type ValidationTester struct { + *ValidationTestBuilder + value any + oldValue any + opts sets.Set[string] +} + +// OldValue sets the oldValue for this ValidationTester. When oldValue is set to +// a non-nil value, update validation will be used to test validation. +// oldValue must be the same type as value. +// Returns ValidationTester to support call chaining. +func (v *ValidationTester) OldValue(oldValue any) *ValidationTester { + v.oldValue = oldValue + return v +} + +// OldValueFuzzed automatically populates the given value using a deterministic fuzzer. +// The fuzzer sets pointers to values and always includes a two map keys and slice elements. +func (v *ValidationTester) OldValueFuzzed(oldValue any) *ValidationTester { + fuzzer().Fuzz(oldValue) + v.oldValue = oldValue + return v +} + +// Opts sets the ValidationOpts to use. +func (v *ValidationTester) Opts(opts sets.Set[string]) *ValidationTester { + v.opts = opts + return v +} + +func multiline(errs field.ErrorList) string { + if len(errs) == 0 { + return "" + } + if len(errs) == 1 { + return errs[0].Error() + } + + var buf bytes.Buffer + for _, err := range errs { + buf.WriteString("\n") + buf.WriteString(err.Error()) + } + return buf.String() +} + +// ExpectValid validates the value and calls t.Errorf if any validation errors are returned. +// Returns ValidationTester to support call chaining. +func (v *ValidationTester) ExpectValid() *ValidationTester { + v.T.Helper() + + v.T.Run(fmt.Sprintf("%T", v.value), func(t *testing.T) { + t.Helper() + + errs := v.validate() + if len(errs) > 0 { + t.Errorf("want no errors, got: %v", multiline(errs)) + } + }) + return v +} + +// ExpectValidAt validates the value and calls t.Errorf for any validation errors at the given path. +// Returns ValidationTester to support call chaining. +func (v *ValidationTester) ExpectValidAt(fldPath *field.Path) *ValidationTester { + v.T.Helper() + + v.T.Run(fmt.Sprintf("%T.%v", v.value, fldPath), func(t *testing.T) { + t.Helper() + + var got field.ErrorList + for _, e := range v.validate() { + if e.Field == fldPath.String() { + got = append(got, e) + } + } + if len(got) > 0 { + t.Errorf("want no errors at %v, got: %v", fldPath, got) + } + }) + return v +} + +// ExpectInvalid validates the value and calls t.Errorf if want does not match the actual errors. +// Returns ValidationTester to support call chaining. +func (v *ValidationTester) ExpectInvalid(want ...*field.Error) *ValidationTester { + v.T.Helper() + + return v.expectInvalid(byFullError, want...) +} + +// ExpectValidateFalse validates the value and calls t.Errorf if the actual errors do not +// match the given validateFalseArgs. For example, if the value to validate has a +// single `+validateFalse="type T1"` tag, ExpectValidateFalse("type T1") will pass. +// Returns ValidationTester to support call chaining. +func (v *ValidationTester) ExpectValidateFalse(validateFalseArgs ...string) *ValidationTester { + v.T.Helper() + + var want []*field.Error + for _, s := range validateFalseArgs { + want = append(want, field.Invalid(nil, "", fmt.Sprintf("forced failure: %s", s))) + } + return v.expectInvalid(byDetail, want...) +} + +func (v *ValidationTester) ExpectValidateFalseByPath(validateFalseArgsByPath map[string][]string) *ValidationTester { + v.T.Helper() + + v.T.Run(fmt.Sprintf("%T", v.value), func(t *testing.T) { + t.Helper() + + byPath := v.ValidateFalseArgsByPath() + // ensure args are sorted + for _, args := range validateFalseArgsByPath { + sort.Strings(args) + } + if !cmp.Equal(validateFalseArgsByPath, byPath) { + t.Errorf("validateFalse args, grouped by field path, differed from expected:\n%s\n", cmp.Diff(validateFalseArgsByPath, byPath, cmpopts.SortMaps(stdcmp.Less[string]))) + } + + }) + return v +} + +func (v *ValidationTester) ValidateFalseArgsByPath() map[string][]string { + byPath := map[string][]string{} + errs := v.validate() + for _, e := range errs { + if strings.HasPrefix(e.Detail, "forced failure: ") { + arg := strings.TrimPrefix(e.Detail, "forced failure: ") + f := e.Field + if f == "" { + f = "" + } + byPath[f] = append(byPath[f], arg) + } + } + // ensure args are sorted + for _, args := range byPath { + sort.Strings(args) + } + return byPath +} + +func (v *ValidationTester) ExpectRegexpsByPath(regexpStringsByPath map[string][]string) *ValidationTester { + v.T.Helper() + + v.T.Run(fmt.Sprintf("%T", v.value), func(t *testing.T) { + t.Helper() + + errorsByPath := v.getErrorsByPath() + + // sanity check + if want, got := len(regexpStringsByPath), len(errorsByPath); got != want { + t.Fatalf("wrong number of error-fields: expected %d, got %d:\nwanted:\n%sgot:\n%s", + want, got, renderByPath(regexpStringsByPath), renderByPath(errorsByPath)) + } + + // compile regexps + regexpsByPath := map[string][]*regexp.Regexp{} + for field, strs := range regexpStringsByPath { + regexps := make([]*regexp.Regexp, 0, len(strs)) + for _, str := range strs { + regexps = append(regexps, regexp.MustCompile(str)) + } + regexpsByPath[field] = regexps + } + + for field := range errorsByPath { + errors := errorsByPath[field] + regexps := regexpsByPath[field] + + // sanity check + if want, got := len(regexps), len(errors); got != want { + t.Fatalf("field %q: wrong number of errors: expected %d, got %d:\nwanted:\n%sgot:\n%s", + field, want, got, renderList(regexpStringsByPath[field]), renderList(errors)) + } + + // build a set of errors and expectations, so we can track them, + expSet := sets.New(regexps...) + + for _, err := range errors { + var found *regexp.Regexp + for _, re := range regexps { + if re.MatchString(err) { + found = re + break // done with regexps + } + } + if found != nil { + expSet.Delete(found) + continue // next error + } + t.Errorf("field %q, error %q did not match any expectation", field, err) + } + if len(expSet) != 0 { + t.Errorf("field %q had unsatisfied expectations: %q", field, expSet.UnsortedList()) + } + } + }) + return v +} + +func (v *ValidationTester) getErrorsByPath() map[string][]string { + byPath := map[string][]string{} + errs := v.validate() + for _, e := range errs { + f := e.Field + if f == "" { + f = "" + } + byPath[f] = append(byPath[f], e.ErrorBody()) + } + // ensure args are sorted + for _, args := range byPath { + sort.Strings(args) + } + return byPath +} + +func renderByPath(byPath map[string][]string) string { + keys := []string{} + for key := range byPath { + keys = append(keys, key) + } + sort.Strings(keys) + + for _, vals := range byPath { + sort.Strings(vals) + } + + buf := strings.Builder{} + for _, key := range keys { + vals := byPath[key] + for _, val := range vals { + buf.WriteString(fmt.Sprintf("\t%s: %q\n", key, val)) + } + } + return buf.String() +} + +func renderList(list []string) string { + buf := strings.Builder{} + for _, item := range list { + buf.WriteString(fmt.Sprintf("\t%q\n", item)) + } + return buf.String() +} + +func (v *ValidationTester) expectInvalid(matcher matcher, errs ...*field.Error) *ValidationTester { + v.T.Helper() + + v.T.Run(fmt.Sprintf("%T", v.value), func(t *testing.T) { + t.Helper() + + want := sets.New[string]() + for _, e := range errs { + want.Insert(matcher(e)) + } + + got := sets.New[string]() + for _, e := range v.validate() { + got.Insert(matcher(e)) + } + if !got.Equal(want) { + t.Errorf("validation errors differed from expected:\n%v\n", cmp.Diff(want, got, cmpopts.SortMaps(stdcmp.Less[string]))) + + for x := range got.Difference(want) { + fmt.Printf("%q,\n", strings.TrimPrefix(x, "forced failure: ")) + } + } + }) + return v +} + +type matcher func(err *field.Error) string + +func byDetail(err *field.Error) string { + return err.Detail +} + +func byFullError(err *field.Error) string { + return err.Error() +} + +func (v *ValidationTester) validate() field.ErrorList { + var errs field.ErrorList + if v.oldValue == nil { + errs = v.s.Validate(context.Background(), v.opts, v.value) + } else { + errs = v.s.ValidateUpdate(context.Background(), v.opts, v.value, v.oldValue) + } + return errs +} diff --git a/staging/src/k8s.io/code-generator/cmd/validation-gen/validators/testing.go b/staging/src/k8s.io/code-generator/cmd/validation-gen/validators/testing.go new file mode 100644 index 00000000000..92769f1c29b --- /dev/null +++ b/staging/src/k8s.io/code-generator/cmd/validation-gen/validators/testing.go @@ -0,0 +1,177 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validators + +import ( + "encoding/json" + "fmt" + "strings" + + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/gengo/v2/types" +) + +const ( + // These tags return a fixed pass/fail state. + validateTrueTagName = "k8s:validateTrue" + validateFalseTagName = "k8s:validateFalse" + + // This tag always returns an error from ExtractValidations. + validateErrorTagName = "k8s:validateError" +) + +func init() { + RegisterTagValidator(fixedResultTagValidator{result: true}) + RegisterTagValidator(fixedResultTagValidator{result: false}) + RegisterTagValidator(fixedResultTagValidator{error: true}) +} + +type fixedResultTagValidator struct { + result bool + error bool +} + +func (fixedResultTagValidator) Init(_ Config) {} + +func (frtv fixedResultTagValidator) TagName() string { + if frtv.error { + return validateErrorTagName + } else if frtv.result { + return validateTrueTagName + } + return validateFalseTagName +} + +var fixedResultTagValidScopes = sets.New(ScopeAny) + +func (fixedResultTagValidator) ValidScopes() sets.Set[Scope] { + return fixedResultTagValidScopes +} + +func (frtv fixedResultTagValidator) GetValidations(context Context, _ []string, payload string) (Validations, error) { + var result Validations + + if frtv.error { + return result, fmt.Errorf("forced error: %q", payload) + } + + tag, err := frtv.parseTagPayload(payload) + if err != nil { + return result, fmt.Errorf("can't decode tag payload: %w", err) + } + result.AddFunction(GenericFunction(frtv.TagName(), tag.flags, fixedResultValidator, tag.typeArgs, frtv.result, tag.msg)) + + return result, nil +} + +var ( + fixedResultValidator = types.Name{Package: libValidationPkg, Name: "FixedResult"} +) + +type fixedResultPayload struct { + flags FunctionFlags + msg string + typeArgs []types.Name +} + +func (fixedResultTagValidator) parseTagPayload(in string) (fixedResultPayload, error) { + type payload struct { + Flags []string `json:"flags"` + Msg string `json:"msg"` + TypeArg string `json:"typeArg,omitempty"` + } + // We expect either a string (maybe empty) or a JSON object. + if len(in) == 0 { + return fixedResultPayload{}, nil + } + var pl payload + if err := json.Unmarshal([]byte(in), &pl); err != nil { + s := "" + if err := json.Unmarshal([]byte(in), &s); err != nil { + return fixedResultPayload{}, fmt.Errorf("error parsing JSON value: %w (%q)", err, in) + } + return fixedResultPayload{msg: s}, nil + } + // The msg field is required in JSON mode. + if pl.Msg == "" { + return fixedResultPayload{}, fmt.Errorf("JSON msg is required") + } + var flags FunctionFlags + for _, fl := range pl.Flags { + switch fl { + case "ShortCircuit": + flags |= ShortCircuit + case "NonError": + flags |= NonError + default: + return fixedResultPayload{}, fmt.Errorf("unknown flag: %q", fl) + } + } + var typeArgs []types.Name + if tn := pl.TypeArg; len(tn) > 0 { + if !strings.HasPrefix(tn, "*") { + tn = "*" + tn // We always need the pointer type. + } + typeArgs = []types.Name{{Package: "", Name: tn}} + } + + return fixedResultPayload{flags, pl.Msg, typeArgs}, nil +} + +func (frtv fixedResultTagValidator) Docs() TagDoc { + doc := TagDoc{ + Tag: frtv.TagName(), + Scopes: frtv.ValidScopes().UnsortedList(), + } + if frtv.error { + doc.Description = "Always fails code generation (useful for testing)." + doc.Payloads = []TagPayloadDoc{{ + Description: "", + Docs: "This string will be included in the error message.", + }} + } else { + // True and false have the same payloads. + doc.Payloads = []TagPayloadDoc{{ + Description: "", + }, { + Description: "", + Docs: "The generated code will include this string.", + }, { + Description: "", + Docs: "", + Schema: []TagPayloadSchema{{ + Key: "flags", + Value: "", + Docs: `values: ShortCircuit, NonError`, + }, { + Key: "msg", + Value: "", + Docs: "The generated code will include this string.", + }, { + Key: "typeArg", + Value: "", + Docs: "The type arg in generated code (must be the value-type, not pointer).", + }}, + }} + if frtv.result { + doc.Description = "Always passes validation (useful for testing)." + } else { + doc.Description = "Always fails validation (useful for testing)." + } + } + return doc +}