From e0011c723674bdc1b0ff16aca9272598aa969bfd Mon Sep 17 00:00:00 2001 From: Joe Betz Date: Wed, 12 Mar 2025 10:24:19 -0400 Subject: [PATCH] Support emulation versioning of formats We plan to add more formats in upcoming releases. This tracks which formats are introduced at a version. Unrecognized formats remain ignored. That is, if a format is not supported at the emulated version, the format is not enforced. This differs from typical field handling, where unsupported field values are forbidden. This is pre-existing behavior and is in compliance with JSON Schema's format handling. Ratcheting of custom resources helps with the introduction of new formats. When a cluster is upgraded to a version of Kubernetes that supports a format already set (but not enforced) in a custom resource definition, the format will start being enforced against custom resources. Ratcheting will tolerate unchanged values of custom resources, even if the value is not valid according to the format. --- .../src/k8s.io/apiextensions-apiserver/go.mod | 2 +- .../pkg/apiserver/validation/formats.go | 139 +++++++++++++----- .../pkg/apiserver/validation/formats_test.go | 115 ++++++++++++++- .../pkg/apiserver/validation/validation.go | 4 +- .../test/integration/ratcheting_test.go | 4 +- test/e2e/apimachinery/crd_publish_openapi.go | 4 +- 6 files changed, 227 insertions(+), 41 deletions(-) diff --git a/staging/src/k8s.io/apiextensions-apiserver/go.mod b/staging/src/k8s.io/apiextensions-apiserver/go.mod index 9e21480d042..625304face3 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/go.mod +++ b/staging/src/k8s.io/apiextensions-apiserver/go.mod @@ -21,6 +21,7 @@ require ( go.etcd.io/etcd/client/v3 v3.5.16 go.opentelemetry.io/otel v1.33.0 go.opentelemetry.io/otel/trace v1.33.0 + golang.org/x/sync v0.11.0 google.golang.org/grpc v1.68.1 google.golang.org/protobuf v1.36.5 gopkg.in/evanphx/json-patch.v4 v4.12.0 @@ -110,7 +111,6 @@ require ( golang.org/x/mod v0.21.0 // indirect golang.org/x/net v0.33.0 // indirect golang.org/x/oauth2 v0.27.0 // indirect - golang.org/x/sync v0.11.0 // indirect golang.org/x/sys v0.30.0 // indirect golang.org/x/term v0.29.0 // indirect golang.org/x/text v0.22.0 // indirect diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation/formats.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation/formats.go index f67c1c58e31..21f37ee1bc4 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation/formats.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation/formats.go @@ -17,49 +17,120 @@ limitations under the License. package validation import ( + "fmt" "strings" + "sync" + + "golang.org/x/sync/singleflight" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/version" "k8s.io/kube-openapi/pkg/validation/spec" ) -var supportedFormats = sets.NewString( - "bsonobjectid", // bson object ID - "uri", // an URI as parsed by Golang net/url.ParseRequestURI - "email", // an email address as parsed by Golang net/mail.ParseAddress - "hostname", // a valid representation for an Internet host name, as defined by RFC 1034, section 3.1 [RFC1034]. - "ipv4", // an IPv4 IP as parsed by Golang net.ParseIP - "ipv6", // an IPv6 IP as parsed by Golang net.ParseIP - "cidr", // a CIDR as parsed by Golang net.ParseCIDR - "mac", // a MAC address as parsed by Golang net.ParseMAC - "uuid", // an UUID that allows uppercase defined by the regex (?i)^[0-9a-f]{8}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{12}$ - "uuid3", // an UUID3 that allows uppercase defined by the regex (?i)^[0-9a-f]{8}-?[0-9a-f]{4}-?3[0-9a-f]{3}-?[0-9a-f]{4}-?[0-9a-f]{12}$ - "uuid4", // an UUID4 that allows uppercase defined by the regex (?i)^[0-9a-f]{8}-?[0-9a-f]{4}-?4[0-9a-f]{3}-?[89ab][0-9a-f]{3}-?[0-9a-f]{12}$ - "uuid5", // an UUID6 that allows uppercase defined by the regex (?i)^[0-9a-f]{8}-?[0-9a-f]{4}-?5[0-9a-f]{3}-?[89ab][0-9a-f]{3}-?[0-9a-f]{12}$ - "isbn", // an ISBN10 or ISBN13 number string like "0321751043" or "978-0321751041" - "isbn10", // an ISBN10 number string like "0321751043" - "isbn13", // an ISBN13 number string like "978-0321751041" - "creditcard", // a credit card number defined by the regex ^(?:4[0-9]{12}(?:[0-9]{3})?|5[1-5][0-9]{14}|6(?:011|5[0-9][0-9])[0-9]{12}|3[47][0-9]{13}|3(?:0[0-5]|[68][0-9])[0-9]{11}|(?:2131|1800|35\\d{3})\\d{11})$ with any non digit characters mixed in - "ssn", // a U.S. social security number following the regex ^\\d{3}[- ]?\\d{2}[- ]?\\d{4}$ - "hexcolor", // an hexadecimal color code like "#FFFFFF", following the regex ^#?([0-9a-fA-F]{3}|[0-9a-fA-F]{6})$ - "rgbcolor", // an RGB color code like rgb like "rgb(255,255,2559" - "byte", // base64 encoded binary data - "password", // any kind of string - "date", // a date string like "2006-01-02" as defined by full-date in RFC3339 - "duration", // a duration string like "22 ns" as parsed by Golang time.ParseDuration or compatible with Scala duration format - "datetime", // a date time string like "2014-12-15T19:30:20.000Z" as defined by date-time in RFC3339 -) +// supportedVersionedFormats tracks the formats supported by CRD schemas, and the version at which support was introduced. +// Formats in CRD schemas are ignored when used in versions where they are not supported. +var supportedVersionedFormats = []versionedFormats{ + { + introducedVersion: version.MajorMinor(1, 0), + formats: sets.New( + "bsonobjectid", // bson object ID + "uri", // an URI as parsed by Golang net/url.ParseRequestURI + "email", // an email address as parsed by Golang net/mail.ParseAddress + "hostname", // a valid representation for an Internet host name, as defined by RFC 1034, section 3.1 [RFC1034]. + "ipv4", // an IPv4 IP as parsed by Golang net.ParseIP + "ipv6", // an IPv6 IP as parsed by Golang net.ParseIP + "cidr", // a CIDR as parsed by Golang net.ParseCIDR + "mac", // a MAC address as parsed by Golang net.ParseMAC + "uuid", // an UUID that allows uppercase defined by the regex (?i)^[0-9a-f]{8}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{12}$ + "uuid3", // an UUID3 that allows uppercase defined by the regex (?i)^[0-9a-f]{8}-?[0-9a-f]{4}-?3[0-9a-f]{3}-?[0-9a-f]{4}-?[0-9a-f]{12}$ + "uuid4", // an UUID4 that allows uppercase defined by the regex (?i)^[0-9a-f]{8}-?[0-9a-f]{4}-?4[0-9a-f]{3}-?[89ab][0-9a-f]{3}-?[0-9a-f]{12}$ + "uuid5", // an UUID6 that allows uppercase defined by the regex (?i)^[0-9a-f]{8}-?[0-9a-f]{4}-?5[0-9a-f]{3}-?[89ab][0-9a-f]{3}-?[0-9a-f]{12}$ + "isbn", // an ISBN10 or ISBN13 number string like "0321751043" or "978-0321751041" + "isbn10", // an ISBN10 number string like "0321751043" + "isbn13", // an ISBN13 number string like "978-0321751041" + "creditcard", // a credit card number defined by the regex ^(?:4[0-9]{12}(?:[0-9]{3})?|5[1-5][0-9]{14}|6(?:011|5[0-9][0-9])[0-9]{12}|3[47][0-9]{13}|3(?:0[0-5]|[68][0-9])[0-9]{11}|(?:2131|1800|35\\d{3})\\d{11})$ with any non digit characters mixed in + "ssn", // a U.S. social security number following the regex ^\\d{3}[- ]?\\d{2}[- ]?\\d{4}$ + "hexcolor", // an hexadecimal color code like "#FFFFFF", following the regex ^#?([0-9a-fA-F]{3}|[0-9a-fA-F]{6})$ + "rgbcolor", // an RGB color code like rgb like "rgb(255,255,2559" + "byte", // base64 encoded binary data + "password", // any kind of string + "date", // a date string like "2006-01-02" as defined by full-date in RFC3339 + "duration", // a duration string like "22 ns" as parsed by Golang time.ParseDuration or compatible with Scala duration format + "datetime", // a date time string like "2014-12-15T19:30:20.000Z" as defined by date-time in RFC3339 + ), + }, +} // StripUnsupportedFormatsPostProcess sets unsupported formats to empty string. +// Only supports formats supported by all known version of Kubernetes. +// Deprecated: Use StripUnsupportedFormatsPostProcessorForVersion instead. func StripUnsupportedFormatsPostProcess(s *spec.Schema) error { - if len(s.Format) == 0 { + return legacyPostProcessor(s) +} + +// StripUnsupportedFormatsPostProcessorForVersion determines the supported formats at the given compatibility version and +// sets unsupported formats to empty string. +func StripUnsupportedFormatsPostProcessorForVersion(compatibilityVersion *version.Version) func(s *spec.Schema) error { + return func(s *spec.Schema) error { + if len(s.Format) == 0 { + return nil + } + + normalized := strings.ReplaceAll(s.Format, "-", "") // go-openapi default format name normalization + if !supportedFormatsAtVersion(compatibilityVersion).supported.Has(normalized) { + s.Format = "" + } + return nil } - - normalized := strings.Replace(s.Format, "-", "", -1) // go-openapi default format name normalization - if !supportedFormats.Has(normalized) { - s.Format = "" - } - - return nil } + +type versionedFormats struct { + introducedVersion *version.Version + formats sets.Set[string] +} +type supportedFormats struct { + compatibilityVersion *version.Version + // supported is a set of formats validated at compatibilityVersion of Kubernetes. + supported sets.Set[string] +} + +var cacheFormatSets = true + +func supportedFormatsAtVersion(ver *version.Version) *supportedFormats { + key := fmt.Sprintf("%d.%d", ver.Major(), ver.Minor()) + var entry interface{} + if entry, ok := baseEnvs.Load(key); ok { + return entry.(*supportedFormats) + } + entry, _, _ = baseEnvsSingleflight.Do(key, func() (interface{}, error) { + entry := newFormatsAtVersion(ver, supportedVersionedFormats) + if cacheFormatSets { + baseEnvs.Store(key, entry) + } + return entry, nil + }) + return entry.(*supportedFormats) +} + +func newFormatsAtVersion(ver *version.Version, versionedFormats []versionedFormats) *supportedFormats { + result := &supportedFormats{ + compatibilityVersion: ver, + supported: sets.New[string](), + } + for _, vf := range versionedFormats { + if ver.AtLeast(vf.introducedVersion) { + result.supported = result.supported.Union(vf.formats) + + } + } + return result +} + +var ( + baseEnvs = sync.Map{} + baseEnvsSingleflight = &singleflight.Group{} +) + +var legacyPostProcessor = StripUnsupportedFormatsPostProcessorForVersion(version.MajorMinor(1, 0)) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation/formats_test.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation/formats_test.go index 17c13ac6ada..c93a7f74d1d 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation/formats_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation/formats_test.go @@ -19,13 +19,122 @@ package validation import ( "testing" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/version" + "k8s.io/kube-openapi/pkg/validation/spec" "k8s.io/kube-openapi/pkg/validation/strfmt" ) func TestRegistryFormats(t *testing.T) { - for f := range supportedFormats { - if !strfmt.Default.ContainsName(f) { - t.Errorf("expected format %q in strfmt default registry", f) + for _, sf := range supportedVersionedFormats { + for f := range sf.formats { + if !strfmt.Default.ContainsName(f) { + t.Errorf("expected format %q in strfmt default registry", f) + } } } } + +func TestSupportedFormats(t *testing.T) { + vf := []versionedFormats{ + { + introducedVersion: version.MajorMinor(1, 0), + formats: sets.New( + "A", + ), + }, + { + introducedVersion: version.MajorMinor(1, 1), + formats: sets.New( + "B", + "C", + ), + }, + // Version 1.2 has no new supported formats + { + introducedVersion: version.MajorMinor(1, 3), + formats: sets.New( + "D", + ), + }, + { + introducedVersion: version.MajorMinor(1, 3), // same version as previous entry + formats: sets.New( + "E", + ), + }, + { + introducedVersion: version.MajorMinor(1, 4), + formats: sets.New[string](), + }, + } + + testCases := []struct { + name string + version *version.Version + expectedFormats sets.Set[string] + }{ + { + name: "version 1.0", + version: version.MajorMinor(1, 0), + expectedFormats: sets.New("A"), + }, + { + name: "version 1.1", + version: version.MajorMinor(1, 1), + expectedFormats: sets.New("A", "B", "C"), + }, + { + name: "version 1.2", + version: version.MajorMinor(1, 2), + expectedFormats: sets.New("A", "B", "C"), + }, + { + name: "version 1.3", + version: version.MajorMinor(1, 3), + expectedFormats: sets.New("A", "B", "C", "D", "E"), + }, + { + name: "version 1.4", + version: version.MajorMinor(1, 4), + expectedFormats: sets.New("A", "B", "C", "D", "E"), + }, + } + allFormats := newFormatsAtVersion(version.MajorMinor(0, 0), vf) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got := newFormatsAtVersion(tc.version, vf) + + t.Run("newFormatsAtVersion", func(t *testing.T) { + if !got.supported.Equal(tc.expectedFormats) { + t.Errorf("expected %v, got %v", tc.expectedFormats, got.supported) + } + + if len(got.supported.Difference(allFormats.supported)) == 0 { + t.Errorf("expected allFormats to be a superset of all formats, but was missing %v", allFormats.supported) + } + }) + + t.Run("StripUnsupportedFormatsPostProcessorForVersion", func(t *testing.T) { + processor := StripUnsupportedFormatsPostProcessorForVersion(tc.version) + for f := range allFormats.supported { + schema := &spec.Schema{SchemaProps: spec.SchemaProps{Format: f}} + err := processor(schema) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + gotFormat := schema.Format + if tc.expectedFormats.Has(f) { + if gotFormat != f { + t.Errorf("expected format %q, got %q", f, gotFormat) + } + } else { + if gotFormat != "" { + t.Errorf("expected format to be stripped out, got %q", gotFormat) + } + } + } + }) + }) + } +} diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation/validation.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation/validation.go index 85f1ffca7fd..e65f863503e 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation/validation.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation/validation.go @@ -25,6 +25,7 @@ import ( "k8s.io/apiextensions-apiserver/pkg/features" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apiserver/pkg/cel/common" + "k8s.io/apiserver/pkg/cel/environment" utilfeature "k8s.io/apiserver/pkg/util/feature" openapierrors "k8s.io/kube-openapi/pkg/validation/errors" "k8s.io/kube-openapi/pkg/validation/spec" @@ -102,7 +103,8 @@ func NewSchemaValidator(customResourceValidation *apiextensions.JSONSchemaProps) openapiSchema := &spec.Schema{} if customResourceValidation != nil { // TODO: replace with NewStructural(...).ToGoOpenAPI - if err := ConvertJSONSchemaPropsWithPostProcess(customResourceValidation, openapiSchema, StripUnsupportedFormatsPostProcess); err != nil { + formatPostProcessor := StripUnsupportedFormatsPostProcessorForVersion(environment.DefaultCompatibilityVersion()) + if err := ConvertJSONSchemaPropsWithPostProcess(customResourceValidation, openapiSchema, formatPostProcessor); err != nil { return nil, nil, err } } diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/ratcheting_test.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/ratcheting_test.go index f0bb72498f5..b7befedf476 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/ratcheting_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/ratcheting_test.go @@ -49,6 +49,7 @@ import ( "k8s.io/apimachinery/pkg/util/version" "k8s.io/apimachinery/pkg/util/wait" utilyaml "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/apiserver/pkg/cel/environment" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/dynamic" featuregatetesting "k8s.io/component-base/featuregate/testing" @@ -1769,7 +1770,8 @@ func newValidator(customResourceValidation *apiextensionsinternal.JSONSchemaProp openapiSchema := &spec.Schema{} if customResourceValidation != nil { // TODO: replace with NewStructural(...).ToGoOpenAPI - if err := apiservervalidation.ConvertJSONSchemaPropsWithPostProcess(customResourceValidation, openapiSchema, apiservervalidation.StripUnsupportedFormatsPostProcess); err != nil { + formatPostProcessor := apiservervalidation.StripUnsupportedFormatsPostProcessorForVersion(environment.DefaultCompatibilityVersion()) + if err := apiservervalidation.ConvertJSONSchemaPropsWithPostProcess(customResourceValidation, openapiSchema, formatPostProcessor); err != nil { return nil, err } } diff --git a/test/e2e/apimachinery/crd_publish_openapi.go b/test/e2e/apimachinery/crd_publish_openapi.go index ac816997810..1c4b3a7c120 100644 --- a/test/e2e/apimachinery/crd_publish_openapi.go +++ b/test/e2e/apimachinery/crd_publish_openapi.go @@ -29,6 +29,7 @@ import ( "github.com/onsi/ginkgo/v2" "sigs.k8s.io/yaml" + "k8s.io/apiserver/pkg/cel/environment" openapiutil "k8s.io/kube-openapi/pkg/util" "k8s.io/utils/pointer" @@ -698,7 +699,8 @@ func convertJSONSchemaProps(in []byte, out *spec.Schema) error { return err } kubeOut := spec.Schema{} - if err := validation.ConvertJSONSchemaPropsWithPostProcess(&internal, &kubeOut, validation.StripUnsupportedFormatsPostProcess); err != nil { + formatPostProcessor := validation.StripUnsupportedFormatsPostProcessorForVersion(environment.DefaultCompatibilityVersion()) + if err := validation.ConvertJSONSchemaPropsWithPostProcess(&internal, &kubeOut, formatPostProcessor); err != nil { return err } bs, err := json.Marshal(kubeOut)