Merge pull request #130783 from jpbetz/versioned-formats

Support emulation versioning of custom resource formats

staging/src/k8s.io/apiextensions-apiserver/go.mod

@@ -21,6 +21,7 @@ require (
 	go.etcd.io/etcd/client/v3 v3.5.16
 	go.opentelemetry.io/otel v1.33.0
 	go.opentelemetry.io/otel/trace v1.33.0
+	golang.org/x/sync v0.11.0
 	google.golang.org/grpc v1.68.1
 	google.golang.org/protobuf v1.36.5
 	gopkg.in/evanphx/json-patch.v4 v4.12.0
@@ -110,7 +111,6 @@ require (
 	golang.org/x/mod v0.21.0 // indirect
 	golang.org/x/net v0.33.0 // indirect
 	golang.org/x/oauth2 v0.27.0 // indirect
-	golang.org/x/sync v0.11.0 // indirect
 	golang.org/x/sys v0.30.0 // indirect
 	golang.org/x/term v0.29.0 // indirect
 	golang.org/x/text v0.22.0 // indirect

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation/formats.go

@@ -17,49 +17,120 @@ limitations under the License.
 package validation
 
 import (
+	"fmt"
 	"strings"
+	"sync"
+
+	"golang.org/x/sync/singleflight"
 
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/kube-openapi/pkg/validation/spec"
 )
 
-var supportedFormats = sets.NewString(
+// supportedVersionedFormats tracks the formats supported by CRD schemas, and the version at which support was introduced.
-	"bsonobjectid", // bson object ID
+// Formats in CRD schemas are ignored when used in versions where they are not supported.
-	"uri",          // an URI as parsed by Golang net/url.ParseRequestURI
+var supportedVersionedFormats = []versionedFormats{
-	"email",        // an email address as parsed by Golang net/mail.ParseAddress
+	{
-	"hostname",     // a valid representation for an Internet host name, as defined by RFC 1034, section 3.1 [RFC1034].
+		introducedVersion: version.MajorMinor(1, 0),
-	"ipv4",         // an IPv4 IP as parsed by Golang net.ParseIP
+		formats: sets.New(
-	"ipv6",         // an IPv6 IP as parsed by Golang net.ParseIP
+			"bsonobjectid", // bson object ID
-	"cidr",         // a CIDR as parsed by Golang net.ParseCIDR
+			"uri",          // an URI as parsed by Golang net/url.ParseRequestURI
-	"mac",          // a MAC address as parsed by Golang net.ParseMAC
+			"email",        // an email address as parsed by Golang net/mail.ParseAddress
-	"uuid",         // an UUID that allows uppercase defined by the regex (?i)^[0-9a-f]{8}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{12}$
+			"hostname",     // a valid representation for an Internet host name, as defined by RFC 1034, section 3.1 [RFC1034].
-	"uuid3",        // an UUID3 that allows uppercase defined by the regex (?i)^[0-9a-f]{8}-?[0-9a-f]{4}-?3[0-9a-f]{3}-?[0-9a-f]{4}-?[0-9a-f]{12}$
+			"ipv4",         // an IPv4 IP as parsed by Golang net.ParseIP
-	"uuid4",        // an UUID4 that allows uppercase defined by the regex (?i)^[0-9a-f]{8}-?[0-9a-f]{4}-?4[0-9a-f]{3}-?[89ab][0-9a-f]{3}-?[0-9a-f]{12}$
+			"ipv6",         // an IPv6 IP as parsed by Golang net.ParseIP
-	"uuid5",        // an UUID6 that allows uppercase defined by the regex (?i)^[0-9a-f]{8}-?[0-9a-f]{4}-?5[0-9a-f]{3}-?[89ab][0-9a-f]{3}-?[0-9a-f]{12}$
+			"cidr",         // a CIDR as parsed by Golang net.ParseCIDR
-	"isbn",         // an ISBN10 or ISBN13 number string like "0321751043" or "978-0321751041"
+			"mac",          // a MAC address as parsed by Golang net.ParseMAC
-	"isbn10",       // an ISBN10 number string like "0321751043"
+			"uuid",         // an UUID that allows uppercase defined by the regex (?i)^[0-9a-f]{8}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{12}$
-	"isbn13",       // an ISBN13 number string like "978-0321751041"
+			"uuid3",        // an UUID3 that allows uppercase defined by the regex (?i)^[0-9a-f]{8}-?[0-9a-f]{4}-?3[0-9a-f]{3}-?[0-9a-f]{4}-?[0-9a-f]{12}$
-	"creditcard",   // a credit card number defined by the regex ^(?:4[0-9]{12}(?:[0-9]{3})?|5[1-5][0-9]{14}|6(?:011|5[0-9][0-9])[0-9]{12}|3[47][0-9]{13}|3(?:0[0-5]|[68][0-9])[0-9]{11}|(?:2131|1800|35\\d{3})\\d{11})$ with any non digit characters mixed in
+			"uuid4",        // an UUID4 that allows uppercase defined by the regex (?i)^[0-9a-f]{8}-?[0-9a-f]{4}-?4[0-9a-f]{3}-?[89ab][0-9a-f]{3}-?[0-9a-f]{12}$
-	"ssn",          // a U.S. social security number following the regex ^\\d{3}[- ]?\\d{2}[- ]?\\d{4}$
+			"uuid5",        // an UUID6 that allows uppercase defined by the regex (?i)^[0-9a-f]{8}-?[0-9a-f]{4}-?5[0-9a-f]{3}-?[89ab][0-9a-f]{3}-?[0-9a-f]{12}$
-	"hexcolor",     // an hexadecimal color code like "#FFFFFF", following the regex ^#?([0-9a-fA-F]{3}|[0-9a-fA-F]{6})$
+			"isbn",         // an ISBN10 or ISBN13 number string like "0321751043" or "978-0321751041"
-	"rgbcolor",     // an RGB color code like rgb like "rgb(255,255,2559"
+			"isbn10",       // an ISBN10 number string like "0321751043"
-	"byte",         // base64 encoded binary data
+			"isbn13",       // an ISBN13 number string like "978-0321751041"
-	"password",     // any kind of string
+			"creditcard",   // a credit card number defined by the regex ^(?:4[0-9]{12}(?:[0-9]{3})?|5[1-5][0-9]{14}|6(?:011|5[0-9][0-9])[0-9]{12}|3[47][0-9]{13}|3(?:0[0-5]|[68][0-9])[0-9]{11}|(?:2131|1800|35\\d{3})\\d{11})$ with any non digit characters mixed in
-	"date",         // a date string like "2006-01-02" as defined by full-date in RFC3339
+			"ssn",          // a U.S. social security number following the regex ^\\d{3}[- ]?\\d{2}[- ]?\\d{4}$
-	"duration",     // a duration string like "22 ns" as parsed by Golang time.ParseDuration or compatible with Scala duration format
+			"hexcolor",     // an hexadecimal color code like "#FFFFFF", following the regex ^#?([0-9a-fA-F]{3}|[0-9a-fA-F]{6})$
-	"datetime",     // a date time string like "2014-12-15T19:30:20.000Z" as defined by date-time in RFC3339
+			"rgbcolor",     // an RGB color code like rgb like "rgb(255,255,2559"
-)
+			"byte",         // base64 encoded binary data
+			"password",     // any kind of string
+			"date",         // a date string like "2006-01-02" as defined by full-date in RFC3339
+			"duration",     // a duration string like "22 ns" as parsed by Golang time.ParseDuration or compatible with Scala duration format
+			"datetime",     // a date time string like "2014-12-15T19:30:20.000Z" as defined by date-time in RFC3339
+		),
+	},
+}
 
 // StripUnsupportedFormatsPostProcess sets unsupported formats to empty string.
+// Only supports formats supported by all known version of Kubernetes.
+// Deprecated: Use StripUnsupportedFormatsPostProcessorForVersion instead.
 func StripUnsupportedFormatsPostProcess(s *spec.Schema) error {
-	if len(s.Format) == 0 {
+	return legacyPostProcessor(s)
+}
+
+// StripUnsupportedFormatsPostProcessorForVersion determines the supported formats at the given compatibility version and
+// sets unsupported formats to empty string.
+func StripUnsupportedFormatsPostProcessorForVersion(compatibilityVersion *version.Version) func(s *spec.Schema) error {
+	return func(s *spec.Schema) error {
+		if len(s.Format) == 0 {
+			return nil
+		}
+
+		normalized := strings.ReplaceAll(s.Format, "-", "") // go-openapi default format name normalization
+		if !supportedFormatsAtVersion(compatibilityVersion).supported.Has(normalized) {
+			s.Format = ""
+		}
+
 		return nil
 	}
-
-	normalized := strings.Replace(s.Format, "-", "", -1) // go-openapi default format name normalization
-	if !supportedFormats.Has(normalized) {
-		s.Format = ""
-	}
-
-	return nil
 }
+
+type versionedFormats struct {
+	introducedVersion *version.Version
+	formats           sets.Set[string]
+}
+type supportedFormats struct {
+	compatibilityVersion *version.Version
+	// supported is a set of formats validated at compatibilityVersion of Kubernetes.
+	supported sets.Set[string]
+}
+
+var cacheFormatSets = true
+
+func supportedFormatsAtVersion(ver *version.Version) *supportedFormats {
+	key := fmt.Sprintf("%d.%d", ver.Major(), ver.Minor())
+	var entry interface{}
+	if entry, ok := baseEnvs.Load(key); ok {
+		return entry.(*supportedFormats)
+	}
+	entry, _, _ = baseEnvsSingleflight.Do(key, func() (interface{}, error) {
+		entry := newFormatsAtVersion(ver, supportedVersionedFormats)
+		if cacheFormatSets {
+			baseEnvs.Store(key, entry)
+		}
+		return entry, nil
+	})
+	return entry.(*supportedFormats)
+}
+
+func newFormatsAtVersion(ver *version.Version, versionedFormats []versionedFormats) *supportedFormats {
+	result := &supportedFormats{
+		compatibilityVersion: ver,
+		supported:            sets.New[string](),
+	}
+	for _, vf := range versionedFormats {
+		if ver.AtLeast(vf.introducedVersion) {
+			result.supported = result.supported.Union(vf.formats)
+
+		}
+	}
+	return result
+}
+
+var (
+	baseEnvs             = sync.Map{}
+	baseEnvsSingleflight = &singleflight.Group{}
+)
+
+var legacyPostProcessor = StripUnsupportedFormatsPostProcessorForVersion(version.MajorMinor(1, 0))

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation/formats_test.go

@@ -19,13 +19,122 @@ package validation
 import (
 	"testing"
 
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/version"
+	"k8s.io/kube-openapi/pkg/validation/spec"
 	"k8s.io/kube-openapi/pkg/validation/strfmt"
 )
 
 func TestRegistryFormats(t *testing.T) {
-	for f := range supportedFormats {
+	for _, sf := range supportedVersionedFormats {
-		if !strfmt.Default.ContainsName(f) {
+		for f := range sf.formats {
-			t.Errorf("expected format %q in strfmt default registry", f)
+			if !strfmt.Default.ContainsName(f) {
+				t.Errorf("expected format %q in strfmt default registry", f)
+			}
 		}
 	}
 }
+
+func TestSupportedFormats(t *testing.T) {
+	vf := []versionedFormats{
+		{
+			introducedVersion: version.MajorMinor(1, 0),
+			formats: sets.New(
+				"A",
+			),
+		},
+		{
+			introducedVersion: version.MajorMinor(1, 1),
+			formats: sets.New(
+				"B",
+				"C",
+			),
+		},
+		// Version 1.2 has no new supported formats
+		{
+			introducedVersion: version.MajorMinor(1, 3),
+			formats: sets.New(
+				"D",
+			),
+		},
+		{
+			introducedVersion: version.MajorMinor(1, 3), // same version as previous entry
+			formats: sets.New(
+				"E",
+			),
+		},
+		{
+			introducedVersion: version.MajorMinor(1, 4),
+			formats:           sets.New[string](),
+		},
+	}
+
+	testCases := []struct {
+		name            string
+		version         *version.Version
+		expectedFormats sets.Set[string]
+	}{
+		{
+			name:            "version 1.0",
+			version:         version.MajorMinor(1, 0),
+			expectedFormats: sets.New("A"),
+		},
+		{
+			name:            "version 1.1",
+			version:         version.MajorMinor(1, 1),
+			expectedFormats: sets.New("A", "B", "C"),
+		},
+		{
+			name:            "version 1.2",
+			version:         version.MajorMinor(1, 2),
+			expectedFormats: sets.New("A", "B", "C"),
+		},
+		{
+			name:            "version 1.3",
+			version:         version.MajorMinor(1, 3),
+			expectedFormats: sets.New("A", "B", "C", "D", "E"),
+		},
+		{
+			name:            "version 1.4",
+			version:         version.MajorMinor(1, 4),
+			expectedFormats: sets.New("A", "B", "C", "D", "E"),
+		},
+	}
+	allFormats := newFormatsAtVersion(version.MajorMinor(0, 0), vf)
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := newFormatsAtVersion(tc.version, vf)
+
+			t.Run("newFormatsAtVersion", func(t *testing.T) {
+				if !got.supported.Equal(tc.expectedFormats) {
+					t.Errorf("expected %v, got %v", tc.expectedFormats, got.supported)
+				}
+
+				if len(got.supported.Difference(allFormats.supported)) == 0 {
+					t.Errorf("expected allFormats to be a superset of all formats, but was missing %v", allFormats.supported)
+				}
+			})
+
+			t.Run("StripUnsupportedFormatsPostProcessorForVersion", func(t *testing.T) {
+				processor := StripUnsupportedFormatsPostProcessorForVersion(tc.version)
+				for f := range allFormats.supported {
+					schema := &spec.Schema{SchemaProps: spec.SchemaProps{Format: f}}
+					err := processor(schema)
+					if err != nil {
+						t.Fatalf("Unexpected error: %v", err)
+					}
+					gotFormat := schema.Format
+					if tc.expectedFormats.Has(f) {
+						if gotFormat != f {
+							t.Errorf("expected format %q, got %q", f, gotFormat)
+						}
+					} else {
+						if gotFormat != "" {
+							t.Errorf("expected format to be stripped out, got %q", gotFormat)
+						}
+					}
+				}
+			})
+		})
+	}
+}

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation/validation.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/apiextensions-apiserver/pkg/features"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	"k8s.io/apiserver/pkg/cel/common"
+	"k8s.io/apiserver/pkg/cel/environment"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	openapierrors "k8s.io/kube-openapi/pkg/validation/errors"
 	"k8s.io/kube-openapi/pkg/validation/spec"
@@ -102,7 +103,8 @@ func NewSchemaValidator(customResourceValidation *apiextensions.JSONSchemaProps)
 	openapiSchema := &spec.Schema{}
 	if customResourceValidation != nil {
 		// TODO: replace with NewStructural(...).ToGoOpenAPI
-		if err := ConvertJSONSchemaPropsWithPostProcess(customResourceValidation, openapiSchema, StripUnsupportedFormatsPostProcess); err != nil {
+		formatPostProcessor := StripUnsupportedFormatsPostProcessorForVersion(environment.DefaultCompatibilityVersion())
+		if err := ConvertJSONSchemaPropsWithPostProcess(customResourceValidation, openapiSchema, formatPostProcessor); err != nil {
 			return nil, nil, err
 		}
 	}

staging/src/k8s.io/apiextensions-apiserver/test/integration/ratcheting_test.go

@@ -49,6 +49,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/apimachinery/pkg/util/wait"
 	utilyaml "k8s.io/apimachinery/pkg/util/yaml"
+	"k8s.io/apiserver/pkg/cel/environment"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/dynamic"
 	featuregatetesting "k8s.io/component-base/featuregate/testing"
@@ -1769,7 +1770,8 @@ func newValidator(customResourceValidation *apiextensionsinternal.JSONSchemaProp
 	openapiSchema := &spec.Schema{}
 	if customResourceValidation != nil {
 		// TODO: replace with NewStructural(...).ToGoOpenAPI
-		if err := apiservervalidation.ConvertJSONSchemaPropsWithPostProcess(customResourceValidation, openapiSchema, apiservervalidation.StripUnsupportedFormatsPostProcess); err != nil {
+		formatPostProcessor := apiservervalidation.StripUnsupportedFormatsPostProcessorForVersion(environment.DefaultCompatibilityVersion())
+		if err := apiservervalidation.ConvertJSONSchemaPropsWithPostProcess(customResourceValidation, openapiSchema, formatPostProcessor); err != nil {
 			return nil, err
 		}
 	}

test/e2e/apimachinery/crd_publish_openapi.go

@@ -29,6 +29,7 @@ import (
 	"github.com/onsi/ginkgo/v2"
 	"sigs.k8s.io/yaml"
 
+	"k8s.io/apiserver/pkg/cel/environment"
 	openapiutil "k8s.io/kube-openapi/pkg/util"
 	"k8s.io/utils/pointer"
 
@@ -698,7 +699,8 @@ func convertJSONSchemaProps(in []byte, out *spec.Schema) error {
 		return err
 	}
 	kubeOut := spec.Schema{}
-	if err := validation.ConvertJSONSchemaPropsWithPostProcess(&internal, &kubeOut, validation.StripUnsupportedFormatsPostProcess); err != nil {
+	formatPostProcessor := validation.StripUnsupportedFormatsPostProcessorForVersion(environment.DefaultCompatibilityVersion())
+	if err := validation.ConvertJSONSchemaPropsWithPostProcess(&internal, &kubeOut, formatPostProcessor); err != nil {
 		return err
 	}
 	bs, err := json.Marshal(kubeOut)