From 291703482d58ae030da71c6d671a96a6f960fc6f Mon Sep 17 00:00:00 2001 From: Joe Betz Date: Wed, 28 Feb 2024 14:06:06 -0500 Subject: [PATCH] Add selectableFields to CRDs --- pkg/features/kube_features.go | 2 + .../pkg/apis/apiextensions/fuzzer/fuzzer.go | 7 +- .../pkg/apis/apiextensions/types.go | 25 + .../pkg/apis/apiextensions/v1/conversion.go | 26 +- .../apis/apiextensions/v1/conversion_test.go | 132 +++- .../pkg/apis/apiextensions/v1/types.go | 22 + .../pkg/apis/apiextensions/v1beta1/types.go | 29 + .../apiextensions/validation/validation.go | 74 ++- .../validation/validation_test.go | 299 ++++++++- .../pkg/apiserver/conversion/converter.go | 38 +- .../pkg/apiserver/customresource_handler.go | 1 + .../pkg/apiserver/schema/cel/compilation.go | 2 +- .../pkg/apiserver/schema/cel/validation.go | 56 +- .../apiserver/schema/cel/validation_test.go | 2 +- .../pkg/apiserver/validation/metrics_test.go | 1 + .../pkg/cmd/server/testing/testserver.go | 12 +- .../pkg/controller/openapi/builder/builder.go | 56 +- .../openapi/builder/builder_test.go | 127 ++-- .../pkg/controller/openapi/controller.go | 8 +- .../pkg/controller/openapiv3/controller.go | 7 +- .../pkg/features/kube_features.go | 10 +- .../pkg/registry/customresource/etcd_test.go | 1 + .../pkg/registry/customresource/strategy.go | 108 ++- .../registry/customresource/strategy_test.go | 66 ++ .../customresourcedefinition/strategy.go | 23 + .../customresourcedefinition/strategy_test.go | 614 +++++++++++++++++- .../apiserver/pkg/endpoints/installer.go | 12 +- 27 files changed, 1623 insertions(+), 137 deletions(-) diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 9951b602adc..549499de179 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -1234,6 +1234,8 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS apiextensionsfeatures.CRDValidationRatcheting: {Default: true, PreRelease: featuregate.Beta}, + apiextensionsfeatures.CustomResourceFieldSelectors: {Default: false, PreRelease: featuregate.Alpha}, + // features that enable backwards compatibility but are scheduled to be removed // ... HPAScaleToZero: {Default: false, PreRelease: featuregate.Alpha}, diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/fuzzer/fuzzer.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/fuzzer/fuzzer.go index a12977c876c..9823b3e8d6c 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/fuzzer/fuzzer.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/fuzzer/fuzzer.go @@ -66,6 +66,7 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} { {Name: "Age", Type: "date", Description: swaggerMetadataDescriptions["creationTimestamp"], JSONPath: ".metadata.creationTimestamp"}, } } + c.Fuzz(&obj.SelectableFields) if obj.Conversion == nil { obj.Conversion = &apiextensions.CustomResourceConversion{ Strategy: apiextensions.NoneConverter, @@ -78,7 +79,7 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} { obj.PreserveUnknownFields = pointer.BoolPtr(true) } - // Move per-version schema, subresources, additionalPrinterColumns to the top-level. + // Move per-version schema, subresources, additionalPrinterColumns, selectableFields to the top-level. // This is required by validation in v1beta1, and by round-tripping in v1. if len(obj.Versions) == 1 { if obj.Versions[0].Schema != nil { @@ -89,6 +90,10 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} { obj.AdditionalPrinterColumns = obj.Versions[0].AdditionalPrinterColumns obj.Versions[0].AdditionalPrinterColumns = nil } + if obj.Versions[0].SelectableFields != nil { + obj.SelectableFields = obj.Versions[0].SelectableFields + obj.Versions[0].SelectableFields = nil + } if obj.Versions[0].Subresources != nil { obj.Subresources = obj.Versions[0].Subresources obj.Versions[0].Subresources = nil diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/types.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/types.go index b1c5f6f4c09..6556eda65d5 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/types.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/types.go @@ -70,6 +70,12 @@ type CustomResourceDefinitionSpec struct { // Top-level and per-version columns are mutually exclusive. // +optional AdditionalPrinterColumns []CustomResourceColumnDefinition + // selectableFields specifies paths to fields that may be used as field selectors. + // A maximum of 8 selectable fields are allowed. + // See https://kubernetes.io/docs/concepts/overview/working-with-objects/field-selectors + // Top-level and per-version columns are mutually exclusive. + // +optional + SelectableFields []SelectableField // `conversion` defines conversion settings for the CRD. Conversion *CustomResourceConversion @@ -207,6 +213,25 @@ type CustomResourceDefinitionVersion struct { // be explicitly set to null // +optional AdditionalPrinterColumns []CustomResourceColumnDefinition + + // selectableFields specifies paths to fields that may be used as field selectors. + // A maximum of 8 selectable fields are allowed. + // See https://kubernetes.io/docs/concepts/overview/working-with-objects/field-selectors + // +optional + SelectableFields []SelectableField +} + +// SelectableField specifies the JSON path of a field that may be used with field selectors. +type SelectableField struct { + // jsonPath is a simple JSON path which is evaluated against each custom resource to produce a + // field selector value. + // Only JSON paths without the array notation are allowed. + // Must point to a field of type string, boolean or integer. Types with enum values + // and strings with formats are allowed. + // If jsonPath refers to absent field in a resource, the jsonPath evaluates to an empty string. + // Must not point to metdata fields. + // Required. + JSONPath string } // CustomResourceColumnDefinition specifies a column for server side printing. diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1/conversion.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1/conversion.go index 4d29ff8235d..2ca72bb16b3 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1/conversion.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1/conversion.go @@ -80,7 +80,7 @@ func Convert_apiextensions_CustomResourceDefinitionSpec_To_v1_CustomResourceDefi out.Versions = []CustomResourceDefinitionVersion{{Name: in.Version, Served: true, Storage: true}} } - // If spec.{subresources,validation,additionalPrinterColumns} exists, move to versions + // If spec.{subresources,validation,additionalPrinterColumns,selectableFields} exists, move to versions if in.Subresources != nil { subresources := &CustomResourceSubresources{} if err := Convert_apiextensions_CustomResourceSubresources_To_v1_CustomResourceSubresources(in.Subresources, subresources, s); err != nil { @@ -110,6 +110,17 @@ func Convert_apiextensions_CustomResourceDefinitionSpec_To_v1_CustomResourceDefi out.Versions[i].AdditionalPrinterColumns = additionalPrinterColumns } } + if in.SelectableFields != nil { + selectableFields := make([]SelectableField, len(in.SelectableFields)) + for i := range in.SelectableFields { + if err := Convert_apiextensions_SelectableField_To_v1_SelectableField(&in.SelectableFields[i], &selectableFields[i], s); err != nil { + return err + } + } + for i := range out.Versions { + out.Versions[i].SelectableFields = selectableFields + } + } return nil } @@ -125,13 +136,15 @@ func Convert_v1_CustomResourceDefinitionSpec_To_apiextensions_CustomResourceDefi // Copy versions[0] to version out.Version = out.Versions[0].Name - // If versions[*].{subresources,schema,additionalPrinterColumns} are identical, move to spec + // If versions[*].{subresources,schema,additionalPrinterColumns,selectableFields} are identical, move to spec subresources := out.Versions[0].Subresources subresourcesIdentical := true validation := out.Versions[0].Schema validationIdentical := true additionalPrinterColumns := out.Versions[0].AdditionalPrinterColumns additionalPrinterColumnsIdentical := true + selectableFields := out.Versions[0].SelectableFields + selectableFieldsIdentical := true // Detect if per-version fields are identical for _, v := range out.Versions { @@ -144,6 +157,9 @@ func Convert_v1_CustomResourceDefinitionSpec_To_apiextensions_CustomResourceDefi if additionalPrinterColumnsIdentical && !apiequality.Semantic.DeepEqual(v.AdditionalPrinterColumns, additionalPrinterColumns) { additionalPrinterColumnsIdentical = false } + if selectableFieldsIdentical && !apiequality.Semantic.DeepEqual(v.SelectableFields, selectableFields) { + selectableFieldsIdentical = false + } } // If they are, set the top-level fields and clear the per-version fields @@ -156,6 +172,9 @@ func Convert_v1_CustomResourceDefinitionSpec_To_apiextensions_CustomResourceDefi if additionalPrinterColumnsIdentical { out.AdditionalPrinterColumns = additionalPrinterColumns } + if selectableFieldsIdentical { + out.SelectableFields = selectableFields + } for i := range out.Versions { if subresourcesIdentical { out.Versions[i].Subresources = nil @@ -166,6 +185,9 @@ func Convert_v1_CustomResourceDefinitionSpec_To_apiextensions_CustomResourceDefi if additionalPrinterColumnsIdentical { out.Versions[i].AdditionalPrinterColumns = nil } + if selectableFieldsIdentical { + out.Versions[i].SelectableFields = nil + } } return nil diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1/conversion_test.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1/conversion_test.go index ab64ece6428..ce3834e10ea 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1/conversion_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1/conversion_test.go @@ -28,6 +28,7 @@ import ( "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" "k8s.io/apimachinery/pkg/runtime" "k8s.io/utils/pointer" + "k8s.io/utils/ptr" ) func TestConversion(t *testing.T) { @@ -85,7 +86,7 @@ func TestConversion(t *testing.T) { Out: &apiextensions.CustomResourceDefinition{}, ExpectOut: &apiextensions.CustomResourceDefinition{ Spec: apiextensions.CustomResourceDefinitionSpec{ - PreserveUnknownFields: pointer.BoolPtr(false), + PreserveUnknownFields: ptr.To(false), }, }, }, @@ -101,7 +102,7 @@ func TestConversion(t *testing.T) { Spec: apiextensions.CustomResourceDefinitionSpec{ Version: "v1", Versions: []apiextensions.CustomResourceDefinitionVersion{{Name: "v1", Served: true, Storage: true}}, - PreserveUnknownFields: pointer.BoolPtr(false), + PreserveUnknownFields: ptr.To(false), }, }, }, @@ -123,7 +124,7 @@ func TestConversion(t *testing.T) { {Name: "v1", Served: true, Storage: true}, {Name: "v2", Served: false, Storage: false}, }, - PreserveUnknownFields: pointer.BoolPtr(false), + PreserveUnknownFields: ptr.To(false), }, }, }, @@ -184,7 +185,7 @@ func TestConversion(t *testing.T) { {Name: "v2", Served: true, Storage: false}, }, Validation: &apiextensions.CustomResourceValidation{OpenAPIV3Schema: &apiextensions.JSONSchemaProps{Type: "object"}}, - PreserveUnknownFields: pointer.BoolPtr(false), + PreserveUnknownFields: ptr.To(false), }, }, }, @@ -206,7 +207,7 @@ func TestConversion(t *testing.T) { {Name: "v1", Served: true, Storage: true, Schema: &apiextensions.CustomResourceValidation{OpenAPIV3Schema: &apiextensions.JSONSchemaProps{Description: "v1", Type: "object"}}}, {Name: "v2", Served: true, Storage: false, Schema: &apiextensions.CustomResourceValidation{OpenAPIV3Schema: &apiextensions.JSONSchemaProps{Description: "v2", Type: "object"}}}, }, - PreserveUnknownFields: pointer.BoolPtr(false), + PreserveUnknownFields: ptr.To(false), }, }, }, @@ -267,7 +268,7 @@ func TestConversion(t *testing.T) { {Name: "v2", Served: true, Storage: false}, }, Subresources: &apiextensions.CustomResourceSubresources{Scale: &apiextensions.CustomResourceSubresourceScale{SpecReplicasPath: "spec.replicas"}}, - PreserveUnknownFields: pointer.BoolPtr(false), + PreserveUnknownFields: ptr.To(false), }, }, }, @@ -289,7 +290,7 @@ func TestConversion(t *testing.T) { {Name: "v1", Served: true, Storage: true, Subresources: &apiextensions.CustomResourceSubresources{Scale: &apiextensions.CustomResourceSubresourceScale{SpecReplicasPath: "spec.replicas1"}}}, {Name: "v2", Served: true, Storage: false, Subresources: &apiextensions.CustomResourceSubresources{Scale: &apiextensions.CustomResourceSubresourceScale{SpecReplicasPath: "spec.replicas2"}}}, }, - PreserveUnknownFields: pointer.BoolPtr(false), + PreserveUnknownFields: ptr.To(false), }, }, }, @@ -350,7 +351,7 @@ func TestConversion(t *testing.T) { {Name: "v2", Served: true, Storage: false}, }, AdditionalPrinterColumns: []apiextensions.CustomResourceColumnDefinition{{Name: "column1"}}, - PreserveUnknownFields: pointer.BoolPtr(false), + PreserveUnknownFields: ptr.To(false), }, }, }, @@ -372,7 +373,114 @@ func TestConversion(t *testing.T) { {Name: "v1", Served: true, Storage: true, AdditionalPrinterColumns: []apiextensions.CustomResourceColumnDefinition{{Name: "column1"}}}, {Name: "v2", Served: true, Storage: false, AdditionalPrinterColumns: []apiextensions.CustomResourceColumnDefinition{{Name: "column2"}}}, }, - PreserveUnknownFields: pointer.BoolPtr(false), + PreserveUnknownFields: ptr.To(false), + }, + }, + }, + // SelectableFields + { + Name: "internal to v1, top-level selectable fields moves to per-version", + In: &apiextensions.CustomResourceDefinition{ + Spec: apiextensions.CustomResourceDefinitionSpec{ + Version: "v1", + SelectableFields: []apiextensions.SelectableField{{JSONPath: ".spec.x"}}, + Versions: []apiextensions.CustomResourceDefinitionVersion{ + {Name: "v1", Served: true, Storage: true}, + }, + }, + }, + Out: &CustomResourceDefinition{}, + ExpectOut: &CustomResourceDefinition{ + Spec: CustomResourceDefinitionSpec{ + Versions: []CustomResourceDefinitionVersion{ + {Name: "v1", Served: true, Storage: true, SelectableFields: []SelectableField{{JSONPath: ".spec.x"}}}, + }, + }, + }, + }, + { + Name: "internal to v1, per-version selectable fields is preserved", + In: &apiextensions.CustomResourceDefinition{ + Spec: apiextensions.CustomResourceDefinitionSpec{ + Versions: []apiextensions.CustomResourceDefinitionVersion{ + {Name: "v1", Served: true, Storage: true, SelectableFields: []apiextensions.SelectableField{{JSONPath: ".spec.x"}}}, + {Name: "v2", Served: false, Storage: false, SelectableFields: []apiextensions.SelectableField{{JSONPath: ".spec.y"}}}, + }, + }, + }, + Out: &CustomResourceDefinition{}, + ExpectOut: &CustomResourceDefinition{ + Spec: CustomResourceDefinitionSpec{ + Versions: []CustomResourceDefinitionVersion{ + {Name: "v1", Served: true, Storage: true, SelectableFields: []SelectableField{{JSONPath: ".spec.x"}}}, + {Name: "v2", Served: false, Storage: false, SelectableFields: []SelectableField{{JSONPath: ".spec.y"}}}, + }, + }, + }, + }, + { + Name: "v1 to internal, identical selectable fields moves to top-level", + In: &CustomResourceDefinition{ + Spec: CustomResourceDefinitionSpec{ + Versions: []CustomResourceDefinitionVersion{ + {Name: "v1", Served: true, Storage: true, SelectableFields: []SelectableField{{JSONPath: ".spec.x"}}}, + {Name: "v2", Served: true, Storage: false, SelectableFields: []SelectableField{{JSONPath: ".spec.x"}}}, + }, + }, + }, + Out: &apiextensions.CustomResourceDefinition{}, + ExpectOut: &apiextensions.CustomResourceDefinition{ + Spec: apiextensions.CustomResourceDefinitionSpec{ + Version: "v1", + Versions: []apiextensions.CustomResourceDefinitionVersion{ + {Name: "v1", Served: true, Storage: true}, + {Name: "v2", Served: true, Storage: false}, + }, + SelectableFields: []apiextensions.SelectableField{{JSONPath: ".spec.x"}}, + PreserveUnknownFields: ptr.To(false), + }, + }, + }, + { + Name: "v1 to internal, single selectable field moves to top-level", + In: &CustomResourceDefinition{ + Spec: CustomResourceDefinitionSpec{ + Versions: []CustomResourceDefinitionVersion{ + {Name: "v1", Served: true, Storage: true, SelectableFields: []SelectableField{{JSONPath: ".spec.x"}}}, + }, + }, + }, + Out: &apiextensions.CustomResourceDefinition{}, + ExpectOut: &apiextensions.CustomResourceDefinition{ + Spec: apiextensions.CustomResourceDefinitionSpec{ + Version: "v1", + Versions: []apiextensions.CustomResourceDefinitionVersion{ + {Name: "v1", Served: true, Storage: true}, + }, + SelectableFields: []apiextensions.SelectableField{{JSONPath: ".spec.x"}}, + PreserveUnknownFields: ptr.To(false), + }, + }, + }, + { + Name: "v1 to internal, distinct selectable fields remains per-version", + In: &CustomResourceDefinition{ + Spec: CustomResourceDefinitionSpec{ + Versions: []CustomResourceDefinitionVersion{ + {Name: "v1", Served: true, Storage: true, SelectableFields: []SelectableField{{JSONPath: ".spec.x"}}}, + {Name: "v2", Served: true, Storage: false, SelectableFields: []SelectableField{{JSONPath: ".spec.y"}}}, + }, + }, + }, + Out: &apiextensions.CustomResourceDefinition{}, + ExpectOut: &apiextensions.CustomResourceDefinition{ + Spec: apiextensions.CustomResourceDefinitionSpec{ + Version: "v1", + Versions: []apiextensions.CustomResourceDefinitionVersion{ + {Name: "v1", Served: true, Storage: true, SelectableFields: []apiextensions.SelectableField{{JSONPath: ".spec.x"}}}, + {Name: "v2", Served: true, Storage: false, SelectableFields: []apiextensions.SelectableField{{JSONPath: ".spec.y"}}}, + }, + PreserveUnknownFields: ptr.To(false), }, }, }, @@ -442,7 +550,7 @@ func TestConversion(t *testing.T) { ExpectOut: &apiextensions.CustomResourceDefinition{ Spec: apiextensions.CustomResourceDefinitionSpec{ Conversion: &apiextensions.CustomResourceConversion{}, - PreserveUnknownFields: pointer.BoolPtr(false), + PreserveUnknownFields: ptr.To(false), }, }, }, @@ -463,7 +571,7 @@ func TestConversion(t *testing.T) { Conversion: &apiextensions.CustomResourceConversion{ WebhookClientConfig: &apiextensions.WebhookClientConfig{URL: pointer.StringPtr("http://example.com")}, }, - PreserveUnknownFields: pointer.BoolPtr(false), + PreserveUnknownFields: ptr.To(false), }, }, }, @@ -484,7 +592,7 @@ func TestConversion(t *testing.T) { Conversion: &apiextensions.CustomResourceConversion{ ConversionReviewVersions: []string{"v1"}, }, - PreserveUnknownFields: pointer.BoolPtr(false), + PreserveUnknownFields: ptr.To(false), }, }, }, diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1/types.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1/types.go index 7a600454508..e1d1e0be390 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1/types.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1/types.go @@ -199,6 +199,28 @@ type CustomResourceDefinitionVersion struct { // +optional // +listType=atomic AdditionalPrinterColumns []CustomResourceColumnDefinition `json:"additionalPrinterColumns,omitempty" protobuf:"bytes,6,rep,name=additionalPrinterColumns"` + + // selectableFields specifies paths to fields that may be used as field selectors. + // A maximum of 8 selectable fields are allowed. + // See https://kubernetes.io/docs/concepts/overview/working-with-objects/field-selectors + // + // +featureGate=CustomResourceFieldSelectors + // +optional + // +listType=atomic + SelectableFields []SelectableField `json:"selectableFields,omitempty" protobuf:"bytes,9,rep,name=selectableFields"` +} + +// SelectableField specifies the JSON path of a field that may be used with field selectors. +type SelectableField struct { + // jsonPath is a simple JSON path which is evaluated against each custom resource to produce a + // field selector value. + // Only JSON paths without the array notation are allowed. + // Must point to a field of type string, boolean or integer. Types with enum values + // and strings with formats are allowed. + // If jsonPath refers to absent field in a resource, the jsonPath evaluates to an empty string. + // Must not point to metdata fields. + // Required. + JSONPath string `json:"jsonPath" protobuf:"bytes,1,opt,name=jsonPath"` } // CustomResourceColumnDefinition specifies a column for server side printing. diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/types.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/types.go index 4a1dee16f2f..153f7233775 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/types.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/types.go @@ -87,6 +87,14 @@ type CustomResourceDefinitionSpec struct { // +listType=atomic AdditionalPrinterColumns []CustomResourceColumnDefinition `json:"additionalPrinterColumns,omitempty" protobuf:"bytes,8,rep,name=additionalPrinterColumns"` + // selectableFields specifies paths to fields that may be used as field selectors. + // See https://kubernetes.io/docs/concepts/overview/working-with-objects/field-selectors + // + // +featureGate=CustomResourceFieldSelectors + // +optional + // +listType=atomic + SelectableFields []SelectableField `json:"selectableFields,omitempty" protobuf:"bytes,11,rep,name=selectableFields"` + // conversion defines conversion settings for the CRD. // +optional Conversion *CustomResourceConversion `json:"conversion,omitempty" protobuf:"bytes,9,opt,name=conversion"` @@ -232,6 +240,27 @@ type CustomResourceDefinitionVersion struct { // +optional // +listType=atomic AdditionalPrinterColumns []CustomResourceColumnDefinition `json:"additionalPrinterColumns,omitempty" protobuf:"bytes,6,rep,name=additionalPrinterColumns"` + + // selectableFields specifies paths to fields that may be used as field selectors. + // See https://kubernetes.io/docs/concepts/overview/working-with-objects/field-selectors + // + // +featureGate=CustomResourceFieldSelectors + // +optional + // +listType=atomic + SelectableFields []SelectableField `json:"selectableFields,omitempty" protobuf:"bytes,9,rep,name=selectableFields"` +} + +// SelectableField specifies the JSON path of a field that may be used with field selectors. +type SelectableField struct { + // jsonPath is a simple JSON path which is evaluated against each custom resource to produce a + // field selector value. + // Only JSON paths without the array notation are allowed. + // Must point to a field of type string, boolean or integer. Types with enum values + // and strings with formats are allowed. + // If jsonPath refers to absent field in a resource, the jsonPath evaluates to an empty string. + // Must not point to metdata fields. + // Required. + JSONPath string `json:"jsonPath" protobuf:"bytes,1,opt,name=jsonPath"` } // CustomResourceColumnDefinition specifies a column for server side printing. diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation.go index 69ac8d96d2a..3df56bc9bbe 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation.go @@ -59,6 +59,8 @@ const ( StaticEstimatedCostLimit = 10000000 // StaticEstimatedCRDCostLimit represents the largest-allowed total cost for the x-kubernetes-validations rules of a CRD. StaticEstimatedCRDCostLimit = 100000000 + + MaxSelectableFields = 8 ) var supportedValidationReason = sets.NewString( @@ -291,6 +293,18 @@ func validateCustomResourceDefinitionVersion(ctx context.Context, version *apiex for i := range version.AdditionalPrinterColumns { allErrs = append(allErrs, ValidateCustomResourceColumnDefinition(&version.AdditionalPrinterColumns[i], fldPath.Child("additionalPrinterColumns").Index(i))...) } + + if len(version.SelectableFields) > 0 { + if version.Schema == nil || version.Schema.OpenAPIV3Schema == nil { + allErrs = append(allErrs, field.Invalid(fldPath.Child("selectableFields"), "", "selectableFields may only be set when version.schema.openAPIV3Schema is not included")) + } else { + schema, err := structuralschema.NewStructural(version.Schema.OpenAPIV3Schema) + if err != nil { + allErrs = append(allErrs, field.Invalid(fldPath.Child("schema.openAPIV3Schema"), "", err.Error())) + } + allErrs = append(allErrs, ValidateCustomResourceSelectableFields(version.SelectableFields, schema, fldPath.Child("selectableFields"))...) + } + } return allErrs } @@ -453,6 +467,19 @@ func validateCustomResourceDefinitionSpec(ctx context.Context, spec *apiextensio } } + if len(spec.SelectableFields) > 0 { + if spec.Validation == nil { + allErrs = append(allErrs, field.Invalid(fldPath.Child("selectableFields"), "", "selectableFields may only be set when validations.schema is included")) + } else { + schema, err := structuralschema.NewStructural(spec.Validation.OpenAPIV3Schema) + if err != nil { + allErrs = append(allErrs, field.Invalid(fldPath.Child("schema.openAPIV3Schema"), "", err.Error())) + } + + allErrs = append(allErrs, ValidateCustomResourceSelectableFields(spec.SelectableFields, schema, fldPath.Child("selectableFields"))...) + } + } + if (spec.Conversion != nil && spec.Conversion.Strategy != apiextensions.NoneConverter) && (spec.PreserveUnknownFields == nil || *spec.PreserveUnknownFields) { allErrs = append(allErrs, field.Invalid(fldPath.Child("conversion").Child("strategy"), spec.Conversion.Strategy, "must be None if spec.preserveUnknownFields is true")) } @@ -766,6 +793,51 @@ func ValidateCustomResourceColumnDefinition(col *apiextensions.CustomResourceCol return allErrs } +func ValidateCustomResourceSelectableFields(selectableFields []apiextensions.SelectableField, schema *structuralschema.Structural, fldPath *field.Path) (allErrs field.ErrorList) { + uniqueSelectableFields := sets.New[string]() + for i, selectableField := range selectableFields { + indexFldPath := fldPath.Index(i) + if len(selectableField.JSONPath) == 0 { + allErrs = append(allErrs, field.Required(indexFldPath.Child("jsonPath"), "")) + continue + } + // Leverage the field path validation originally built for use with CEL features + path, foundSchema, err := cel.ValidFieldPath(selectableField.JSONPath, schema, cel.WithFieldPathAllowArrayNotation(false)) + if err != nil { + allErrs = append(allErrs, field.Invalid(indexFldPath.Child("jsonPath"), selectableField.JSONPath, fmt.Sprintf("is an invalid path: %v", err))) + continue + } + if path.Root().String() == "metadata" { + allErrs = append(allErrs, field.Invalid(indexFldPath, selectableField.JSONPath, "must not point to fields in metadata")) + } + if !allowedSelectableFieldSchema(foundSchema) { + allErrs = append(allErrs, field.Invalid(indexFldPath, selectableField.JSONPath, "must point to a field of type string, boolean or integer. Enum string fields and strings with formats are allowed.")) + } + if uniqueSelectableFields.Has(path.String()) { + allErrs = append(allErrs, field.Duplicate(indexFldPath, selectableField.JSONPath)) + } else { + uniqueSelectableFields.Insert(path.String()) + } + } + uniqueSelectableFieldCount := uniqueSelectableFields.Len() + if uniqueSelectableFieldCount > MaxSelectableFields { + allErrs = append(allErrs, field.TooMany(fldPath, uniqueSelectableFieldCount, MaxSelectableFields)) + } + return allErrs +} + +func allowedSelectableFieldSchema(schema *structuralschema.Structural) bool { + if schema == nil { + return false + } + switch schema.Type { + case "string", "boolean", "integer": + return true + default: + return false + } +} + // specStandardValidator applies validations for different OpenAPI specification versions. type specStandardValidator interface { validate(spec *apiextensions.JSONSchemaProps, fldPath *field.Path) field.ErrorList @@ -1201,7 +1273,7 @@ func ValidateCustomResourceDefinitionOpenAPISchema(schema *apiextensions.JSONSch func pathValid(schema *apiextensions.JSONSchemaProps, path string) bool { // To avoid duplicated code and better maintain, using ValidaFieldPath func to check if the path is valid if ss, err := structuralschema.NewStructural(schema); err == nil { - _, err := cel.ValidFieldPath(path, ss) + _, _, err := cel.ValidFieldPath(path, ss) return err == nil } return true diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation_test.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation_test.go index 43f10f99fe5..d3520681f22 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation_test.go @@ -31,6 +31,7 @@ import ( apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" celschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel" + apiextensionsfeatures "k8s.io/apiextensions-apiserver/pkg/features" "k8s.io/apimachinery/pkg/api/apitesting/fuzzer" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -40,6 +41,8 @@ import ( "k8s.io/apimachinery/pkg/util/version" "k8s.io/apiserver/pkg/cel/environment" "k8s.io/apiserver/pkg/cel/library" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/utils/pointer" "k8s.io/utils/ptr" ) @@ -71,6 +74,12 @@ func immutable(path ...string) validationMatch { func forbidden(path ...string) validationMatch { return validationMatch{path: field.NewPath(path[0], path[1:]...), errorType: field.ErrorTypeForbidden} } +func duplicate(path ...string) validationMatch { + return validationMatch{path: field.NewPath(path[0], path[1:]...), errorType: field.ErrorTypeDuplicate} +} +func tooMany(path ...string) validationMatch { + return validationMatch{path: field.NewPath(path[0], path[1:]...), errorType: field.ErrorTypeTooMany} +} func (v validationMatch) matches(err *field.Error) bool { return err.Type == v.errorType && err.Field == v.path.String() && strings.Contains(err.Error(), v.containsString) @@ -4363,6 +4372,294 @@ func TestValidateCustomResourceDefinition(t *testing.T) { } } +func TestSelectableFields(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceFieldSelectors, true)() + singleVersionList := []apiextensions.CustomResourceDefinitionVersion{ + { + Name: "version", + Served: true, + Storage: true, + }, + } + tests := []struct { + name string + resource *apiextensions.CustomResourceDefinition + errors []validationMatch + }{ + { + name: "selectableFields with jsonPaths that do not refer to a field in the schema are invalid", + resource: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "plural.group.com"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Group: "group.com", + Version: "version", + Versions: []apiextensions.CustomResourceDefinitionVersion{ + {Name: "version", Served: true, Storage: true, + Schema: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{"foo": {Type: "string"}}, + Required: []string{"foo"}, + }, + }, + SelectableFields: []apiextensions.SelectableField{{JSONPath: ".foo"}, {JSONPath: ".xyz"}}, + }, + {Name: "version2", Served: true, Storage: false, + Schema: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{"foo": {Type: "integer"}}, + Required: []string{"foo"}, + }, + }, + SelectableFields: []apiextensions.SelectableField{{JSONPath: ".xyz"}, {JSONPath: ".foo"}, {JSONPath: ".abc"}}, + }, + }, + Scope: apiextensions.NamespaceScoped, + Names: apiextensions.CustomResourceDefinitionNames{ + Plural: "plural", + Singular: "singular", + Kind: "Plural", + ListKind: "PluralList", + }, + PreserveUnknownFields: ptr.To(false), + }, + Status: apiextensions.CustomResourceDefinitionStatus{ + StoredVersions: []string{"version"}, + }, + }, + errors: []validationMatch{ + invalid("spec", "versions[0]", "selectableFields[1]"), + invalid("spec", "versions[1]", "selectableFields[0]"), + invalid("spec", "versions[1]", "selectableFields[2]"), + }, + }, + { + name: "in top level schema, selectableFields with jsonPaths that do not refer to a field in the schema are invalid", + resource: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "plural.group.com"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Group: "group.com", + Version: "version", + Versions: singleVersionList, + Validation: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{ + "spec": { + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{"foo": {Type: "string"}}, + Required: []string{"foo"}, + }, + "status": { + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{"phase": {Type: "string"}}, + Required: []string{"phase"}, + }, + }, + }, + }, + SelectableFields: []apiextensions.SelectableField{{JSONPath: ".spec.foo"}, {JSONPath: ".spec.xyz"}, {JSONPath: ".status.phase"}}, + Scope: apiextensions.NamespaceScoped, + Names: apiextensions.CustomResourceDefinitionNames{ + Plural: "plural", + Singular: "singular", + Kind: "Plural", + ListKind: "PluralList", + }, + PreserveUnknownFields: ptr.To(false), + }, + Status: apiextensions.CustomResourceDefinitionStatus{ + StoredVersions: []string{"version"}, + }, + }, + errors: []validationMatch{ + invalid("spec", "selectableFields[1]"), + }, + }, + { + name: "selectableFields with jsonPaths that do not refer to fields that are not strings, booleans or integers are invalid", + resource: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "plural.group.com"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Group: "group.com", + Version: "version", + Versions: []apiextensions.CustomResourceDefinitionVersion{ + {Name: "version", Served: true, Storage: true, + Schema: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{"foo": {Type: "string"}, "obj": {Type: "object"}}, + Required: []string{"foo", "obj"}, + }, + }, + SelectableFields: []apiextensions.SelectableField{{JSONPath: ".foo"}, {JSONPath: ".obj"}}, + }, + {Name: "version2", Served: true, Storage: false, + Schema: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{"foo": {Type: "integer"}, "obj": {Type: "object"}, "bool": {Type: "boolean"}}, + Required: []string{"foo", "obj", "bool"}, + }, + }, + SelectableFields: []apiextensions.SelectableField{{JSONPath: ".obj"}, {JSONPath: ".foo"}, {JSONPath: ".bool"}}, + }, + }, + Scope: apiextensions.NamespaceScoped, + Names: apiextensions.CustomResourceDefinitionNames{ + Plural: "plural", + Singular: "singular", + Kind: "Plural", + ListKind: "PluralList", + }, + PreserveUnknownFields: ptr.To(false), + }, + Status: apiextensions.CustomResourceDefinitionStatus{ + StoredVersions: []string{"version"}, + }, + }, + errors: []validationMatch{ + invalid("spec", "versions[0]", "selectableFields[1]"), + invalid("spec", "versions[1]", "selectableFields[0]"), + }, + }, + { + name: "selectableFields with duplicate jsonPaths are invalid", + resource: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "plural.group.com"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Group: "group.com", + Version: "version", + Versions: []apiextensions.CustomResourceDefinitionVersion{ + {Name: "version", Served: true, Storage: true, + Schema: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{"foo": {Type: "string"}}, + Required: []string{"foo"}, + }, + }, + SelectableFields: []apiextensions.SelectableField{{JSONPath: ".foo"}, {JSONPath: ".foo"}}, + }, + {Name: "version2", Served: true, Storage: false, + Schema: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{"foo": {Type: "integer"}}, + Required: []string{"foo"}, + }, + }, + SelectableFields: []apiextensions.SelectableField{{JSONPath: ".foo"}, {JSONPath: ".foo"}}, + }, + }, + Scope: apiextensions.NamespaceScoped, + Names: apiextensions.CustomResourceDefinitionNames{ + Plural: "plural", + Singular: "singular", + Kind: "Plural", + ListKind: "PluralList", + }, + PreserveUnknownFields: ptr.To(false), + }, + Status: apiextensions.CustomResourceDefinitionStatus{ + StoredVersions: []string{"version"}, + }, + }, + errors: []validationMatch{ + duplicate("spec", "versions[0]", "selectableFields[1]"), + duplicate("spec", "versions[1]", "selectableFields[1]"), + }, + }, + { + name: "too many selectableFields are not allowed", + resource: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "plural.group.com"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Group: "group.com", + Version: "version", + Versions: []apiextensions.CustomResourceDefinitionVersion{ + {Name: "version", Served: true, Storage: true, + Schema: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{ + "a1": {Type: "string"}, "a2": {Type: "string"}, "a3": {Type: "string"}, + "a4": {Type: "string"}, "a5": {Type: "string"}, "a6": {Type: "string"}, + "a7": {Type: "string"}, "a8": {Type: "string"}, "a9": {Type: "string"}, + }, + Required: []string{"a1", "a2", "a3", "a4", "a5", "a6", "a7", "a8", "a9"}, + }, + }, + SelectableFields: []apiextensions.SelectableField{ + {JSONPath: ".a1"}, {JSONPath: ".a2"}, {JSONPath: ".a3"}, + {JSONPath: ".a4"}, {JSONPath: ".a5"}, {JSONPath: ".a6"}, + {JSONPath: ".a7"}, {JSONPath: ".a8"}, {JSONPath: ".a9"}, + }, + }, + {Name: "version2", Served: true, Storage: false, + Schema: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{"foo": {Type: "integer"}}, + }, + }, + }, + }, + Scope: apiextensions.NamespaceScoped, + Names: apiextensions.CustomResourceDefinitionNames{ + Plural: "plural", + Singular: "singular", + Kind: "Plural", + ListKind: "PluralList", + }, + PreserveUnknownFields: ptr.To(false), + }, + Status: apiextensions.CustomResourceDefinitionStatus{ + StoredVersions: []string{"version"}, + }, + }, + errors: []validationMatch{ + tooMany("spec", "versions[0]", "selectableFields"), + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + + // duplicate defaulting behaviour + if tc.resource.Spec.Conversion != nil && tc.resource.Spec.Conversion.Strategy == apiextensions.WebhookConverter && len(tc.resource.Spec.Conversion.ConversionReviewVersions) == 0 { + tc.resource.Spec.Conversion.ConversionReviewVersions = []string{"v1beta1"} + } + ctx := context.TODO() + errs := ValidateCustomResourceDefinition(ctx, tc.resource) + seenErrs := make([]bool, len(errs)) + + for _, expectedError := range tc.errors { + found := false + for i, err := range errs { + if expectedError.matches(err) && !seenErrs[i] { + found = true + seenErrs[i] = true + break + } + } + + if !found { + t.Errorf("expected %v at %v, got %v", expectedError.errorType, expectedError.path.String(), errs) + } + } + + for i, seen := range seenErrs { + if !seen { + t.Errorf("unexpected error: %v", errs[i]) + } + } + }) + } +} + func TestValidateFieldPath(t *testing.T) { schema := apiextensions.JSONSchemaProps{ Type: "object", @@ -4589,7 +4886,7 @@ func TestValidateFieldPath(t *testing.T) { if err != nil { t.Fatalf("error when converting schema to structural schema: %v", err) } - _, err = celschema.ValidFieldPath(tc.fieldPath, ss) + _, _, err = celschema.ValidFieldPath(tc.fieldPath, ss) if err == nil && tc.errMsg != "" { t.Errorf("expected err contains: %v but get nil", tc.errMsg) } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/converter.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/converter.go index 96b2b89e0e8..7fa43af8eec 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/converter.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/converter.go @@ -18,12 +18,16 @@ package conversion import ( "fmt" + "strings" autoscalingv1 "k8s.io/api/autoscaling/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apiextensionsfeatures "k8s.io/apiextensions-apiserver/pkg/features" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/apiserver/pkg/util/webhook" typedscheme "k8s.io/client-go/kubernetes/scheme" ) @@ -76,17 +80,27 @@ func (m *CRConverterFactory) NewConverter(crd *apiextensionsv1.CustomResourceDef // Determine whether we should expect to be asked to "convert" autoscaling/v1 Scale types convertScale := false + selectableFields := map[schema.GroupVersion]sets.Set[string]{} for _, version := range crd.Spec.Versions { + gv := schema.GroupVersion{Group: crd.Spec.Group, Version: version.Name} if version.Subresources != nil && version.Subresources.Scale != nil { convertScale = true } + if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceFieldSelectors) { + fieldPaths := sets.New[string]() + for _, sf := range version.SelectableFields { + fieldPaths.Insert(strings.TrimPrefix(sf.JSONPath, ".")) + } + selectableFields[gv] = fieldPaths + } } unsafe = &crConverter{ - convertScale: convertScale, - validVersions: validVersions, - clusterScoped: crd.Spec.Scope == apiextensionsv1.ClusterScoped, - converter: converter, + convertScale: convertScale, + validVersions: validVersions, + clusterScoped: crd.Spec.Scope == apiextensionsv1.ClusterScoped, + converter: converter, + selectableFields: selectableFields, } return &safeConverterWrapper{unsafe}, unsafe, nil } @@ -102,20 +116,26 @@ type crConverterInterface interface { // crConverter extends the delegate converter with generic CR conversion behaviour. The delegate will implement the // user defined conversion strategy given in the CustomResourceDefinition. type crConverter struct { - convertScale bool - converter crConverterInterface - validVersions map[schema.GroupVersion]bool - clusterScoped bool + convertScale bool + converter crConverterInterface + validVersions map[schema.GroupVersion]bool + clusterScoped bool + selectableFields map[schema.GroupVersion]sets.Set[string] } func (c *crConverter) ConvertFieldLabel(gvk schema.GroupVersionKind, label, value string) (string, string, error) { - // We currently only support metadata.namespace and metadata.name. switch { case label == "metadata.name": return label, value, nil case !c.clusterScoped && label == "metadata.namespace": return label, value, nil default: + if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceFieldSelectors) { + groupFields := c.selectableFields[gvk.GroupVersion()] + if groupFields != nil && groupFields.Has(label) { + return label, value, nil + } + } return "", "", fmt.Errorf("field label not supported: %s", label) } } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go index 7a55d8e5453..9c0ee416f52 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go @@ -826,6 +826,7 @@ func (r *crdHandler) getOrCreateServingInfoFor(uid types.UID, name string) (*crd structuralSchemas[v.Name], statusSpec, scaleSpec, + v.SelectableFields, ), crdConversionRESTOptionsGetter{ RESTOptionsGetter: r.restOptionsGetter, diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/compilation.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/compilation.go index b5035eadb26..4f065a18fb7 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/compilation.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/compilation.go @@ -282,7 +282,7 @@ func compileRule(s *schema.Structural, rule apiextensions.ValidationRule, envSet compilationResult.MessageExpressionMaxCost = costEst.Max } if rule.FieldPath != "" { - validFieldPath, err := ValidFieldPath(rule.FieldPath, s) + validFieldPath, _, err := ValidFieldPath(rule.FieldPath, s) if err == nil { compilationResult.NormalizedRuleFieldPath = validFieldPath.String() } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/validation.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/validation.go index 7ee9dbebbf7..b4c8afa9af7 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/validation.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/validation.go @@ -31,9 +31,6 @@ import ( "github.com/google/cel-go/common/types/ref" "github.com/google/cel-go/interpreter" - "k8s.io/klog/v2" - "k8s.io/utils/ptr" - apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/model" @@ -45,6 +42,8 @@ import ( "k8s.io/apiserver/pkg/cel/metrics" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/apiserver/pkg/warning" + "k8s.io/klog/v2" + "k8s.io/utils/ptr" celconfig "k8s.io/apiserver/pkg/apis/cel" ) @@ -441,9 +440,30 @@ func unescapeSingleQuote(s string) (string, error) { return unescaped, err } +type validFieldPathOptions struct { + allowArrayNotation bool +} + +// ValidFieldPathOption provides vararg options for ValidFieldPath. +type ValidFieldPathOption func(*validFieldPathOptions) + +// WithFieldPathAllowArrayNotation sets of array annotation ('[]') is allowed +// in field paths. +// Defaults to true +func WithFieldPathAllowArrayNotation(allow bool) ValidFieldPathOption { + return func(options *validFieldPathOptions) { + options.allowArrayNotation = allow + } +} + // ValidFieldPath validates that jsonPath is a valid JSON Path containing only field and map accessors // that are valid for the given schema, and returns a field.Path representation of the validated jsonPath or an error. -func ValidFieldPath(jsonPath string, schema *schema.Structural) (validFieldPath *field.Path, err error) { +func ValidFieldPath(jsonPath string, schema *schema.Structural, options ...ValidFieldPathOption) (validFieldPath *field.Path, foundSchema *schema.Structural, err error) { + opts := &validFieldPathOptions{allowArrayNotation: true} + for _, opt := range options { + opt(opts) + } + appendToPath := func(name string, isNamed bool) error { if !isNamed { validFieldPath = validFieldPath.Key(name) @@ -504,16 +524,19 @@ func ValidFieldPath(jsonPath string, schema *schema.Structural) (validFieldPath tok = scanner.Text() switch tok { case "[": + if !opts.allowArrayNotation { + return nil, nil, fmt.Errorf("array notation is not allowed") + } if !scanner.Scan() { - return nil, fmt.Errorf("unexpected end of JSON path") + return nil, nil, fmt.Errorf("unexpected end of JSON path") } tok = scanner.Text() if len(tok) < 2 || tok[0] != '\'' || tok[len(tok)-1] != '\'' { - return nil, fmt.Errorf("expected single quoted string but got %s", tok) + return nil, nil, fmt.Errorf("expected single quoted string but got %s", tok) } unescaped, err := unescapeSingleQuote(tok[1 : len(tok)-1]) if err != nil { - return nil, fmt.Errorf("invalid string literal: %v", err) + return nil, nil, fmt.Errorf("invalid string literal: %w", err) } if schema.Properties != nil { @@ -521,21 +544,21 @@ func ValidFieldPath(jsonPath string, schema *schema.Structural) (validFieldPath } else if schema.AdditionalProperties != nil { isNamed = false } else { - return nil, fmt.Errorf("does not refer to a valid field") + return nil, nil, fmt.Errorf("does not refer to a valid field") } if err := appendToPath(unescaped, isNamed); err != nil { - return nil, err + return nil, nil, err } if !scanner.Scan() { - return nil, fmt.Errorf("unexpected end of JSON path") + return nil, nil, fmt.Errorf("unexpected end of JSON path") } tok = scanner.Text() if tok != "]" { - return nil, fmt.Errorf("expected ] but got %s", tok) + return nil, nil, fmt.Errorf("expected ] but got %s", tok) } case ".": if !scanner.Scan() { - return nil, fmt.Errorf("unexpected end of JSON path") + return nil, nil, fmt.Errorf("unexpected end of JSON path") } tok = scanner.Text() if schema.Properties != nil { @@ -543,16 +566,17 @@ func ValidFieldPath(jsonPath string, schema *schema.Structural) (validFieldPath } else if schema.AdditionalProperties != nil { isNamed = false } else { - return nil, fmt.Errorf("does not refer to a valid field") + return nil, nil, fmt.Errorf("does not refer to a valid field") } if err := appendToPath(tok, isNamed); err != nil { - return nil, err + return nil, nil, err } default: - return nil, fmt.Errorf("expected [ or . but got: %s", tok) + return nil, nil, fmt.Errorf("expected [ or . but got: %s", tok) } } - return validFieldPath, nil + + return validFieldPath, schema, nil } func fieldErrorForReason(fldPath *field.Path, value interface{}, detail string, reason *apiextensions.FieldValueErrorReason) *field.Error { diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/validation_test.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/validation_test.go index c5fe2e081b2..7681970ad11 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/validation_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/validation_test.go @@ -3119,7 +3119,7 @@ func TestValidateFieldPath(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - validField, err := ValidFieldPath(tc.fieldPath, tc.schema) + validField, _, err := ValidFieldPath(tc.fieldPath, tc.schema) if err == nil && tc.errDetail != "" { t.Errorf("expected err contains: %v but get nil", tc.errDetail) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation/metrics_test.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation/metrics_test.go index 56775866cca..dbf224ec248 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation/metrics_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation/metrics_test.go @@ -275,6 +275,7 @@ func TestMetrics(t *testing.T) { sts, nil, nil, + nil, ) iters := 1 diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go index e2340bc704d..c27c38abc5f 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go @@ -27,11 +27,14 @@ import ( "github.com/spf13/pflag" - "k8s.io/apiextensions-apiserver/pkg/apiserver" + extensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" "k8s.io/apiextensions-apiserver/pkg/cmd/server/options" + generatedopenapi "k8s.io/apiextensions-apiserver/pkg/generated/openapi" "k8s.io/apimachinery/pkg/util/wait" + openapinamer "k8s.io/apiserver/pkg/endpoints/openapi" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/storage/storagebackend" + "k8s.io/apiserver/pkg/util/openapi" "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" logsapi "k8s.io/component-base/logs/api/v1" @@ -58,7 +61,7 @@ type TestServer struct { ServerOpts *options.CustomResourceDefinitionsServerOptions // ServerOpts TearDownFn TearDownFunc // TearDown function TmpDir string // Temp Dir used, by the apiserver - CompletedConfig apiserver.CompletedConfig + CompletedConfig extensionsapiserver.CompletedConfig } // Logger allows t.Testing and b.Testing to be passed to StartTestServer and StartTestServerOrDie @@ -151,6 +154,11 @@ func StartTestServer(t Logger, _ *TestServerInstanceOptions, customFlags []strin if err != nil { return result, fmt.Errorf("failed to create config from options: %v", err) } + + getOpenAPIDefinitions := openapi.GetOpenAPIDefinitionsWithoutDisabledFeatures(generatedopenapi.GetOpenAPIDefinitions) + namer := openapinamer.NewDefinitionNamer(extensionsapiserver.Scheme) + config.GenericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(getOpenAPIDefinitions, namer) + completedConfig := config.Complete() server, err := completedConfig.New(genericapiserver.NewEmptyDelegate()) if err != nil { diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder/builder.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder/builder.go index 676c8ad36b9..45e4604d3e0 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder/builder.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder/builder.go @@ -97,6 +97,8 @@ type Options struct { // AllowNonStructural indicates swagger should be built for a schema that fits into the structural type but does not meet all structural invariants AllowNonStructural bool + + IncludeSelectableFields bool } func generateBuilder(crd *apiextensionsv1.CustomResourceDefinition, version string, opts Options) (*builder, error) { @@ -313,12 +315,12 @@ func (b *builder) buildRoute(root, path, httpMethod, actionVerb, operationVerb s Doc(b.descriptionFor(path, operationVerb)). Param(b.ws.QueryParameter("pretty", "If 'true', then the output is pretty printed. Defaults to 'false' unless the user-agent indicates a browser or command-line HTTP tool (curl and wget).")). Operation(operationVerb+namespaced+b.kind+strings.Title(subresource(path))). - Metadata(endpoints.ROUTE_META_GVK, metav1.GroupVersionKind{ + Metadata(endpoints.RouteMetaGVK, metav1.GroupVersionKind{ Group: b.group, Version: b.version, Kind: b.kind, }). - Metadata(endpoints.ROUTE_META_ACTION, actionVerb). + Metadata(endpoints.RouteMetaAction, actionVerb). Produces("application/json", "application/yaml"). Returns(http.StatusOK, "OK", sample). Writes(sample) @@ -374,7 +376,7 @@ func (b *builder) buildRoute(root, path, httpMethod, actionVerb, operationVerb s // buildKubeNative builds input schema with Kubernetes' native object meta, type meta and // extensions -func (b *builder) buildKubeNative(schema *structuralschema.Structural, opts Options, crdPreserveUnknownFields bool) (ret *spec.Schema) { +func (b *builder) buildKubeNative(crd *apiextensionsv1.CustomResourceDefinition, schema *structuralschema.Structural, opts Options, crdPreserveUnknownFields bool) (ret *spec.Schema) { // only add properties if we have a schema. Otherwise, kubectl would (wrongly) assume additionalProperties=false // and forbid anything outside of apiVersion, kind and metadata. We have to fix kubectl to stop doing this, e.g. by // adding additionalProperties=true support to explicitly allow additional fields. @@ -395,7 +397,7 @@ func (b *builder) buildKubeNative(schema *structuralschema.Structural, opts Opti addTypeMetaProperties(ret, opts.V2) addEmbeddedProperties(ret, opts) } - ret.AddExtension(endpoints.ROUTE_META_GVK, []interface{}{ + ret.AddExtension(endpoints.RouteMetaGVK, []interface{}{ map[string]interface{}{ "group": b.group, "version": b.version, @@ -403,6 +405,12 @@ func (b *builder) buildKubeNative(schema *structuralschema.Structural, opts Opti }, }) + if opts.IncludeSelectableFields { + if selectableFields := buildSelectableFields(crd, b.version); selectableFields != nil { + ret.AddExtension(endpoints.RouteMetaSelectableFields, selectableFields) + } + } + return ret } @@ -486,24 +494,29 @@ func addTypeMetaProperties(s *spec.Schema, v2 bool) { } // buildListSchema builds the list kind schema for the CRD -func (b *builder) buildListSchema(v2 bool) *spec.Schema { +func (b *builder) buildListSchema(crd *apiextensionsv1.CustomResourceDefinition, opts Options) *spec.Schema { name := definitionPrefix + util.ToRESTFriendlyName(fmt.Sprintf("%s/%s/%s", b.group, b.version, b.kind)) doc := fmt.Sprintf("List of %s. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md", b.plural) s := new(spec.Schema). Typed("object", ""). WithDescription(fmt.Sprintf("%s is a list of %s", b.listKind, b.kind)). WithRequired("items"). - SetProperty("items", *spec.ArrayProperty(spec.RefSchema(refForOpenAPIVersion(name, v2))).WithDescription(doc)). - SetProperty("metadata", *spec.RefSchema(refForOpenAPIVersion(listMetaSchemaRef, v2)).WithDescription(swaggerPartialObjectMetadataListDescriptions["metadata"])) + SetProperty("items", *spec.ArrayProperty(spec.RefSchema(refForOpenAPIVersion(name, opts.V2))).WithDescription(doc)). + SetProperty("metadata", *spec.RefSchema(refForOpenAPIVersion(listMetaSchemaRef, opts.V2)).WithDescription(swaggerPartialObjectMetadataListDescriptions["metadata"])) - addTypeMetaProperties(s, v2) - s.AddExtension(endpoints.ROUTE_META_GVK, []map[string]string{ + addTypeMetaProperties(s, opts.V2) + s.AddExtension(endpoints.RouteMetaGVK, []map[string]string{ { "group": b.group, "version": b.version, "kind": b.listKind, }, }) + if opts.IncludeSelectableFields { + if selectableFields := buildSelectableFields(crd, b.version); selectableFields != nil { + s.AddExtension(endpoints.RouteMetaSelectableFields, selectableFields) + } + } return s } @@ -596,8 +609,29 @@ func newBuilder(crd *apiextensionsv1.CustomResourceDefinition, version string, s } // Pre-build schema with Kubernetes native properties - b.schema = b.buildKubeNative(schema, opts, crd.Spec.PreserveUnknownFields) - b.listSchema = b.buildListSchema(opts.V2) + b.schema = b.buildKubeNative(crd, schema, opts, crd.Spec.PreserveUnknownFields) + b.listSchema = b.buildListSchema(crd, opts) return b } + +func buildSelectableFields(crd *apiextensionsv1.CustomResourceDefinition, version string) any { + var specVersion *apiextensionsv1.CustomResourceDefinitionVersion + for _, v := range crd.Spec.Versions { + if v.Name == version { + specVersion = &v + break + } + } + if specVersion == nil && len(specVersion.SelectableFields) == 0 { + return nil + } + selectableFields := make([]any, len(specVersion.SelectableFields)) + for i, sf := range specVersion.SelectableFields { + props := map[string]any{ + "fieldPath": strings.TrimPrefix(sf.JSONPath, "."), + } + selectableFields[i] = props + } + return selectableFields +} diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder/builder_test.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder/builder_test.go index 10173fef267..0283bcc4b82 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder/builder_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder/builder_test.go @@ -33,6 +33,7 @@ import ( "k8s.io/apiserver/pkg/endpoints" "k8s.io/kube-openapi/pkg/validation/spec" utilpointer "k8s.io/utils/pointer" + "k8s.io/utils/ptr" ) func TestNewBuilder(t *testing.T) { @@ -353,7 +354,7 @@ func TestCRDRouteParameterBuilder(t *testing.T) { actions := sets.NewString() for _, operation := range []*spec.Operation{path.Get, path.Post, path.Put, path.Patch, path.Delete} { if operation != nil { - action, ok := operation.VendorExtensible.Extensions.GetString(endpoints.ROUTE_META_ACTION) + action, ok := operation.VendorExtensible.Extensions.GetString(endpoints.RouteMetaAction) if ok { actions.Insert(action) } @@ -391,48 +392,57 @@ func TestBuildOpenAPIV2(t *testing.T) { preserveUnknownFields *bool wantedSchema string opts Options + selectableFields []apiextensionsv1.SelectableField }{ { - "nil", - "", - nil, - `{"type":"object","x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, - Options{V2: true}, + name: "nil", + wantedSchema: `{"type":"object","x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, + opts: Options{V2: true}, }, { - "with properties", - `{"type":"object","properties":{"spec":{"type":"object"},"status":{"type":"object"}}}`, - nil, - `{"type":"object","properties":{"apiVersion":{"type":"string"},"kind":{"type":"string"},"metadata":{"$ref":"#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta"},"spec":{"type":"object"},"status":{"type":"object"}},"x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, - Options{V2: true}, + name: "with properties", + schema: `{"type":"object","properties":{"spec":{"type":"object"},"status":{"type":"object"}}}`, + wantedSchema: `{"type":"object","properties":{"apiVersion":{"type":"string"},"kind":{"type":"string"},"metadata":{"$ref":"#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta"},"spec":{"type":"object"},"status":{"type":"object"}},"x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, + opts: Options{V2: true}, }, { - "with invalid-typed properties", - `{"type":"object","properties":{"spec":{"type":"bug"},"status":{"type":"object"}}}`, - nil, - `{"type":"object","x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, - Options{V2: true}, + name: "with invalid-typed properties", + schema: `{"type":"object","properties":{"spec":{"type":"bug"},"status":{"type":"object"}}}`, + wantedSchema: `{"type":"object","x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, + opts: Options{V2: true}, }, { - "with non-structural schema", - `{"type":"object","properties":{"foo":{"type":"array"}}}`, - nil, - `{"type":"object","x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, - Options{V2: true}, + name: "with non-structural schema", + schema: `{"type":"object","properties":{"foo":{"type":"array"}}}`, + wantedSchema: `{"type":"object","x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, + opts: Options{V2: true}, }, { - "with spec.preseveUnknownFields=true", - `{"type":"object","properties":{"foo":{"type":"string"}}}`, - utilpointer.BoolPtr(true), - `{"type":"object","x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, - Options{V2: true}, + name: "with spec.preseveUnknownFields=true", + schema: `{"type":"object","properties":{"foo":{"type":"string"}}}`, + preserveUnknownFields: ptr.To(true), + wantedSchema: `{"type":"object","x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, + opts: Options{V2: true}, }, { - "v2", - `{"type":"object","properties":{"foo":{"type":"string","oneOf":[{"pattern":"a"},{"pattern":"b"}]}}}`, - nil, - `{"type":"object","properties":{"apiVersion":{"type":"string"},"kind":{"type":"string"},"metadata":{"$ref":"#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta"},"foo":{"type":"string"}},"x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, - Options{V2: true}, + name: "v2", + schema: `{"type":"object","properties":{"foo":{"type":"string","oneOf":[{"pattern":"a"},{"pattern":"b"}]}}}`, + wantedSchema: `{"type":"object","properties":{"apiVersion":{"type":"string"},"kind":{"type":"string"},"metadata":{"$ref":"#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta"},"foo":{"type":"string"}},"x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, + opts: Options{V2: true}, + }, + { + name: "with selectable fields enabled", + schema: `{"type":"object","properties":{"foo":{"type":"string"}}}`, + wantedSchema: `{"type":"object","properties":{"apiVersion":{"type":"string"},"kind":{"type":"string"},"metadata":{"$ref":"#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta"},"foo":{"type":"string"}},"x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}], "x-kubernetes-selectable-fields": [{"fieldPath":"foo"}]}`, + opts: Options{V2: true, IncludeSelectableFields: true}, + selectableFields: []apiextensionsv1.SelectableField{{JSONPath: "foo"}}, + }, + { + name: "with selectable fields disabled", + schema: `{"type":"object","properties":{"foo":{"type":"string"}}}`, + wantedSchema: `{"type":"object","properties":{"apiVersion":{"type":"string"},"kind":{"type":"string"},"metadata":{"$ref":"#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta"},"foo":{"type":"string"}},"x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, + opts: Options{V2: true}, + selectableFields: []apiextensionsv1.SelectableField{{JSONPath: "foo"}}, }, } for _, tt := range tests { @@ -457,8 +467,9 @@ func TestBuildOpenAPIV2(t *testing.T) { Group: "bar.k8s.io", Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ { - Name: "v1", - Schema: validation, + Name: "v1", + Schema: validation, + SelectableFields: tt.selectableFields, }, }, Names: apiextensionsv1.CustomResourceDefinitionNames{ @@ -509,34 +520,39 @@ func TestBuildOpenAPIV3(t *testing.T) { preserveUnknownFields *bool wantedSchema string opts Options + selectableFields []apiextensionsv1.SelectableField }{ { - "nil", - "", - nil, - `{"type":"object","x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, - Options{}, + name: "nil", + wantedSchema: `{"type":"object","x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, }, { - "with properties", - `{"type":"object","properties":{"spec":{"type":"object"},"status":{"type":"object"}}}`, - nil, - `{"type":"object","properties":{"apiVersion":{"type":"string"},"kind":{"type":"string"},"metadata":{"allOf":[{"$ref":"#/components/schemas/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta"}]},"spec":{"type":"object"},"status":{"type":"object"}},"x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, - Options{}, + name: "with properties", + schema: `{"type":"object","properties":{"spec":{"type":"object"},"status":{"type":"object"}}}`, + wantedSchema: `{"type":"object","properties":{"apiVersion":{"type":"string"},"kind":{"type":"string"},"metadata":{"allOf":[{"$ref":"#/components/schemas/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta"}]},"spec":{"type":"object"},"status":{"type":"object"}},"x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, }, { - "with v3 nullable field", - `{"type":"object","properties":{"spec":{"type":"object", "nullable": true},"status":{"type":"object"}}}`, - nil, - `{"type":"object","properties":{"apiVersion":{"type":"string"},"kind":{"type":"string"},"metadata":{"allOf":[{"$ref":"#/components/schemas/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta"}]},"spec":{"type":"object", "nullable": true},"status":{"type":"object"}},"x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, - Options{}, + name: "with v3 nullable field", + schema: `{"type":"object","properties":{"spec":{"type":"object", "nullable": true},"status":{"type":"object"}}}`, + wantedSchema: `{"type":"object","properties":{"apiVersion":{"type":"string"},"kind":{"type":"string"},"metadata":{"allOf":[{"$ref":"#/components/schemas/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta"}]},"spec":{"type":"object", "nullable": true},"status":{"type":"object"}},"x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, }, { - "with default not pruned for v3", - `{"type":"object","properties":{"spec":{"type":"object","properties":{"field":{"type":"string","default":"foo"}}},"status":{"type":"object"}}}`, - nil, - `{"type":"object","properties":{"apiVersion":{"type":"string"},"kind":{"type":"string"},"metadata":{"allOf":[{"$ref":"#/components/schemas/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta"}]},"spec":{"type":"object","properties":{"field":{"type":"string","default":"foo"}}},"status":{"type":"object"}},"x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, - Options{}, + name: "with default not pruned for v3", + schema: `{"type":"object","properties":{"spec":{"type":"object","properties":{"field":{"type":"string","default":"foo"}}},"status":{"type":"object"}}}`, + wantedSchema: `{"type":"object","properties":{"apiVersion":{"type":"string"},"kind":{"type":"string"},"metadata":{"allOf":[{"$ref":"#/components/schemas/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta"}]},"spec":{"type":"object","properties":{"field":{"type":"string","default":"foo"}}},"status":{"type":"object"}},"x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, + }, + { + name: "with selectable fields enabled", + schema: `{"type":"object","properties":{"spec":{"type":"object","properties":{"field":{"type":"string","default":"foo"}}},"status":{"type":"object"}}}`, + wantedSchema: `{"type":"object","properties":{"apiVersion":{"type":"string"},"kind":{"type":"string"},"metadata":{"allOf":[{"$ref":"#/components/schemas/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta"}]},"spec":{"type":"object","properties":{"field":{"type":"string","default":"foo"}}},"status":{"type":"object"}},"x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}], "x-kubernetes-selectable-fields": [{"fieldPath":"spec.field"}]}`, + opts: Options{IncludeSelectableFields: true}, + selectableFields: []apiextensionsv1.SelectableField{{JSONPath: "spec.field"}}, + }, + { + name: "with selectable fields disabled", + schema: `{"type":"object","properties":{"spec":{"type":"object","properties":{"field":{"type":"string","default":"foo"}}},"status":{"type":"object"}}}`, + wantedSchema: `{"type":"object","properties":{"apiVersion":{"type":"string"},"kind":{"type":"string"},"metadata":{"allOf":[{"$ref":"#/components/schemas/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta"}]},"spec":{"type":"object","properties":{"field":{"type":"string","default":"foo"}}},"status":{"type":"object"}},"x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, + selectableFields: []apiextensionsv1.SelectableField{{JSONPath: "spec.field"}}, }, } for _, tt := range tests { @@ -560,8 +576,9 @@ func TestBuildOpenAPIV3(t *testing.T) { Group: "bar.k8s.io", Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ { - Name: "v1", - Schema: validation, + Name: "v1", + Schema: validation, + SelectableFields: tt.selectableFields, }, }, Names: apiextensionsv1.CustomResourceDefinitionNames{ diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/controller.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/controller.go index b2ffe6b3db8..54255de099b 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/controller.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/controller.go @@ -22,10 +22,13 @@ import ( "time" "github.com/google/uuid" + + apiextensionsfeatures "k8s.io/apiextensions-apiserver/pkg/features" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" "k8s.io/klog/v2" @@ -88,7 +91,10 @@ func createSpecCache(crd *apiextensionsv1.CustomResourceDefinition) *specCache { if !v.Served { continue } - s, err := builder.BuildOpenAPIV2(crd, v.Name, builder.Options{V2: true}) + s, err := builder.BuildOpenAPIV2(crd, v.Name, builder.Options{ + V2: true, + IncludeSelectableFields: utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceFieldSelectors), + }) // Defaults must be pruned here for CRDs to cleanly merge with the static // spec that already has defaults pruned if err != nil { diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapiv3/controller.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapiv3/controller.go index dfd65810679..c9cc4b302c3 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapiv3/controller.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapiv3/controller.go @@ -22,11 +22,13 @@ import ( "sync" "time" + apiextensionsfeatures "k8s.io/apiextensions-apiserver/pkg/features" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" "k8s.io/klog/v2" @@ -230,7 +232,10 @@ func (c *Controller) updateCRDSpec(crd *apiextensionsv1.CustomResourceDefinition } func (c *Controller) buildV3Spec(crd *apiextensionsv1.CustomResourceDefinition, name, versionName string) error { - v3, err := builder.BuildOpenAPIV3(crd, versionName, builder.Options{V2: false}) + v3, err := builder.BuildOpenAPIV3(crd, versionName, builder.Options{ + V2: false, + IncludeSelectableFields: utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceFieldSelectors), + }) if err != nil { return err diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/features/kube_features.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/features/kube_features.go index 7394b87895a..7773f3d1465 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/features/kube_features.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/features/kube_features.go @@ -34,6 +34,13 @@ const ( // Ignores errors raised on unchanged fields of Custom Resources // across UPDATE/PATCH requests. CRDValidationRatcheting featuregate.Feature = "CRDValidationRatcheting" + + // owner: @jpbetz + // alpha: v1.30 + // + // CustomResourceDefinitions may include SelectableFields to declare which fields + // may be used as field selectors. + CustomResourceFieldSelectors featuregate.Feature = "CustomResourceFieldSelectors" ) func init() { @@ -44,5 +51,6 @@ func init() { // To add a new feature, define a key for it above and add it here. The features will be // available throughout Kubernetes binaries. var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ - CRDValidationRatcheting: {Default: true, PreRelease: featuregate.Beta}, + CRDValidationRatcheting: {Default: true, PreRelease: featuregate.Beta}, + CustomResourceFieldSelectors: {Default: false, PreRelease: featuregate.Alpha}, } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/etcd_test.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/etcd_test.go index 0d3e2ea5b88..a0c364f1b3f 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/etcd_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/etcd_test.go @@ -106,6 +106,7 @@ func newStorage(t *testing.T) (customresource.CustomResourceStorage, *etcd3testi nil, status, scale, + nil, ), restOptions, []string{"all"}, diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/strategy.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/strategy.go index 3713627ed87..822a2f8f55b 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/strategy.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/strategy.go @@ -19,8 +19,12 @@ package customresource import ( "context" "fmt" + "strings" + + "sigs.k8s.io/structured-merge-diff/v4/fieldpath" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel" "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/model" @@ -41,11 +45,11 @@ import ( celconfig "k8s.io/apiserver/pkg/apis/cel" "k8s.io/apiserver/pkg/cel/common" "k8s.io/apiserver/pkg/features" + "k8s.io/apiserver/pkg/registry/generic" apiserverstorage "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/names" utilfeature "k8s.io/apiserver/pkg/util/feature" - - "sigs.k8s.io/structured-merge-diff/v4/fieldpath" + "k8s.io/client-go/util/jsonpath" ) // customResourceStrategy implements behavior for CustomResources for a single @@ -54,22 +58,29 @@ type customResourceStrategy struct { runtime.ObjectTyper names.NameGenerator - namespaceScoped bool - validator customResourceValidator - structuralSchema *structuralschema.Structural - celValidator *cel.Validator - status *apiextensions.CustomResourceSubresourceStatus - scale *apiextensions.CustomResourceSubresourceScale - kind schema.GroupVersionKind + namespaceScoped bool + validator customResourceValidator + structuralSchema *structuralschema.Structural + celValidator *cel.Validator + status *apiextensions.CustomResourceSubresourceStatus + scale *apiextensions.CustomResourceSubresourceScale + kind schema.GroupVersionKind + selectableFieldSet []selectableField } -func NewStrategy(typer runtime.ObjectTyper, namespaceScoped bool, kind schema.GroupVersionKind, schemaValidator, statusSchemaValidator validation.SchemaValidator, structuralSchema *structuralschema.Structural, status *apiextensions.CustomResourceSubresourceStatus, scale *apiextensions.CustomResourceSubresourceScale) customResourceStrategy { +type selectableField struct { + name string + fieldPath *jsonpath.JSONPath + err error +} + +func NewStrategy(typer runtime.ObjectTyper, namespaceScoped bool, kind schema.GroupVersionKind, schemaValidator, statusSchemaValidator validation.SchemaValidator, structuralSchema *structuralschema.Structural, status *apiextensions.CustomResourceSubresourceStatus, scale *apiextensions.CustomResourceSubresourceScale, selectableFields []v1.SelectableField) customResourceStrategy { var celValidator *cel.Validator if utilfeature.DefaultFeatureGate.Enabled(features.CustomResourceValidationExpressions) { celValidator = cel.NewValidator(structuralSchema, true, celconfig.PerCallLimit) // CEL programs are compiled and cached here } - return customResourceStrategy{ + strategy := customResourceStrategy{ ObjectTyper: typer, NameGenerator: names.SimpleNameGenerator, namespaceScoped: namespaceScoped, @@ -85,6 +96,34 @@ func NewStrategy(typer runtime.ObjectTyper, namespaceScoped bool, kind schema.Gr celValidator: celValidator, kind: kind, } + if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceFieldSelectors) { + strategy.selectableFieldSet = prepareSelectableFields(selectableFields) + } + return strategy +} + +func prepareSelectableFields(selectableFields []v1.SelectableField) []selectableField { + result := make([]selectableField, len(selectableFields)) + for i, sf := range selectableFields { + name := strings.TrimPrefix(sf.JSONPath, ".") + + parser := jsonpath.New("selectableField") + parser.AllowMissingKeys(true) + err := parser.Parse("{" + sf.JSONPath + "}") + if err == nil { + result[i] = selectableField{ + name: name, + fieldPath: parser, + } + } else { + result[i] = selectableField{ + name: name, + err: err, + } + } + } + + return result } func (a customResourceStrategy) NamespaceScoped() bool { @@ -293,7 +332,52 @@ func (a customResourceStrategy) GetAttrs(obj runtime.Object) (labels.Set, fields if err != nil { return nil, nil, err } - return labels.Set(accessor.GetLabels()), objectMetaFieldsSet(accessor, a.namespaceScoped), nil + sFields, err := a.selectableFields(obj, accessor) + if err != nil { + return nil, nil, err + } + return accessor.GetLabels(), sFields, nil +} + +// selectableFields returns a field set that can be used for filter selection. +// This includes metadata.name, metadata.namespace and all custom selectable fields. +func (a customResourceStrategy) selectableFields(obj runtime.Object, objectMeta metav1.Object) (fields.Set, error) { + objectMetaFields := objectMetaFieldsSet(objectMeta, a.namespaceScoped) + var selectableFieldsSet fields.Set + + if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceFieldSelectors) && len(a.selectableFieldSet) > 0 { + us, ok := obj.(runtime.Unstructured) + if !ok { + return nil, fmt.Errorf("unexpected error casting a custom resource to unstructured") + } + uc := us.UnstructuredContent() + + selectableFieldsSet = fields.Set{} + for _, sf := range a.selectableFieldSet { + if sf.err != nil { + return nil, fmt.Errorf("unexpected error parsing jsonPath: %w", sf.err) + } + results, err := sf.fieldPath.FindResults(uc) + if err != nil { + return nil, fmt.Errorf("unexpected error finding value with jsonPath: %w", err) + } + var value any + + if len(results) > 0 && len(results[0]) == 1 { + if len(results) > 1 || len(results[0]) > 1 { + return nil, fmt.Errorf("unexpectedly received more than one JSON path result") + } + value = results[0][0].Interface() + } + + if value != nil { + selectableFieldsSet[sf.name] = fmt.Sprint(value) + } else { + selectableFieldsSet[sf.name] = "" + } + } + } + return generic.MergeFieldsSets(objectMetaFields, selectableFieldsSet), nil } // objectMetaFieldsSet returns a fields that represent the ObjectMeta. diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/strategy_test.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/strategy_test.go index 8e8f178700b..3deafc31fd2 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/strategy_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/strategy_test.go @@ -22,7 +22,12 @@ import ( "testing" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apiextensionsfeatures "k8s.io/apiextensions-apiserver/pkg/features" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/fields" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" ) func generation1() map[string]interface{} { @@ -255,3 +260,64 @@ func TestStrategyPrepareForUpdate(t *testing.T) { } } } + +func TestSelectableFields(t *testing.T) { + tcs := []struct { + name string + selectableFields []v1.SelectableField + obj *unstructured.Unstructured + expectFields fields.Set + }{ + { + name: "valid path", + selectableFields: []v1.SelectableField{ + {JSONPath: ".spec.foo"}, + }, + obj: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "example", + "generation": int64(1), + "other": "new", + }, + "spec": map[string]interface{}{ + "foo": "x", + }, + }, + }, + expectFields: map[string]string{"spec.foo": "x", "metadata.name": "example"}, + }, + { + name: "missing value", + selectableFields: []v1.SelectableField{ + {JSONPath: ".spec.foo"}, + }, + obj: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "example", + "generation": int64(1), + "other": "new", + }, + "spec": map[string]interface{}{}, + }, + }, + expectFields: map[string]string{"spec.foo": "", "metadata.name": "example"}, + }, + } + + for _, tc := range tcs { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceFieldSelectors, true)() + t.Run(tc.name, func(t *testing.T) { + strategy := customResourceStrategy{selectableFieldSet: prepareSelectableFields(tc.selectableFields)} + + _, fields, err := strategy.GetAttrs(tc.obj) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(tc.expectFields, fields) { + t.Errorf("Expected fields '%+#v' but got '%+#v'", tc.expectFields, fields) + } + }) + } +} diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/strategy.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/strategy.go index 08ac3c90910..d26866b9917 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/strategy.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/strategy.go @@ -242,6 +242,9 @@ func dropDisabledFields(newCRD *apiextensions.CustomResourceDefinition, oldCRD * } } } + if !utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceFieldSelectors) && (oldCRD == nil || (oldCRD != nil && !specHasSelectableFields(&oldCRD.Spec))) { + dropSelectableFields(&newCRD.Spec) + } } // dropOptionalOldSelfField drops field optionalOldSelf from CRD schema @@ -284,3 +287,23 @@ func schemaHasOptionalOldSelf(s *apiextensions.JSONSchemaProps) bool { return false }) } + +func dropSelectableFields(spec *apiextensions.CustomResourceDefinitionSpec) { + spec.SelectableFields = nil + for i := range spec.Versions { + spec.Versions[i].SelectableFields = nil + } +} + +func specHasSelectableFields(spec *apiextensions.CustomResourceDefinitionSpec) bool { + if spec.SelectableFields != nil { + return true + } + for _, v := range spec.Versions { + if v.SelectableFields != nil { + return true + } + } + + return false +} diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/strategy_test.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/strategy_test.go index 2aa22f23857..6f2ba6ad07d 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/strategy_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/strategy_test.go @@ -21,6 +21,7 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation" @@ -198,11 +199,12 @@ func TestValidateAPIApproval(t *testing.T) { // TestDropDisabledFields tests if the drop functionality is working fine or not with feature gate switch func TestDropDisabledFields(t *testing.T) { testCases := []struct { - name string - enableRatcheting bool - crd *apiextensions.CustomResourceDefinition - oldCRD *apiextensions.CustomResourceDefinition - expectedCRD *apiextensions.CustomResourceDefinition + name string + enableRatcheting bool + enableSelectableFields bool + crd *apiextensions.CustomResourceDefinition + oldCRD *apiextensions.CustomResourceDefinition + expectedCRD *apiextensions.CustomResourceDefinition }{ { name: "Ratcheting, For creation, FG disabled, no OptionalOldSelf, no field drop", @@ -693,10 +695,612 @@ func TestDropDisabledFields(t *testing.T) { }, }, }, + // SelectableFields + { + name: "SelectableFields, For create, FG disabled, SelectableFields in update, dropped", + enableSelectableFields: false, + crd: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Validation: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{ + "field": { + Type: "string", + }, + }, + }, + }, + SelectableFields: []apiextensions.SelectableField{ + { + JSONPath: ".field", + }, + }, + }, + }, + expectedCRD: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Validation: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{ + "field": { + Type: "string", + }, + }, + }, + }, + }, + }, + }, + { + name: "SelectableFields, For create, FG enabled, no SelectableFields in update, no drop", + enableSelectableFields: true, + crd: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Validation: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{ + "field": { + Type: "string", + }, + }, + }, + }, + }, + }, + expectedCRD: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Validation: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{ + "field": { + Type: "string", + }, + }, + }, + }, + }, + }, + }, + { + name: "SelectableFields, For create, FG enabled, SelectableFields in update, no drop", + enableSelectableFields: true, + crd: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Validation: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{ + "field": { + Type: "string", + }, + }, + }, + }, + SelectableFields: []apiextensions.SelectableField{ + { + JSONPath: ".field", + }, + }, + }, + }, + expectedCRD: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Validation: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{ + "field": { + Type: "string", + }, + }, + }, + }, + SelectableFields: []apiextensions.SelectableField{ + { + JSONPath: ".field", + }, + }, + }, + }, + }, + { + name: "SelectableFields, For update, FG disabled, oldCRD has SelectableFields, SelectableFields in update, no drop", + enableSelectableFields: false, + crd: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Validation: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{ + "field1": { + Type: "string", + }, + "field2": { + Type: "string", + }, + }, + }, + }, + SelectableFields: []apiextensions.SelectableField{ + { + JSONPath: ".field1", + }, + { + JSONPath: ".field2", + }, + }, + }, + }, + oldCRD: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Validation: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{ + "field1": { + Type: "string", + }, + }, + }, + }, + SelectableFields: []apiextensions.SelectableField{ + { + JSONPath: ".field1", + }, + }, + }, + }, + expectedCRD: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Validation: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{ + "field1": { + Type: "string", + }, + "field2": { + Type: "string", + }, + }, + }, + }, + SelectableFields: []apiextensions.SelectableField{ + { + JSONPath: ".field1", + }, + { + JSONPath: ".field2", + }, + }, + }, + }, + }, + { + name: "SelectableFields, For update, FG disabled, oldCRD does not have SelectableFields, no SelectableFields in update, no drop", + enableSelectableFields: false, + crd: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Validation: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{ + "field1": { + Type: "string", + }, + "field2": { + Type: "string", + }, + }, + }, + }, + }, + }, + oldCRD: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Validation: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{ + "field1": { + Type: "string", + }, + }, + }, + }, + }, + }, + expectedCRD: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Validation: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{ + "field1": { + Type: "string", + }, + "field2": { + Type: "string", + }, + }, + }, + }, + }, + }, + }, + { + name: "SelectableFields, For update, FG disabled, oldCRD does not have SelectableFields, SelectableFields in update, dropped", + enableSelectableFields: false, + crd: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Validation: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{ + "field1": { + Type: "string", + }, + "field2": { + Type: "string", + }, + }, + }, + }, + SelectableFields: []apiextensions.SelectableField{ + { + JSONPath: ".field1", + }, + { + JSONPath: ".field2", + }, + }, + }, + }, + oldCRD: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Validation: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{ + "field1": { + Type: "string", + }, + }, + }, + }, + }, + }, + expectedCRD: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Validation: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{ + "field1": { + Type: "string", + }, + "field2": { + Type: "string", + }, + }, + }, + }, + }, + }, + }, + { + name: "SelectableFields, For update, FG enabled, oldCRD has SelectableFields, SelectableFields in update, no drop", + enableSelectableFields: true, + crd: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Validation: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{ + "field1": { + Type: "string", + }, + "field2": { + Type: "string", + }, + }, + }, + }, + SelectableFields: []apiextensions.SelectableField{ + { + JSONPath: ".field1", + }, + { + JSONPath: ".field2", + }, + }, + }, + }, + oldCRD: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Validation: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{ + "field1": { + Type: "string", + }, + }, + }, + }, + SelectableFields: []apiextensions.SelectableField{ + { + JSONPath: ".field1", + }, + }, + }, + }, + expectedCRD: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Validation: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{ + "field1": { + Type: "string", + }, + "field2": { + Type: "string", + }, + }, + }, + }, + SelectableFields: []apiextensions.SelectableField{ + { + JSONPath: ".field1", + }, + { + JSONPath: ".field2", + }, + }, + }, + }, + }, + { + name: "SelectableFields, For update, FG enabled, oldCRD does not have SelectableFields, SelectableFields in update, no drop", + enableSelectableFields: true, + crd: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Validation: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{ + "field1": { + Type: "string", + }, + "field2": { + Type: "string", + }, + }, + }, + }, + SelectableFields: []apiextensions.SelectableField{ + { + JSONPath: ".field1", + }, + { + JSONPath: ".field2", + }, + }, + }, + }, + oldCRD: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Validation: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{ + "field1": { + Type: "string", + }, + }, + }, + }, + SelectableFields: []apiextensions.SelectableField{ + { + JSONPath: ".field1", + }, + }, + }, + }, + expectedCRD: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Validation: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{ + "field1": { + Type: "string", + }, + "field2": { + Type: "string", + }, + }, + }, + }, + SelectableFields: []apiextensions.SelectableField{ + { + JSONPath: ".field1", + }, + { + JSONPath: ".field2", + }, + }, + }, + }, + }, + { + name: "pre-version SelectableFields, For update, FG disabled, oldCRD does not have SelectableFields, SelectableFields in update, dropped", + enableSelectableFields: false, + crd: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Versions: []apiextensions.CustomResourceDefinitionVersion{ + { + Name: "v1", + Schema: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{ + "field1": { + Type: "string", + }, + "field2": { + Type: "string", + }, + }, + }, + }, + SelectableFields: []apiextensions.SelectableField{ + { + JSONPath: ".field1", + }, + { + JSONPath: ".field2", + }, + }, + }, + { + Name: "v2", + Schema: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{ + "field3": { + Type: "string", + }, + "field4": { + Type: "string", + }, + }, + }, + }, + SelectableFields: []apiextensions.SelectableField{ + { + JSONPath: ".field3", + }, + { + JSONPath: ".field4", + }, + }, + }, + }, + }, + }, + oldCRD: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Versions: []apiextensions.CustomResourceDefinitionVersion{ + { + Name: "v1", + Schema: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{ + "field1": { + Type: "string", + }, + "field2": { + Type: "string", + }, + }, + }, + }, + }, + { + Name: "v2", + Schema: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{ + "field3": { + Type: "string", + }, + "field4": { + Type: "string", + }, + }, + }, + }, + }, + }, + }, + }, + expectedCRD: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Versions: []apiextensions.CustomResourceDefinitionVersion{ + { + Name: "v1", + Schema: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{ + "field1": { + Type: "string", + }, + "field2": { + Type: "string", + }, + }, + }, + }, + }, + { + Name: "v2", + Schema: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensions.JSONSchemaProps{ + "field3": { + Type: "string", + }, + "field4": { + Type: "string", + }, + }, + }, + }, + }, + }, + }, + }, + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CRDValidationRatcheting, tc.enableRatcheting)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceFieldSelectors, tc.enableSelectableFields)() old := tc.oldCRD.DeepCopy() dropDisabledFields(tc.crd, tc.oldCRD) diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go b/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go index ffd4a7dcbfb..5315b470ea4 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go @@ -26,6 +26,8 @@ import ( "unicode" restful "github.com/emicklei/go-restful/v3" + "sigs.k8s.io/structured-merge-diff/v4/fieldpath" + apidiscoveryv2beta1 "k8s.io/api/apidiscovery/v2beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/conversion" @@ -46,12 +48,12 @@ import ( "k8s.io/apiserver/pkg/storageversion" utilfeature "k8s.io/apiserver/pkg/util/feature" versioninfo "k8s.io/component-base/version" - "sigs.k8s.io/structured-merge-diff/v4/fieldpath" ) const ( - ROUTE_META_GVK = "x-kubernetes-group-version-kind" - ROUTE_META_ACTION = "x-kubernetes-action" + RouteMetaGVK = "x-kubernetes-group-version-kind" + RouteMetaSelectableFields = "x-kubernetes-selectable-fields" + RouteMetaAction = "x-kubernetes-action" ) type APIInstaller struct { @@ -1059,12 +1061,12 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag return nil, nil, fmt.Errorf("unrecognized action verb: %s", action.Verb) } for _, route := range routes { - route.Metadata(ROUTE_META_GVK, metav1.GroupVersionKind{ + route.Metadata(RouteMetaGVK, metav1.GroupVersionKind{ Group: reqScope.Kind.Group, Version: reqScope.Kind.Version, Kind: reqScope.Kind.Kind, }) - route.Metadata(ROUTE_META_ACTION, strings.ToLower(action.Verb)) + route.Metadata(RouteMetaAction, strings.ToLower(action.Verb)) ws.Route(route) } // Note: update GetAuthorizerAttributes() when adding a custom handler.