From 439d2f7b4028638b3d8d9261bb046c3ba8d9bfcb Mon Sep 17 00:00:00 2001 From: Ben Luddy Date: Fri, 1 Nov 2024 16:05:32 -0400 Subject: [PATCH] Wire serving codecs to CBOR feature gate. Integration testing has to this point relied on patching serving codecs for built-in APIs. The test-only patching is removed and replaced by feature gated checks at runtime. --- .../core/rest/storage_core_generic.go | 7 +++ .../pkg/apiserver/customresource_handler.go | 18 +------ .../pkg/runtime/serializer/cbor/cbor.go | 15 ++++++ .../pkg/endpoints/handlers/delete.go | 13 ++--- .../pkg/endpoints/handlers/response.go | 6 +-- .../apiserver/pkg/server/genericapiserver.go | 5 ++ .../apiserver/pkg/util/apihelpers/helpers.go | 18 +++++++ staging/src/k8s.io/client-go/rest/config.go | 14 +----- .../admissionwebhook/admission_test.go | 5 +- test/integration/apiserver/apiserver_test.go | 5 -- .../integration/apiserver/apply/apply_test.go | 5 +- .../apiserver/apply/status_test.go | 5 +- test/integration/apiserver/cbor_test.go | 5 +- test/integration/client/client_test.go | 7 ++- .../integration/client/dynamic_client_test.go | 7 ++- .../transformation/kms_transformation_test.go | 5 -- .../etcd/etcd_storage_path_test.go | 5 -- test/integration/etcd/server.go | 5 +- test/integration/framework/cbor.go | 48 ------------------- 19 files changed, 86 insertions(+), 112 deletions(-) diff --git a/pkg/registry/core/rest/storage_core_generic.go b/pkg/registry/core/rest/storage_core_generic.go index 193b5b98f47..a4fe9f007fc 100644 --- a/pkg/registry/core/rest/storage_core_generic.go +++ b/pkg/registry/core/rest/storage_core_generic.go @@ -33,6 +33,10 @@ import ( "k8s.io/client-go/informers" restclient "k8s.io/client-go/rest" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/runtime/serializer/cbor" + "k8s.io/apiserver/pkg/features" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/api/legacyscheme" api "k8s.io/kubernetes/pkg/apis/core" configmapstore "k8s.io/kubernetes/pkg/registry/core/configmap/storage" @@ -68,6 +72,9 @@ func (c *GenericConfig) NewRESTStorage(apiResourceConfigSource serverstorage.API ParameterCodec: legacyscheme.ParameterCodec, NegotiatedSerializer: legacyscheme.Codecs, } + if utilfeature.DefaultFeatureGate.Enabled(features.CBORServingAndStorage) { + apiGroupInfo.NegotiatedSerializer = serializer.NewCodecFactory(legacyscheme.Scheme, serializer.WithSerializer(cbor.NewSerializerInfo)) + } eventStorage, err := eventstore.NewREST(restOptionsGetter, uint64(c.EventTTL.Seconds())) if err != nil { diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go index 955ba1b5a37..55d31fc8eda 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go @@ -606,20 +606,6 @@ func (r *crdHandler) GetCustomResourceListerCollectionDeleter(crd *apiextensions return info.storages[info.storageVersion].CustomResource, nil } -func newCBORSerializerInfo(creater runtime.ObjectCreater, typer runtime.ObjectTyper) runtime.SerializerInfo { - return runtime.SerializerInfo{ - MediaType: "application/cbor", - MediaTypeType: "application", - MediaTypeSubType: "cbor", - Serializer: cbor.NewSerializer(creater, typer), - StrictSerializer: cbor.NewSerializer(creater, typer, cbor.Strict(true)), - StreamSerializer: &runtime.StreamSerializerInfo{ - Framer: cbor.NewFramer(), - Serializer: cbor.NewSerializer(creater, typer, cbor.Transcode(false)), - }, - } -} - // getOrCreateServingInfoFor gets the CRD serving info for the given CRD UID if the key exists in the storage map. // Otherwise the function fetches the up-to-date CRD using the given CRD name and creates CRD serving info. func (r *crdHandler) getOrCreateServingInfoFor(uid types.UID, name string) (*crdInfo, error) { @@ -914,7 +900,7 @@ func (r *crdHandler) getOrCreateServingInfoFor(uid types.UID, name string) (*crd } if utilfeature.DefaultFeatureGate.Enabled(features.CBORServingAndStorage) { - negotiatedSerializer.supportedMediaTypes = append(negotiatedSerializer.supportedMediaTypes, newCBORSerializerInfo(creator, typer)) + negotiatedSerializer.supportedMediaTypes = append(negotiatedSerializer.supportedMediaTypes, cbor.NewSerializerInfo(creator, typer)) } var standardSerializers []runtime.SerializerInfo @@ -982,7 +968,7 @@ func (r *crdHandler) getOrCreateServingInfoFor(uid types.UID, name string) (*crd scaleScope.Subresource = "scale" var opts []serializer.CodecFactoryOptionsMutator if utilfeature.DefaultFeatureGate.Enabled(features.CBORServingAndStorage) { - opts = append(opts, serializer.WithSerializer(newCBORSerializerInfo)) + opts = append(opts, serializer.WithSerializer(cbor.NewSerializerInfo)) } scaleScope.Serializer = serializer.NewCodecFactory(scaleConverter.Scheme(), opts...) scaleScope.Kind = autoscalingv1.SchemeGroupVersion.WithKind("Scale") diff --git a/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/cbor/cbor.go b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/cbor/cbor.go index 20fa7eb04cf..4d069a903ae 100644 --- a/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/cbor/cbor.go +++ b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/cbor/cbor.go @@ -372,3 +372,18 @@ var selfDescribedCBOR = []byte{0xd9, 0xd9, 0xf7} func (s *serializer) RecognizesData(data []byte) (ok, unknown bool, err error) { return bytes.HasPrefix(data, selfDescribedCBOR), false, nil } + +// NewSerializerInfo returns a default SerializerInfo for CBOR using the given creater and typer. +func NewSerializerInfo(creater runtime.ObjectCreater, typer runtime.ObjectTyper) runtime.SerializerInfo { + return runtime.SerializerInfo{ + MediaType: "application/cbor", + MediaTypeType: "application", + MediaTypeSubType: "cbor", + Serializer: NewSerializer(creater, typer), + StrictSerializer: NewSerializer(creater, typer, Strict(true)), + StreamSerializer: &runtime.StreamSerializerInfo{ + Framer: NewFramer(), + Serializer: NewSerializer(creater, typer, Transcode(false)), + }, + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/delete.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/delete.go index edd6b4da021..781a0e0d1f9 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/delete.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/delete.go @@ -40,6 +40,7 @@ import ( "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/features" "k8s.io/apiserver/pkg/registry/rest" + "k8s.io/apiserver/pkg/util/apihelpers" "k8s.io/apiserver/pkg/util/dryrun" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/component-base/tracing" @@ -85,7 +86,7 @@ func DeleteResource(r rest.GracefulDeleter, allowsOptions bool, scope *RequestSc } span.AddEvent("limitedReadBody succeeded", attribute.Int("len", len(body))) if len(body) > 0 { - s, err := negotiation.NegotiateInputSerializer(req, false, metainternalversionscheme.Codecs) + s, err := negotiation.NegotiateInputSerializer(req, false, apihelpers.GetMetaInternalVersionCodecs()) if err != nil { scope.err(err, w, req) return @@ -93,7 +94,7 @@ func DeleteResource(r rest.GracefulDeleter, allowsOptions bool, scope *RequestSc // For backwards compatibility, we need to allow existing clients to submit per group DeleteOptions // It is also allowed to pass a body with meta.k8s.io/v1.DeleteOptions defaultGVK := scope.MetaGroupVersion.WithKind("DeleteOptions") - obj, gvk, err := metainternalversionscheme.Codecs.DecoderToVersion(s.Serializer, defaultGVK.GroupVersion()).Decode(body, &defaultGVK, options) + obj, gvk, err := apihelpers.GetMetaInternalVersionCodecs().DecoderToVersion(s.Serializer, defaultGVK.GroupVersion()).Decode(body, &defaultGVK, options) if err != nil { scope.err(err, w, req) return @@ -105,7 +106,7 @@ func DeleteResource(r rest.GracefulDeleter, allowsOptions bool, scope *RequestSc span.AddEvent("Decoded delete options") objGV := gvk.GroupVersion() - audit.LogRequestObject(req.Context(), obj, objGV, scope.Resource, scope.Subresource, metainternalversionscheme.Codecs) + audit.LogRequestObject(req.Context(), obj, objGV, scope.Resource, scope.Subresource, apihelpers.GetMetaInternalVersionCodecs()) span.AddEvent("Recorded the audit event") } else { if err := metainternalversionscheme.ParameterCodec.DecodeParameters(req.URL.Query(), scope.MetaGroupVersion, options); err != nil { @@ -231,7 +232,7 @@ func DeleteCollection(r rest.CollectionDeleter, checkBody bool, scope *RequestSc } span.AddEvent("limitedReadBody succeeded", attribute.Int("len", len(body))) if len(body) > 0 { - s, err := negotiation.NegotiateInputSerializer(req, false, metainternalversionscheme.Codecs) + s, err := negotiation.NegotiateInputSerializer(req, false, apihelpers.GetMetaInternalVersionCodecs()) if err != nil { scope.err(err, w, req) return @@ -239,7 +240,7 @@ func DeleteCollection(r rest.CollectionDeleter, checkBody bool, scope *RequestSc // For backwards compatibility, we need to allow existing clients to submit per group DeleteOptions // It is also allowed to pass a body with meta.k8s.io/v1.DeleteOptions defaultGVK := scope.MetaGroupVersion.WithKind("DeleteOptions") - obj, gvk, err := metainternalversionscheme.Codecs.DecoderToVersion(s.Serializer, defaultGVK.GroupVersion()).Decode(body, &defaultGVK, options) + obj, gvk, err := apihelpers.GetMetaInternalVersionCodecs().DecoderToVersion(s.Serializer, defaultGVK.GroupVersion()).Decode(body, &defaultGVK, options) if err != nil { scope.err(err, w, req) return @@ -250,7 +251,7 @@ func DeleteCollection(r rest.CollectionDeleter, checkBody bool, scope *RequestSc } objGV := gvk.GroupVersion() - audit.LogRequestObject(req.Context(), obj, objGV, scope.Resource, scope.Subresource, metainternalversionscheme.Codecs) + audit.LogRequestObject(req.Context(), obj, objGV, scope.Resource, scope.Subresource, apihelpers.GetMetaInternalVersionCodecs()) } else { if err := metainternalversionscheme.ParameterCodec.DecodeParameters(req.URL.Query(), scope.MetaGroupVersion, options); err != nil { err = errors.NewBadRequest(err.Error()) diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/response.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/response.go index f3550ae8ab5..3f7ad6121c5 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/response.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/response.go @@ -40,7 +40,7 @@ import ( "k8s.io/apiserver/pkg/endpoints/metrics" endpointsrequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/storage" - + "k8s.io/apiserver/pkg/util/apihelpers" "k8s.io/klog/v2" ) @@ -281,7 +281,7 @@ func doTransformObject(ctx context.Context, obj runtime.Object, opts interface{} return asTable(ctx, obj, options, scope, target.GroupVersion()) default: - accepted, _ := negotiation.MediaTypesForSerializer(metainternalversionscheme.Codecs) + accepted, _ := negotiation.MediaTypesForSerializer(apihelpers.GetMetaInternalVersionCodecs()) err := negotiation.NewNotAcceptableError(accepted) return nil, err } @@ -315,7 +315,7 @@ func targetEncodingForTransform(scope *RequestScope, mediaType negotiation.Media case target == nil: case (target.Kind == "PartialObjectMetadata" || target.Kind == "PartialObjectMetadataList" || target.Kind == "Table") && (target.GroupVersion() == metav1beta1.SchemeGroupVersion || target.GroupVersion() == metav1.SchemeGroupVersion): - return *target, metainternalversionscheme.Codecs, true + return *target, apihelpers.GetMetaInternalVersionCodecs(), true } return scope.Kind, scope.Serializer, false } diff --git a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go index 7daa3a7aef5..e810a460879 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go +++ b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go @@ -35,6 +35,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/runtime/serializer/cbor" "k8s.io/apimachinery/pkg/util/managedfields" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" @@ -51,6 +52,7 @@ import ( "k8s.io/apiserver/pkg/server/healthz" "k8s.io/apiserver/pkg/server/routes" "k8s.io/apiserver/pkg/storageversion" + utilfeature "k8s.io/apiserver/pkg/util/feature" restclient "k8s.io/client-go/rest" "k8s.io/component-base/featuregate" utilversion "k8s.io/component-base/version" @@ -989,6 +991,9 @@ func (s *GenericAPIServer) newAPIGroupVersion(apiGroupInfo *APIGroupInfo, groupV // NewDefaultAPIGroupInfo returns an APIGroupInfo stubbed with "normal" values // exposed for easier composition from other packages func NewDefaultAPIGroupInfo(group string, scheme *runtime.Scheme, parameterCodec runtime.ParameterCodec, codecs serializer.CodecFactory) APIGroupInfo { + if utilfeature.DefaultFeatureGate.Enabled(features.CBORServingAndStorage) { + codecs = serializer.NewCodecFactory(scheme, serializer.WithSerializer(cbor.NewSerializerInfo)) + } return APIGroupInfo{ PrioritizedVersions: scheme.PrioritizedVersionsForGroup(group), VersionedResourcesStorageMap: map[string]map[string]rest.Storage{}, diff --git a/staging/src/k8s.io/apiserver/pkg/util/apihelpers/helpers.go b/staging/src/k8s.io/apiserver/pkg/util/apihelpers/helpers.go index 9a6b2a28ef1..e5365fc2226 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/apihelpers/helpers.go +++ b/staging/src/k8s.io/apiserver/pkg/util/apihelpers/helpers.go @@ -20,6 +20,12 @@ import ( "sort" flowcontrol "k8s.io/api/flowcontrol/v1" + metainternalversionscheme "k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/runtime/serializer/cbor" + "k8s.io/apiserver/pkg/features" + utilfeature "k8s.io/apiserver/pkg/util/feature" ) // SetFlowSchemaCondition sets conditions. @@ -98,3 +104,15 @@ func (s FlowSchemaSequence) Less(i, j int) bool { func (s FlowSchemaSequence) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +var metaInternalVersionCodecsWithCBOR = serializer.NewCodecFactory(metainternalversionscheme.Scheme, serializer.WithSerializer(cbor.NewSerializerInfo)) + +// GetMetaInternalVersionCodecs returns a negotiated serializer that recognizes the types from +// k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme.Scheme. It will or will include a CBOR +// serializer if CBOR is enabled. +func GetMetaInternalVersionCodecs() runtime.NegotiatedSerializer { + if utilfeature.DefaultFeatureGate.Enabled(features.CBORServingAndStorage) { + return metaInternalVersionCodecsWithCBOR + } + return metainternalversionscheme.Codecs +} diff --git a/staging/src/k8s.io/client-go/rest/config.go b/staging/src/k8s.io/client-go/rest/config.go index aebd990af4a..f2e813d075e 100644 --- a/staging/src/k8s.io/client-go/rest/config.go +++ b/staging/src/k8s.io/client-go/rest/config.go @@ -689,17 +689,5 @@ func CodecFactoryForGeneratedClient(scheme *runtime.Scheme, codecs serializer.Co return codecs } - return serializer.NewCodecFactory(scheme, serializer.WithSerializer(func(creater runtime.ObjectCreater, typer runtime.ObjectTyper) runtime.SerializerInfo { - return runtime.SerializerInfo{ - MediaType: "application/cbor", - MediaTypeType: "application", - MediaTypeSubType: "cbor", - Serializer: cbor.NewSerializer(creater, typer), - StrictSerializer: cbor.NewSerializer(creater, typer, cbor.Strict(true)), - StreamSerializer: &runtime.StreamSerializerInfo{ - Framer: cbor.NewFramer(), - Serializer: cbor.NewSerializer(creater, typer, cbor.Transcode(false)), - }, - } - })) + return serializer.NewCodecFactory(scheme, serializer.WithSerializer(cbor.NewSerializerInfo)) } diff --git a/test/integration/apiserver/admissionwebhook/admission_test.go b/test/integration/apiserver/admissionwebhook/admission_test.go index f3c05f93f32..6080169193c 100644 --- a/test/integration/apiserver/admissionwebhook/admission_test.go +++ b/test/integration/apiserver/admissionwebhook/admission_test.go @@ -52,12 +52,15 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" + "k8s.io/apiserver/pkg/features" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/dynamic" clientfeatures "k8s.io/client-go/features" clientfeaturestesting "k8s.io/client-go/features/testing" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/util/retry" + featuregatetesting "k8s.io/component-base/featuregate/testing" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" apisv1beta1 "k8s.io/kubernetes/pkg/apis/admissionregistration/v1beta1" "k8s.io/kubernetes/test/integration/etcd" @@ -459,7 +462,7 @@ func TestWebhookAdmissionWithoutWatchCache(t *testing.T) { } func TestWebhookAdmissionWithCBOR(t *testing.T) { - framework.EnableCBORServingAndStorageForTest(t) + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CBORServingAndStorage, true) clientfeaturestesting.SetFeatureDuringTest(t, clientfeatures.ClientsAllowCBOR, true) clientfeaturestesting.SetFeatureDuringTest(t, clientfeatures.ClientsPreferCBOR, true) testWebhookAdmission(t, false, func(t testing.TB, config *rest.Config) { diff --git a/test/integration/apiserver/apiserver_test.go b/test/integration/apiserver/apiserver_test.go index 7809f750293..58e4df84c26 100644 --- a/test/integration/apiserver/apiserver_test.go +++ b/test/integration/apiserver/apiserver_test.go @@ -63,8 +63,6 @@ import ( utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/discovery/cached/memory" "k8s.io/client-go/dynamic" - clientfeatures "k8s.io/client-go/features" - clientfeaturestesting "k8s.io/client-go/features/testing" clientset "k8s.io/client-go/kubernetes" appsv1 "k8s.io/client-go/kubernetes/typed/apps/v1" "k8s.io/client-go/metadata" @@ -3367,9 +3365,6 @@ func TestDefaultStorageEncoding(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, "AllAlpha", true) featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, "AllBeta", true) - // TODO: Remove this override when the codecs used for serving all built-in APIs are wired to the apiserver feature gate. - clientfeaturestesting.SetFeatureDuringTest(t, clientfeatures.ClientsPreferCBOR, false) - protobufRecognizer := protobuf.NewSerializer(runtime.NewScheme(), runtime.NewScheme()) var recognizersByGroup map[string]recognizer.RecognizingDecoder { diff --git a/test/integration/apiserver/apply/apply_test.go b/test/integration/apiserver/apply/apply_test.go index ac2750e2be8..a4f2db15d7b 100644 --- a/test/integration/apiserver/apply/apply_test.go +++ b/test/integration/apiserver/apply/apply_test.go @@ -43,6 +43,8 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/wait" yamlutil "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/apiserver/pkg/features" + utilfeature "k8s.io/apiserver/pkg/util/feature" appsv1ac "k8s.io/client-go/applyconfigurations/apps/v1" corev1ac "k8s.io/client-go/applyconfigurations/core/v1" metav1ac "k8s.io/client-go/applyconfigurations/meta/v1" @@ -51,6 +53,7 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" restclient "k8s.io/client-go/rest" + featuregatetesting "k8s.io/component-base/featuregate/testing" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" "k8s.io/kubernetes/test/integration/authutil" "k8s.io/kubernetes/test/integration/framework" @@ -1058,7 +1061,7 @@ func TestPatchVeryLargeObject(t *testing.T) { // TestPatchVeryLargeObjectCBORApply mirrors TestPatchVeryLargeObject using the +cbor structured // syntax suffix for application/apply-patch and with CBOR enabled. func TestPatchVeryLargeObjectCBORApply(t *testing.T) { - framework.EnableCBORServingAndStorageForTest(t) + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CBORServingAndStorage, true) clientfeaturestesting.SetFeatureDuringTest(t, clientfeatures.ClientsAllowCBOR, true) client, closeFn := setup(t) diff --git a/test/integration/apiserver/apply/status_test.go b/test/integration/apiserver/apply/status_test.go index f2528a6fdcd..aae28a15685 100644 --- a/test/integration/apiserver/apply/status_test.go +++ b/test/integration/apiserver/apply/status_test.go @@ -28,11 +28,14 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/json" + "k8s.io/apiserver/pkg/features" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/dynamic" clientfeatures "k8s.io/client-go/features" clientfeaturestesting "k8s.io/client-go/features/testing" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + featuregatetesting "k8s.io/component-base/featuregate/testing" apiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" "k8s.io/kubernetes/test/integration/etcd" "k8s.io/kubernetes/test/integration/framework" @@ -101,7 +104,7 @@ func TestApplyStatus(t *testing.T) { // TestApplyStatus makes sure that applying the status works for all known types. func TestApplyStatusWithCBOR(t *testing.T) { - framework.EnableCBORServingAndStorageForTest(t) + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CBORServingAndStorage, true) clientfeaturestesting.SetFeatureDuringTest(t, clientfeatures.ClientsAllowCBOR, true) clientfeaturestesting.SetFeatureDuringTest(t, clientfeatures.ClientsPreferCBOR, true) testApplyStatus(t, func(t testing.TB, config *rest.Config) { diff --git a/test/integration/apiserver/cbor_test.go b/test/integration/apiserver/cbor_test.go index fbd16b1d468..815c91ddf30 100644 --- a/test/integration/apiserver/cbor_test.go +++ b/test/integration/apiserver/cbor_test.go @@ -36,11 +36,14 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer/cbor/direct" "k8s.io/apimachinery/pkg/runtime/serializer/streaming" "k8s.io/apimachinery/pkg/types" + "k8s.io/apiserver/pkg/features" + utilfeature "k8s.io/apiserver/pkg/util/feature" clientfeatures "k8s.io/client-go/features" clientfeaturestesting "k8s.io/client-go/features/testing" "k8s.io/client-go/kubernetes/scheme" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/rest" + featuregatetesting "k8s.io/component-base/featuregate/testing" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" "k8s.io/kubernetes/test/integration/framework" "k8s.io/utils/ptr" @@ -58,7 +61,7 @@ func TestNondeterministicResponseEncoding(t *testing.T) { // about 2^-NTrials, so NTrials needs to be big enough to make sure that doesn't happen. const NTrials = 40 - framework.EnableCBORServingAndStorageForTest(t) + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CBORServingAndStorage, true) clientfeaturestesting.SetFeatureDuringTest(t, clientfeatures.ClientsAllowCBOR, true) clientfeaturestesting.SetFeatureDuringTest(t, clientfeatures.ClientsPreferCBOR, true) diff --git a/test/integration/client/client_test.go b/test/integration/client/client_test.go index cbba4e0bbfa..54cad7d10b3 100644 --- a/test/integration/client/client_test.go +++ b/test/integration/client/client_test.go @@ -46,6 +46,8 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" + "k8s.io/apiserver/pkg/features" + utilfeature "k8s.io/apiserver/pkg/util/feature" appsv1ac "k8s.io/client-go/applyconfigurations/apps/v1" autoscalingv1ac "k8s.io/client-go/applyconfigurations/autoscaling/v1" corev1ac "k8s.io/client-go/applyconfigurations/core/v1" @@ -59,6 +61,7 @@ import ( corev1client "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/rest" "k8s.io/client-go/util/retry" + featuregatetesting "k8s.io/component-base/featuregate/testing" utilversion "k8s.io/component-base/version" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" "k8s.io/kubernetes/pkg/api/legacyscheme" @@ -1723,7 +1726,7 @@ func TestClientCBOREnablement(t *testing.T) { // Batch test cases with their server configuration instead of starting and stopping // a new apiserver for each test case. if served { - framework.EnableCBORServingAndStorageForTest(t) + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CBORServingAndStorage, true) } server := kubeapiservertesting.StartTestServerOrDie(t, nil, framework.DefaultTestServerFlags(), framework.SharedEtcd()) @@ -1835,7 +1838,7 @@ func TestClientCBOREnablement(t *testing.T) { } func TestCBORWithTypedClient(t *testing.T) { - framework.EnableCBORServingAndStorageForTest(t) + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CBORServingAndStorage, true) clientfeaturestesting.SetFeatureDuringTest(t, clientfeatures.ClientsAllowCBOR, true) clientfeaturestesting.SetFeatureDuringTest(t, clientfeatures.ClientsPreferCBOR, true) diff --git a/test/integration/client/dynamic_client_test.go b/test/integration/client/dynamic_client_test.go index 6f7866233ac..e7bbd28090e 100644 --- a/test/integration/client/dynamic_client_test.go +++ b/test/integration/client/dynamic_client_test.go @@ -38,6 +38,8 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" + "k8s.io/apiserver/pkg/features" + utilfeature "k8s.io/apiserver/pkg/util/feature" metav1ac "k8s.io/client-go/applyconfigurations/meta/v1" "k8s.io/client-go/discovery" "k8s.io/client-go/dynamic" @@ -46,6 +48,7 @@ import ( clientset "k8s.io/client-go/kubernetes" clientscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + featuregatetesting "k8s.io/component-base/featuregate/testing" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" "k8s.io/kubernetes/test/integration/framework" ) @@ -146,7 +149,7 @@ func TestDynamicClientWatch(t *testing.T) { } func TestDynamicClientWatchWithCBOR(t *testing.T) { - framework.EnableCBORServingAndStorageForTest(t) + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CBORServingAndStorage, true) clientfeaturestesting.SetFeatureDuringTest(t, clientfeatures.ClientsAllowCBOR, true) clientfeaturestesting.SetFeatureDuringTest(t, clientfeatures.ClientsPreferCBOR, true) @@ -570,7 +573,7 @@ func TestDynamicClientCBOREnablement(t *testing.T) { for _, serving := range []bool{true, false} { t.Run(fmt.Sprintf("serving=%t", serving), func(t *testing.T) { if serving { - framework.EnableCBORServingAndStorageForTest(t) + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CBORServingAndStorage, true) } server := kubeapiservertesting.StartTestServerOrDie(t, nil, framework.DefaultTestServerFlags(), framework.SharedEtcd()) diff --git a/test/integration/controlplane/transformation/kms_transformation_test.go b/test/integration/controlplane/transformation/kms_transformation_test.go index 3ecc322ab26..01858351ea3 100644 --- a/test/integration/controlplane/transformation/kms_transformation_test.go +++ b/test/integration/controlplane/transformation/kms_transformation_test.go @@ -53,8 +53,6 @@ import ( mock "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing/v1beta1" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/dynamic" - clientfeatures "k8s.io/client-go/features" - clientfeaturestesting "k8s.io/client-go/features/testing" "k8s.io/client-go/rest" featuregatetesting "k8s.io/component-base/featuregate/testing" kmsapi "k8s.io/kms/apis/v1beta1" @@ -631,9 +629,6 @@ resources: // Need to enable this explicitly as the feature is deprecated featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true) - // TODO: Remove this override when the codecs used for serving all built-in APIs are wired to the apiserver feature gate. - clientfeaturestesting.SetFeatureDuringTest(t, clientfeatures.ClientsPreferCBOR, false) - test, err := newTransformTest(t, encryptionConfig, false, "", nil) if err != nil { t.Fatalf("failed to start KUBE API Server with encryptionConfig") diff --git a/test/integration/etcd/etcd_storage_path_test.go b/test/integration/etcd/etcd_storage_path_test.go index 5d13544dd1b..bedc000856b 100644 --- a/test/integration/etcd/etcd_storage_path_test.go +++ b/test/integration/etcd/etcd_storage_path_test.go @@ -41,8 +41,6 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/dynamic" - clientfeatures "k8s.io/client-go/features" - clientfeaturestesting "k8s.io/client-go/features/testing" featuregatetesting "k8s.io/component-base/featuregate/testing" ) @@ -83,9 +81,6 @@ func TestEtcdStoragePath(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllAlpha", true) featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllBeta", true) - // TODO: Remove this override when the codecs used for serving all built-in APIs are wired to the apiserver feature gate. - clientfeaturestesting.SetFeatureDuringTest(t, clientfeatures.ClientsPreferCBOR, false) - apiServer := StartRealAPIServerOrDie(t) defer apiServer.Cleanup() defer dumpEtcdKVOnFailure(t, apiServer.KV) diff --git a/test/integration/etcd/server.go b/test/integration/etcd/server.go index ed69e3d532f..8a9c2869b6b 100644 --- a/test/integration/etcd/server.go +++ b/test/integration/etcd/server.go @@ -18,7 +18,6 @@ package etcd import ( "context" - "encoding/json" "net" "net/http" "os" @@ -26,8 +25,6 @@ import ( "testing" "time" - utiltesting "k8s.io/client-go/util/testing" - clientv3 "go.etcd.io/etcd/client/v3" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -38,6 +35,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/json" "k8s.io/apimachinery/pkg/util/wait" genericapiserveroptions "k8s.io/apiserver/pkg/server/options" cacheddiscovery "k8s.io/client-go/discovery/cached/memory" @@ -45,6 +43,7 @@ import ( clientset "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" "k8s.io/client-go/restmapper" + utiltesting "k8s.io/client-go/util/testing" "k8s.io/kubernetes/cmd/kube-apiserver/app" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" "k8s.io/kubernetes/test/integration" diff --git a/test/integration/framework/cbor.go b/test/integration/framework/cbor.go index 10ef193498c..6e12d808e71 100644 --- a/test/integration/framework/cbor.go +++ b/test/integration/framework/cbor.go @@ -22,59 +22,11 @@ import ( "net/http" "testing" - apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" - metainternalscheme "k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/serializer" - "k8s.io/apimachinery/pkg/runtime/serializer/cbor" "k8s.io/apimachinery/pkg/runtime/serializer/cbor/direct" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apiserver/pkg/features" - utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/transport" - featuregatetesting "k8s.io/component-base/featuregate/testing" - aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" - "k8s.io/kubernetes/pkg/api/legacyscheme" ) -// EnableCBORForTest patches global state to enable the CBOR serializer and reverses those changes -// at the end of the test. As a risk mitigation, integration tests are initially written this way so -// that integration tests can be implemented fully and incrementally before exposing options -// (including feature gates) that can enable CBOR at runtime. After integration test coverage is -// complete, feature gates will be introduced to completely supersede this mechanism. -func EnableCBORServingAndStorageForTest(tb testing.TB) { - featuregatetesting.SetFeatureGateDuringTest(tb, utilfeature.DefaultFeatureGate, features.CBORServingAndStorage, true) - - newCBORSerializerInfo := func(creater runtime.ObjectCreater, typer runtime.ObjectTyper) runtime.SerializerInfo { - return runtime.SerializerInfo{ - MediaType: "application/cbor", - MediaTypeType: "application", - MediaTypeSubType: "cbor", - Serializer: cbor.NewSerializer(creater, typer), - StrictSerializer: cbor.NewSerializer(creater, typer, cbor.Strict(true)), - StreamSerializer: &runtime.StreamSerializerInfo{ - Framer: cbor.NewFramer(), - Serializer: cbor.NewSerializer(creater, typer, cbor.Transcode(false)), - }, - } - } - - // Codecs for built-in types are constructed at package initialization time and read by - // value from REST storage providers. - codecs := map[*runtime.Scheme]*serializer.CodecFactory{ - legacyscheme.Scheme: &legacyscheme.Codecs, - metainternalscheme.Scheme: &metainternalscheme.Codecs, - aggregatorscheme.Scheme: &aggregatorscheme.Codecs, - apiextensionsapiserver.Scheme: &apiextensionsapiserver.Codecs, - } - - for scheme, factory := range codecs { - original := *factory // shallow copy of original value - tb.Cleanup(func() { *codecs[scheme] = original }) - *codecs[scheme] = serializer.NewCodecFactory(scheme, serializer.WithSerializer(newCBORSerializerInfo)) - } -} - // AssertRequestResponseAsCBOR returns a transport.WrapperFunc that will report a test error if a // non-empty request or response body contains data that does not appear to be CBOR-encoded. func AssertRequestResponseAsCBOR(t testing.TB) transport.WrapperFunc {