From 90ab7580aaeca1c6e949df15554ad5bc408dca8e Mon Sep 17 00:00:00 2001 From: David Eads Date: Wed, 12 Jul 2023 18:27:27 -0400 Subject: [PATCH] add list of served versions to storage version --- .../apiserver/pkg/endpoints/groupversion.go | 5 +++++ .../k8s.io/apiserver/pkg/endpoints/installer.go | 3 +++ .../apiserver/pkg/server/genericapiserver.go | 17 +++++++++++++++-- .../apiserver/pkg/storageversion/manager.go | 17 ++++++++--------- .../kube-aggregator/pkg/apiserver/apiserver.go | 2 +- .../storage_version_filter_test.go | 5 ++--- 6 files changed, 34 insertions(+), 15 deletions(-) diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/groupversion.go b/staging/src/k8s.io/apiserver/pkg/endpoints/groupversion.go index 3c70e89ec0e..0ce06ab1069 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/groupversion.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/groupversion.go @@ -56,6 +56,11 @@ type APIGroupVersion struct { // GroupVersion is the external group version GroupVersion schema.GroupVersion + // AllServedVersionsByResource is indexed by resource and maps to a list of versions that resource exists in. + // This was created so that StorageVersion for APIs can include a list of all version that are served for each + // GroupResource tuple. + AllServedVersionsByResource map[string][]string + // OptionsExternalVersion controls the Kubernetes APIVersion used for common objects in the apiserver // schema like api.Status, api.DeleteOptions, and metav1.ListOptions. Other implementors may // define a version "v1beta1" but want to use the Kubernetes "v1" internal objects. If diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go b/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go index f1adddba54a..342c3b3d264 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go @@ -600,6 +600,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag if a.group.ConvertabilityChecker != nil { decodableVersions = a.group.ConvertabilityChecker.VersionsForGroupKind(fqKindToRegister.GroupKind()) } + resourceInfo = &storageversion.ResourceInfo{ GroupResource: schema.GroupResource{ Group: a.group.GroupVersion.Group, @@ -612,6 +613,8 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag EquivalentResourceMapper: a.group.EquivalentResourceRegistry, DirectlyDecodableVersions: decodableVersions, + + ServedVersions: a.group.AllServedVersionsByResource[path], } } diff --git a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go index 794a5db2f77..665f20bebdb 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go +++ b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go @@ -911,9 +911,22 @@ func (s *GenericAPIServer) getAPIGroupVersion(apiGroupInfo *APIGroupInfo, groupV } func (s *GenericAPIServer) newAPIGroupVersion(apiGroupInfo *APIGroupInfo, groupVersion schema.GroupVersion) *genericapi.APIGroupVersion { + + allServedVersionsByResource := map[string][]string{} + for version, resourcesInVersion := range apiGroupInfo.VersionedResourcesStorageMap { + for resource := range resourcesInVersion { + if len(groupVersion.Group) == 0 { + allServedVersionsByResource[resource] = append(allServedVersionsByResource[resource], version) + } else { + allServedVersionsByResource[resource] = append(allServedVersionsByResource[resource], fmt.Sprintf("%s/%s", groupVersion.Group, version)) + } + } + } + return &genericapi.APIGroupVersion{ - GroupVersion: groupVersion, - MetaGroupVersion: apiGroupInfo.MetaGroupVersion, + GroupVersion: groupVersion, + AllServedVersionsByResource: allServedVersionsByResource, + MetaGroupVersion: apiGroupInfo.MetaGroupVersion, ParameterCodec: apiGroupInfo.ParameterCodec, Serializer: apiGroupInfo.NegotiatedSerializer, diff --git a/staging/src/k8s.io/apiserver/pkg/storageversion/manager.go b/staging/src/k8s.io/apiserver/pkg/storageversion/manager.go index 87ab4ac7824..d7d3863118a 100644 --- a/staging/src/k8s.io/apiserver/pkg/storageversion/manager.go +++ b/staging/src/k8s.io/apiserver/pkg/storageversion/manager.go @@ -25,7 +25,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - serverstorage "k8s.io/apiserver/pkg/server/storage" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" _ "k8s.io/component-base/metrics/prometheus/workqueue" // for workqueue metric registration @@ -45,6 +44,10 @@ type ResourceInfo struct { // DirectlyDecodableVersions is a list of versions that the converter for REST storage knows how to convert. This // contains items like apiextensions.k8s.io/v1beta1 even if we don't serve that version. DirectlyDecodableVersions []schema.GroupVersion + + // ServedVersions holds a list of all versions of GroupResource that are served. Note that a server may be able to + // decode a particular version, but still not serve it. + ServedVersions []string } // Manager records the resources whose StorageVersions need updates, and provides a method to update those StorageVersions. @@ -52,7 +55,7 @@ type Manager interface { // AddResourceInfo records resources whose StorageVersions need updates AddResourceInfo(resources ...*ResourceInfo) // UpdateStorageVersions tries to update the StorageVersions of the recorded resources - UpdateStorageVersions(kubeAPIServerClientConfig *rest.Config, apiserverID string, apiResourceConfigSource serverstorage.APIResourceConfigSource) + UpdateStorageVersions(kubeAPIServerClientConfig *rest.Config, apiserverID string) // PendingUpdate returns true if the StorageVersion of the given resource is still pending update. PendingUpdate(gr schema.GroupResource) bool // LastUpdateError returns the last error hit when updating the storage version of the given resource. @@ -112,7 +115,7 @@ func (s *defaultManager) addPendingManagedStatusLocked(r *ResourceInfo) { } // UpdateStorageVersions tries to update the StorageVersions of the recorded resources -func (s *defaultManager) UpdateStorageVersions(kubeAPIServerClientConfig *rest.Config, serverID string, apiResourceConfigSource serverstorage.APIResourceConfigSource) { +func (s *defaultManager) UpdateStorageVersions(kubeAPIServerClientConfig *rest.Config, serverID string) { clientset, err := kubernetes.NewForConfig(kubeAPIServerClientConfig) if err != nil { utilruntime.HandleError(fmt.Errorf("failed to get clientset: %v", err)) @@ -145,12 +148,8 @@ func (s *defaultManager) UpdateStorageVersions(kubeAPIServerClientConfig *rest.C gr.Group = "core" } - servedVersions := []string{} - for _, dv := range decodableVersions { - if apiResourceConfigSource.ResourceEnabled(gr.WithVersion(dv)) { - servedVersions = append(servedVersions, dv) - } - } + servedVersions := r.ServedVersions + if err := updateStorageVersionFor(sc, serverID, gr, r.EncodingVersion, decodableVersions, servedVersions); err != nil { utilruntime.HandleError(fmt.Errorf("failed to update storage version for %v: %v", r.GroupResource, err)) s.recordStatusFailure(&r, err) diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go index a5b0372ad78..bc4da3a8946 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go @@ -367,7 +367,7 @@ func (c completedConfig) NewWithDelegate(delegationTarget genericapiserver.Deleg // All apiservers (aggregator-apiserver, kube-apiserver, apiextensions-apiserver) // share the same generic apiserver config. The same StorageVersion manager is used // to register all built-in resources when the generic apiservers install APIs. - s.GenericAPIServer.StorageVersionManager.UpdateStorageVersions(hookContext.LoopbackClientConfig, s.GenericAPIServer.APIServerID, c.GenericConfig.MergedResourceConfig) + s.GenericAPIServer.StorageVersionManager.UpdateStorageVersions(hookContext.LoopbackClientConfig, s.GenericAPIServer.APIServerID) return false, nil }, hookContext.StopCh) // Once the storage version updater finishes the first round of update, diff --git a/test/integration/storageversion/storage_version_filter_test.go b/test/integration/storageversion/storage_version_filter_test.go index 018ba8c4f8b..a22fde91626 100644 --- a/test/integration/storageversion/storage_version_filter_test.go +++ b/test/integration/storageversion/storage_version_filter_test.go @@ -31,7 +31,6 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/features" - serverstorage "k8s.io/apiserver/pkg/server/storage" "k8s.io/apiserver/pkg/storageversion" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/dynamic" @@ -53,9 +52,9 @@ type wrappedStorageVersionManager struct { completed <-chan struct{} } -func (w *wrappedStorageVersionManager) UpdateStorageVersions(loopbackClientConfig *rest.Config, serverID string, apiResourceConfigSource serverstorage.APIResourceConfigSource) { +func (w *wrappedStorageVersionManager) UpdateStorageVersions(loopbackClientConfig *rest.Config, serverID string) { <-w.startUpdateSV - w.Manager.UpdateStorageVersions(loopbackClientConfig, serverID, apiResourceConfigSource) + w.Manager.UpdateStorageVersions(loopbackClientConfig, serverID) close(w.updateFinished) <-w.finishUpdateSV }