diff --git a/pkg/controlplane/BUILD b/pkg/controlplane/BUILD index 91295ebfa55..0eb2b49da9e 100644 --- a/pkg/controlplane/BUILD +++ b/pkg/controlplane/BUILD @@ -5,7 +5,6 @@ go_library( srcs = [ "client_util.go", "controller.go", - "deleted_kinds.go", "doc.go", "import_known_versions.go", "instance.go", @@ -116,13 +115,10 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/version:go_default_library", "//staging/src/k8s.io/apiserver/pkg/endpoints/discovery:go_default_library", "//staging/src/k8s.io/apiserver/pkg/features:go_default_library", "//staging/src/k8s.io/apiserver/pkg/registry/generic:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/registry/rest:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/healthz:go_default_library", @@ -148,7 +144,6 @@ go_test( timeout = "long", srcs = [ "controller_test.go", - "deleted_kinds_test.go", "import_known_versions_test.go", "instance_openapi_test.go", "instance_test.go", @@ -172,7 +167,6 @@ go_test( "//staging/src/k8s.io/api/discovery/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/apitesting/naming:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", @@ -180,7 +174,6 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/version:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory:go_default_library", "//staging/src/k8s.io/apiserver/pkg/endpoints/openapi:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/registry/rest:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/resourceconfig:go_default_library", @@ -193,7 +186,6 @@ go_test( "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/testing:go_default_library", "//staging/src/k8s.io/component-base/version:go_default_library", - "//vendor/github.com/davecgh/go-spew/spew:go_default_library", "//vendor/github.com/go-openapi/loads:go_default_library", "//vendor/github.com/go-openapi/spec:go_default_library", "//vendor/github.com/go-openapi/strfmt:go_default_library", diff --git a/pkg/controlplane/instance.go b/pkg/controlplane/instance.go index 00551b826de..ed3c4ca5946 100644 --- a/pkg/controlplane/instance.go +++ b/pkg/controlplane/instance.go @@ -580,7 +580,7 @@ func (m *Instance) InstallAPIs(apiResourceConfigSource serverstorage.APIResource apiGroupsInfo := []*genericapiserver.APIGroupInfo{} // used later in the loop to filter the served resource by those that have expired. - resourceExpirationEvaluator, err := newResourceExpirationEvaluator(version.Get()) + resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(version.Get()) if err != nil { return err } @@ -603,7 +603,7 @@ func (m *Instance) InstallAPIs(apiResourceConfigSource serverstorage.APIResource // Remove resources that serving kinds that are removed. // We do this here so that we don't accidentally serve versions without resources or openapi information that for kinds we don't serve. // This is a spot above the construction of individual storage handlers so that no sig accidentally forgets to check. - resourceExpirationEvaluator.removeDeletedKinds(groupName, apiGroupInfo.VersionedResourcesStorageMap) + resourceExpirationEvaluator.RemoveDeletedKinds(groupName, apiGroupInfo.VersionedResourcesStorageMap) if len(apiGroupInfo.VersionedResourcesStorageMap) == 0 { klog.V(1).Infof("Removing API group %v because it is time to stop serving it because it has no versions per APILifecycle.", groupName) continue diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go index 3c621b5c779..ece52922ee9 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go @@ -135,9 +135,15 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) GenericAPIServer: genericServer, } + // used later to filter the served resource by those that have expired. + resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(*c.GenericConfig.Version) + if err != nil { + return nil, err + } + apiResourceConfig := c.GenericConfig.MergedResourceConfig apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(apiextensions.GroupName, Scheme, metav1.ParameterCodec, Codecs) - if apiResourceConfig.VersionEnabled(v1beta1.SchemeGroupVersion) { + if resourceExpirationEvaluator.ShouldServeForVersion(1, 22) && apiResourceConfig.VersionEnabled(v1beta1.SchemeGroupVersion) { storage := map[string]rest.Storage{} // customresourcedefinitions customResourceDefinitionStorage, err := customresourcedefinition.NewREST(Scheme, c.GenericConfig.RESTOptionsGetter) diff --git a/staging/src/k8s.io/apiserver/pkg/server/BUILD b/staging/src/k8s.io/apiserver/pkg/server/BUILD index 39e7481ed7f..0e22e4a2344 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/BUILD @@ -11,6 +11,7 @@ go_test( srcs = [ "config_selfclient_test.go", "config_test.go", + "deleted_kinds_test.go", "genericapiserver_test.go", "healthz_test.go", ], @@ -44,6 +45,7 @@ go_test( "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", + "//vendor/github.com/davecgh/go-spew/spew:go_default_library", "//vendor/github.com/go-openapi/spec:go_default_library", "//vendor/github.com/google/go-cmp/cmp:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", @@ -57,6 +59,7 @@ go_library( srcs = [ "config.go", "config_selfclient.go", + "deleted_kinds.go", "deprecated_insecure_serving.go", "doc.go", "genericapiserver.go", diff --git a/pkg/controlplane/deleted_kinds.go b/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds.go similarity index 85% rename from pkg/controlplane/deleted_kinds.go rename to staging/src/k8s.io/apiserver/pkg/server/deleted_kinds.go index f9604c8761f..931264a0d15 100644 --- a/pkg/controlplane/deleted_kinds.go +++ b/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controlplane +package server import ( "os" @@ -41,7 +41,16 @@ type resourceExpirationEvaluator struct { serveRemovedAPIsOneMoreRelease bool } -func newResourceExpirationEvaluator(currentVersion apimachineryversion.Info) (*resourceExpirationEvaluator, error) { +// ResourceExpirationEvaluator indicates whether or not a resource should be served. +type ResourceExpirationEvaluator interface { + // RemoveDeletedKinds inspects the storage map and modifies it in place by removing storage for kinds that have been deleted. + // versionedResourcesStorageMap mirrors the field on APIGroupInfo, it's a map from version to resource to the storage. + RemoveDeletedKinds(groupName string, versionedResourcesStorageMap map[string]map[string]rest.Storage) + // ShouldServeForVersion returns true if a particular version cut off is after the current version + ShouldServeForVersion(majorRemoved, minorRemoved int) bool +} + +func NewResourceExpirationEvaluator(currentVersion apimachineryversion.Info) (ResourceExpirationEvaluator, error) { ret := &resourceExpirationEvaluator{} if len(currentVersion.Major) > 0 { currentMajor64, err := strconv.ParseInt(currentVersion.Major, 10, 32) @@ -89,6 +98,10 @@ func (e *resourceExpirationEvaluator) shouldServe(resourceServingInfo rest.Stora return true } majorRemoved, minorRemoved := removed.APILifecycleRemoved() + return e.ShouldServeForVersion(majorRemoved, minorRemoved) +} + +func (e *resourceExpirationEvaluator) ShouldServeForVersion(majorRemoved, minorRemoved int) bool { if e.currentMajor < majorRemoved { return true } @@ -122,7 +135,7 @@ type removedInterface interface { // removeDeletedKinds inspects the storage map and modifies it in place by removing storage for kinds that have been deleted. // versionedResourcesStorageMap mirrors the field on APIGroupInfo, it's a map from version to resource to the storage. -func (e *resourceExpirationEvaluator) removeDeletedKinds(groupName string, versionedResourcesStorageMap map[string]map[string]rest.Storage) { +func (e *resourceExpirationEvaluator) RemoveDeletedKinds(groupName string, versionedResourcesStorageMap map[string]map[string]rest.Storage) { versionsToRemove := sets.NewString() for apiVersion, versionToResource := range versionedResourcesStorageMap { resourcesToRemove := sets.NewString() diff --git a/pkg/controlplane/deleted_kinds_test.go b/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds_test.go similarity index 97% rename from pkg/controlplane/deleted_kinds_test.go rename to staging/src/k8s.io/apiserver/pkg/server/deleted_kinds_test.go index b920e8533ac..54ad13a4d3d 100644 --- a/pkg/controlplane/deleted_kinds_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controlplane +package server import ( "reflect" @@ -75,14 +75,14 @@ func Test_newResourceExpirationEvaluator(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - actual, actualErr := newResourceExpirationEvaluator(tt.currentVersion) + actual, actualErr := NewResourceExpirationEvaluator(tt.currentVersion) checkErr(t, actualErr, tt.expectedErr) if actualErr != nil { return } - if !reflect.DeepEqual(tt.expected, *actual) { + if !reflect.DeepEqual(tt.expected, *actual.(*resourceExpirationEvaluator)) { t.Fatal(actual) } }) @@ -309,7 +309,7 @@ func Test_removeDeletedKinds(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - tt.resourceExpirationEvaluator.removeDeletedKinds("group.name", tt.versionedResourcesStorageMap) + tt.resourceExpirationEvaluator.RemoveDeletedKinds("group.name", tt.versionedResourcesStorageMap) if !reflect.DeepEqual(tt.expectedStorage, tt.versionedResourcesStorageMap) { t.Fatal(spew.Sdump(tt.versionedResourcesStorageMap)) } diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go index 671f39730e8..912645a35ec 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go @@ -195,7 +195,13 @@ func (c completedConfig) NewWithDelegate(delegationTarget genericapiserver.Deleg proxyCurrentCertKeyContent: func() (bytes []byte, bytes2 []byte) { return nil, nil }, } - apiGroupInfo := apiservicerest.NewRESTStorage(c.GenericConfig.MergedResourceConfig, c.GenericConfig.RESTOptionsGetter) + // used later to filter the served resource by those that have expired. + resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(*c.GenericConfig.Version) + if err != nil { + return nil, err + } + + apiGroupInfo := apiservicerest.NewRESTStorage(c.GenericConfig.MergedResourceConfig, c.GenericConfig.RESTOptionsGetter, resourceExpirationEvaluator.ShouldServeForVersion(1, 22)) if err := s.GenericAPIServer.InstallAPIGroup(&apiGroupInfo); err != nil { return nil, err } diff --git a/staging/src/k8s.io/kube-aggregator/pkg/registry/apiservice/rest/storage_apiservice.go b/staging/src/k8s.io/kube-aggregator/pkg/registry/apiservice/rest/storage_apiservice.go index c10b5b233f4..fc3fd6b2f1a 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/registry/apiservice/rest/storage_apiservice.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/registry/apiservice/rest/storage_apiservice.go @@ -24,17 +24,17 @@ import ( serverstorage "k8s.io/apiserver/pkg/server/storage" "k8s.io/kube-aggregator/pkg/apis/apiregistration" - "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" + v1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1" aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" apiservicestorage "k8s.io/kube-aggregator/pkg/registry/apiservice/etcd" ) // NewRESTStorage returns an APIGroupInfo object that will work against apiservice. -func NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) genericapiserver.APIGroupInfo { +func NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter, shouldServeBeta bool) genericapiserver.APIGroupInfo { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(apiregistration.GroupName, aggregatorscheme.Scheme, metav1.ParameterCodec, aggregatorscheme.Codecs) - if apiResourceConfigSource.VersionEnabled(v1beta1.SchemeGroupVersion) { + if shouldServeBeta && apiResourceConfigSource.VersionEnabled(v1beta1.SchemeGroupVersion) { storage := map[string]rest.Storage{} apiServiceREST := apiservicestorage.NewREST(aggregatorscheme.Scheme, restOptionsGetter) storage["apiservices"] = apiServiceREST