diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go index 02095d30ab8..dd2d7a51103 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go @@ -213,7 +213,7 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) s.GenericAPIServer.Handler.NonGoRestfulMux.HandlePrefix("/apis/", crdHandler) s.GenericAPIServer.RegisterDestroyFunc(crdHandler.destroy) - discoveryController := NewDiscoveryController(s.Informers.Apiextensions().V1().CustomResourceDefinitions(), versionDiscoveryHandler, groupDiscoveryHandler) + discoveryController := NewDiscoveryController(s.Informers.Apiextensions().V1().CustomResourceDefinitions(), versionDiscoveryHandler, groupDiscoveryHandler, genericServer.AggregatedDiscoveryGroupManager) namingController := status.NewNamingConditionController(s.Informers.Apiextensions().V1().CustomResourceDefinitions(), crdClient.ApiextensionsV1()) nonStructuralSchemaController := nonstructuralschema.NewConditionController(s.Informers.Apiextensions().V1().CustomResourceDefinitions(), crdClient.ApiextensionsV1()) apiApprovalController := apiapproval.NewKubernetesAPIApprovalPolicyConformantConditionController(s.Informers.Apiextensions().V1().CustomResourceDefinitions(), crdClient.ApiextensionsV1()) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller.go index aad0482633d..593a2b10b4d 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller.go @@ -23,6 +23,7 @@ import ( "k8s.io/klog/v2" + apidiscoveryv2beta1 "k8s.io/api/apidiscovery/v2beta1" autoscaling "k8s.io/api/autoscaling/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -31,6 +32,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/version" "k8s.io/apiserver/pkg/endpoints/discovery" + discoveryendpoint "k8s.io/apiserver/pkg/endpoints/discovery/aggregated" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" @@ -41,8 +43,9 @@ import ( ) type DiscoveryController struct { - versionHandler *versionDiscoveryHandler - groupHandler *groupDiscoveryHandler + versionHandler *versionDiscoveryHandler + groupHandler *groupDiscoveryHandler + resourceManager discoveryendpoint.ResourceManager crdLister listers.CustomResourceDefinitionLister crdsSynced cache.InformerSynced @@ -53,12 +56,18 @@ type DiscoveryController struct { queue workqueue.RateLimitingInterface } -func NewDiscoveryController(crdInformer informers.CustomResourceDefinitionInformer, versionHandler *versionDiscoveryHandler, groupHandler *groupDiscoveryHandler) *DiscoveryController { +func NewDiscoveryController( + crdInformer informers.CustomResourceDefinitionInformer, + versionHandler *versionDiscoveryHandler, + groupHandler *groupDiscoveryHandler, + resourceManager discoveryendpoint.ResourceManager, +) *DiscoveryController { c := &DiscoveryController{ - versionHandler: versionHandler, - groupHandler: groupHandler, - crdLister: crdInformer.Lister(), - crdsSynced: crdInformer.Informer().HasSynced, + versionHandler: versionHandler, + groupHandler: groupHandler, + resourceManager: resourceManager, + crdLister: crdInformer.Lister(), + crdsSynced: crdInformer.Informer().HasSynced, queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "DiscoveryController"), } @@ -78,6 +87,7 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error { apiVersionsForDiscovery := []metav1.GroupVersionForDiscovery{} apiResourcesForDiscovery := []metav1.APIResource{} + aggregatedApiResourcesForDiscovery := []apidiscoveryv2beta1.APIResourceDiscovery{} versionsForDiscoveryMap := map[metav1.GroupVersion]bool{} crds, err := c.crdLister.List(labels.Everything()) @@ -146,6 +156,53 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error { if err != nil { return err } + + if c.resourceManager != nil { + var scope apidiscoveryv2beta1.ResourceScope + if crd.Spec.Scope == apiextensionsv1.NamespaceScoped { + scope = apidiscoveryv2beta1.ScopeNamespace + } else { + scope = apidiscoveryv2beta1.ScopeCluster + } + apiResourceDiscovery := apidiscoveryv2beta1.APIResourceDiscovery{ + Resource: crd.Status.AcceptedNames.Plural, + SingularResource: crd.Status.AcceptedNames.Singular, + Scope: scope, + ResponseKind: &metav1.GroupVersionKind{ + Group: version.Group, + Version: version.Version, + Kind: crd.Status.AcceptedNames.Kind, + }, + Verbs: verbs, + ShortNames: crd.Status.AcceptedNames.ShortNames, + Categories: crd.Status.AcceptedNames.Categories, + } + if subresources != nil && subresources.Status != nil { + apiResourceDiscovery.Subresources = append(apiResourceDiscovery.Subresources, apidiscoveryv2beta1.APISubresourceDiscovery{ + Subresource: "status", + ResponseKind: &metav1.GroupVersionKind{ + Group: version.Group, + Version: version.Version, + Kind: crd.Status.AcceptedNames.Kind, + }, + Verbs: metav1.Verbs([]string{"get", "patch", "update"}), + }) + } + if subresources != nil && subresources.Scale != nil { + apiResourceDiscovery.Subresources = append(apiResourceDiscovery.Subresources, apidiscoveryv2beta1.APISubresourceDiscovery{ + Subresource: "scale", + ResponseKind: &metav1.GroupVersionKind{ + Group: autoscaling.GroupName, + Version: "v1", + Kind: "Scale", + }, + Verbs: metav1.Verbs([]string{"get", "patch", "update"}), + }) + + } + aggregatedApiResourcesForDiscovery = append(aggregatedApiResourcesForDiscovery, apiResourceDiscovery) + } + if subresources != nil && subresources.Status != nil { apiResourcesForDiscovery = append(apiResourcesForDiscovery, metav1.APIResource{ Name: crd.Status.AcceptedNames.Plural + "/status", @@ -170,6 +227,10 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error { if !foundGroup { c.groupHandler.unsetDiscovery(version.Group) c.versionHandler.unsetDiscovery(version) + + if c.resourceManager != nil { + c.resourceManager.RemoveGroup(version.Group) + } return nil } @@ -186,12 +247,30 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error { if !foundVersion { c.versionHandler.unsetDiscovery(version) + + if c.resourceManager != nil { + c.resourceManager.RemoveGroupVersion(metav1.GroupVersion{ + Group: version.Group, + Version: version.Version, + }) + } return nil } c.versionHandler.setDiscovery(version, discovery.NewAPIVersionHandler(Codecs, version, discovery.APIResourceListerFunc(func() []metav1.APIResource { return apiResourcesForDiscovery }))) + sort.Slice(aggregatedApiResourcesForDiscovery[:], func(i, j int) bool { + return aggregatedApiResourcesForDiscovery[i].Resource < aggregatedApiResourcesForDiscovery[j].Resource + }) + if c.resourceManager != nil { + c.resourceManager.AddGroupVersion(version.Group, apidiscoveryv2beta1.APIVersionDiscovery{ + Version: version.Version, + Resources: aggregatedApiResourcesForDiscovery, + }) + // Default priority for CRDs + c.resourceManager.SetGroupPriority(version.Group, 1000) + } return nil } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller_test.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller_test.go new file mode 100644 index 00000000000..f05a9ed00b3 --- /dev/null +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller_test.go @@ -0,0 +1,408 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiserver + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + apidiscoveryv2beta1 "k8s.io/api/apidiscovery/v2beta1" + v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake" + "k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apiserver/pkg/endpoints/discovery" + "k8s.io/apiserver/pkg/endpoints/discovery/aggregated" +) + +var coolFooCRD = &v1.CustomResourceDefinition{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apiextensions.k8s.io/v1", + Kind: "CustomResourceDefinition", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "coolfoo.stable.example.com", + }, + Spec: v1.CustomResourceDefinitionSpec{ + Group: "stable.example.com", + Names: v1.CustomResourceDefinitionNames{ + Plural: "coolfoos", + Singular: "coolfoo", + ShortNames: []string{"foo"}, + Kind: "CoolFoo", + ListKind: "CoolFooList", + Categories: []string{"cool"}, + }, + Scope: v1.ClusterScoped, + Versions: []v1.CustomResourceDefinitionVersion{ + { + Name: "v1", + Served: true, + Storage: true, + Deprecated: false, + Subresources: &v1.CustomResourceSubresources{ + // This CRD has a /status subresource + Status: &v1.CustomResourceSubresourceStatus{}, + }, + Schema: &v1.CustomResourceValidation{ + // Unused by discovery + OpenAPIV3Schema: &v1.JSONSchemaProps{}, + }, + }, + }, + Conversion: &v1.CustomResourceConversion{}, + PreserveUnknownFields: false, + }, + Status: v1.CustomResourceDefinitionStatus{ + Conditions: []v1.CustomResourceDefinitionCondition{ + { + Type: v1.Established, + Status: v1.ConditionTrue, + }, + }, + }, +} + +var coolBarCRD = &v1.CustomResourceDefinition{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apiextensions.k8s.io/v1", + Kind: "CustomResourceDefinition", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "coolbar.stable.example.com", + }, + Spec: v1.CustomResourceDefinitionSpec{ + Group: "stable.example.com", + Names: v1.CustomResourceDefinitionNames{ + Plural: "coolbars", + Singular: "coolbar", + ShortNames: []string{"bar"}, + Kind: "CoolBar", + ListKind: "CoolBarList", + Categories: []string{"cool"}, + }, + Scope: v1.ClusterScoped, + Versions: []v1.CustomResourceDefinitionVersion{ + { + Name: "v1", + Served: true, + Storage: true, + Deprecated: false, + Schema: &v1.CustomResourceValidation{ + // Unused by discovery + OpenAPIV3Schema: &v1.JSONSchemaProps{}, + }, + }, + }, + Conversion: &v1.CustomResourceConversion{}, + PreserveUnknownFields: false, + }, + Status: v1.CustomResourceDefinitionStatus{ + Conditions: []v1.CustomResourceDefinitionCondition{ + { + Type: v1.Established, + Status: v1.ConditionTrue, + }, + }, + }, +} + +var coolFooDiscovery apidiscoveryv2beta1.APIVersionDiscovery = apidiscoveryv2beta1.APIVersionDiscovery{ + Version: "v1", + Resources: []apidiscoveryv2beta1.APIResourceDiscovery{ + { + Resource: "coolfoos", + Scope: apidiscoveryv2beta1.ScopeCluster, + SingularResource: "coolfoo", + Verbs: []string{"delete", "deletecollection", "get", "list", "patch", "create", "update", "watch"}, + ShortNames: []string{"foo"}, + Categories: []string{"cool"}, + ResponseKind: &metav1.GroupVersionKind{ + Group: "stable.example.com", + Version: "v1", + Kind: "CoolFoo", + }, + Subresources: []apidiscoveryv2beta1.APISubresourceDiscovery{ + { + Subresource: "status", + Verbs: []string{"get", "patch", "update"}, + AcceptedTypes: nil, // is this correct? + ResponseKind: &metav1.GroupVersionKind{ + Group: "stable.example.com", + Version: "v1", + Kind: "CoolFoo", + }, + }, + }, + }, + }, +} + +var mergedDiscovery apidiscoveryv2beta1.APIVersionDiscovery = apidiscoveryv2beta1.APIVersionDiscovery{ + Version: "v1", + Resources: []apidiscoveryv2beta1.APIResourceDiscovery{ + { + Resource: "coolbars", + Scope: apidiscoveryv2beta1.ScopeCluster, + SingularResource: "coolbar", + Verbs: []string{"delete", "deletecollection", "get", "list", "patch", "create", "update", "watch"}, + ShortNames: []string{"bar"}, + Categories: []string{"cool"}, + ResponseKind: &metav1.GroupVersionKind{ + Group: "stable.example.com", + Version: "v1", + Kind: "CoolBar", + }, + }, { + Resource: "coolfoos", + Scope: apidiscoveryv2beta1.ScopeCluster, + SingularResource: "coolfoo", + Verbs: []string{"delete", "deletecollection", "get", "list", "patch", "create", "update", "watch"}, + ShortNames: []string{"foo"}, + Categories: []string{"cool"}, + ResponseKind: &metav1.GroupVersionKind{ + Group: "stable.example.com", + Version: "v1", + Kind: "CoolFoo", + }, + Subresources: []apidiscoveryv2beta1.APISubresourceDiscovery{ + { + Subresource: "status", + Verbs: []string{"get", "patch", "update"}, + AcceptedTypes: nil, // is this correct? + ResponseKind: &metav1.GroupVersionKind{ + Group: "stable.example.com", + Version: "v1", + Kind: "CoolFoo", + }, + }, + }, + }, + }, +} + +func init() { + // Not testing against an apiserver, so just assume names are accepted + coolFooCRD.Status.AcceptedNames = coolFooCRD.Spec.Names + coolBarCRD.Status.AcceptedNames = coolBarCRD.Spec.Names +} + +// Provides an apiextensions-apiserver client +type testEnvironment struct { + clientset.Interface + + // Discovery test details + versionDiscoveryHandler + groupDiscoveryHandler + + aggregated.FakeResourceManager +} + +func (env *testEnvironment) Start(ctx context.Context) { + discoverySyncedCh := make(chan struct{}) + + factory := externalversions.NewSharedInformerFactoryWithOptions( + env.Interface, 30*time.Second) + + discoveryController := NewDiscoveryController( + factory.Apiextensions().V1().CustomResourceDefinitions(), + &env.versionDiscoveryHandler, + &env.groupDiscoveryHandler, + env.FakeResourceManager, + ) + + factory.Start(ctx.Done()) + go discoveryController.Run(ctx.Done(), discoverySyncedCh) + + select { + case <-discoverySyncedCh: + case <-ctx.Done(): + } +} + +func setup() *testEnvironment { + env := &testEnvironment{ + Interface: fake.NewSimpleClientset(), + FakeResourceManager: aggregated.NewFakeResourceManager(), + versionDiscoveryHandler: versionDiscoveryHandler{ + discovery: make(map[schema.GroupVersion]*discovery.APIVersionHandler), + }, + groupDiscoveryHandler: groupDiscoveryHandler{ + discovery: make(map[string]*discovery.APIGroupHandler), + }, + } + + return env +} + +func TestResourceManagerExistingCRD(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + env := setup() + _, err := env.Interface. + ApiextensionsV1(). + CustomResourceDefinitions(). + Create( + ctx, + coolFooCRD, + metav1.CreateOptions{ + FieldManager: "resource-manager-test", + }, + ) + + require.NoError(t, err) + + env.FakeResourceManager.Expect(). + AddGroupVersion(coolFooCRD.Spec.Group, coolFooDiscovery) + env.FakeResourceManager.Expect(). + SetGroupPriority(coolFooCRD.Spec.Group, 1000) + + env.FakeResourceManager.Expect(). + AddGroupVersion(coolFooCRD.Spec.Group, coolFooDiscovery) + env.FakeResourceManager.Expect(). + SetGroupPriority(coolFooCRD.Spec.Group, 1000) + + env.Start(ctx) + err = env.FakeResourceManager.WaitForActions(ctx, 1*time.Second) + require.NoError(t, err) +} + +// Tests that if a CRD is added a runtime, the discovery controller will +// put its information in the discovery document +func TestResourceManagerAddedCRD(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + env := setup() + env.FakeResourceManager.Expect(). + AddGroupVersion(coolFooCRD.Spec.Group, coolFooDiscovery) + env.FakeResourceManager.Expect().SetGroupPriority(coolFooCRD.Spec.Group, 1000) + + env.Start(ctx) + + // Create CRD after the controller has already started + _, err := env.Interface. + ApiextensionsV1(). + CustomResourceDefinitions(). + Create( + ctx, + coolFooCRD, + metav1.CreateOptions{ + FieldManager: "resource-manager-test", + }, + ) + + require.NoError(t, err) + + err = env.FakeResourceManager.WaitForActions(ctx, 1*time.Second) + require.NoError(t, err) +} + +// Test that having multiple CRDs in the same version will add both +// versions to discovery. +func TestMultipleCRDSameVersion(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + env := setup() + env.Start(ctx) + + _, err := env.Interface. + ApiextensionsV1(). + CustomResourceDefinitions(). + Create( + ctx, + coolFooCRD, + metav1.CreateOptions{ + FieldManager: "resource-manager-test", + }, + ) + + require.NoError(t, err) + env.FakeResourceManager.Expect(). + AddGroupVersion(coolFooCRD.Spec.Group, coolFooDiscovery) + env.FakeResourceManager.Expect().SetGroupPriority(coolFooCRD.Spec.Group, 1000) + err = env.FakeResourceManager.WaitForActions(ctx, 1*time.Second) + require.NoError(t, err) + + _, err = env.Interface. + ApiextensionsV1(). + CustomResourceDefinitions(). + Create( + ctx, + coolBarCRD, + metav1.CreateOptions{ + FieldManager: "resource-manager-test", + }, + ) + require.NoError(t, err) + + env.FakeResourceManager.Expect(). + AddGroupVersion(coolFooCRD.Spec.Group, mergedDiscovery) + env.FakeResourceManager.Expect().SetGroupPriority(coolFooCRD.Spec.Group, 1000) + err = env.FakeResourceManager.WaitForActions(ctx, 1*time.Second) + require.NoError(t, err) +} + +// Tests that if a CRD is deleted at runtime, the discovery controller will +// remove its information from its ResourceManager +func TestDiscoveryControllerResourceManagerRemovedCRD(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + env := setup() + env.Start(ctx) + + // Create CRD after the controller has already started + _, err := env.Interface. + ApiextensionsV1(). + CustomResourceDefinitions(). + Create( + ctx, + coolFooCRD, + metav1.CreateOptions{}, + ) + + require.NoError(t, err) + + // Wait for the Controller to pick up the Create event and add it to the + // Resource Manager + env.FakeResourceManager.Expect(). + AddGroupVersion(coolFooCRD.Spec.Group, coolFooDiscovery) + env.FakeResourceManager.Expect().SetGroupPriority(coolFooCRD.Spec.Group, 1000) + err = env.FakeResourceManager.WaitForActions(ctx, 1*time.Second) + require.NoError(t, err) + + err = env.Interface. + ApiextensionsV1(). + CustomResourceDefinitions(). + Delete(ctx, coolFooCRD.Name, metav1.DeleteOptions{}) + + require.NoError(t, err) + + // Wait for the Controller to detect there are no more CRDs of this group + // and remove the entire group + env.FakeResourceManager.Expect().RemoveGroup(coolFooCRD.Spec.Group) + + err = env.FakeResourceManager.WaitForActions(ctx, 1*time.Second) + require.NoError(t, err) +}