add crds to aggregated discovery

Co-authored-by: Jeffrey Ying <jeffrey.ying86@live.com>

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go

@@ -213,7 +213,7 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
 	s.GenericAPIServer.Handler.NonGoRestfulMux.HandlePrefix("/apis/", crdHandler)
 	s.GenericAPIServer.RegisterDestroyFunc(crdHandler.destroy)
 
-	discoveryController := NewDiscoveryController(s.Informers.Apiextensions().V1().CustomResourceDefinitions(), versionDiscoveryHandler, groupDiscoveryHandler)
+	discoveryController := NewDiscoveryController(s.Informers.Apiextensions().V1().CustomResourceDefinitions(), versionDiscoveryHandler, groupDiscoveryHandler, genericServer.AggregatedDiscoveryGroupManager)
 	namingController := status.NewNamingConditionController(s.Informers.Apiextensions().V1().CustomResourceDefinitions(), crdClient.ApiextensionsV1())
 	nonStructuralSchemaController := nonstructuralschema.NewConditionController(s.Informers.Apiextensions().V1().CustomResourceDefinitions(), crdClient.ApiextensionsV1())
 	apiApprovalController := apiapproval.NewKubernetesAPIApprovalPolicyConformantConditionController(s.Informers.Apiextensions().V1().CustomResourceDefinitions(), crdClient.ApiextensionsV1())

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller.go

@@ -23,6 +23,7 @@ import (
 
 	"k8s.io/klog/v2"
 
+	apidiscoveryv2beta1 "k8s.io/api/apidiscovery/v2beta1"
 	autoscaling "k8s.io/api/autoscaling/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
@@ -31,6 +32,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apimachinery/pkg/version"
 	"k8s.io/apiserver/pkg/endpoints/discovery"
+	discoveryendpoint "k8s.io/apiserver/pkg/endpoints/discovery/aggregated"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/util/workqueue"
 
@@ -41,8 +43,9 @@ import (
 )
 
 type DiscoveryController struct {
-	versionHandler *versionDiscoveryHandler
+	versionHandler  *versionDiscoveryHandler
-	groupHandler   *groupDiscoveryHandler
+	groupHandler    *groupDiscoveryHandler
+	resourceManager discoveryendpoint.ResourceManager
 
 	crdLister  listers.CustomResourceDefinitionLister
 	crdsSynced cache.InformerSynced
@@ -53,12 +56,18 @@ type DiscoveryController struct {
 	queue workqueue.RateLimitingInterface
 }
 
-func NewDiscoveryController(crdInformer informers.CustomResourceDefinitionInformer, versionHandler *versionDiscoveryHandler, groupHandler *groupDiscoveryHandler) *DiscoveryController {
+func NewDiscoveryController(
+	crdInformer informers.CustomResourceDefinitionInformer,
+	versionHandler *versionDiscoveryHandler,
+	groupHandler *groupDiscoveryHandler,
+	resourceManager discoveryendpoint.ResourceManager,
+) *DiscoveryController {
 	c := &DiscoveryController{
-		versionHandler: versionHandler,
+		versionHandler:  versionHandler,
-		groupHandler:   groupHandler,
+		groupHandler:    groupHandler,
-		crdLister:      crdInformer.Lister(),
+		resourceManager: resourceManager,
-		crdsSynced:     crdInformer.Informer().HasSynced,
+		crdLister:       crdInformer.Lister(),
+		crdsSynced:      crdInformer.Informer().HasSynced,
 
 		queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "DiscoveryController"),
 	}
@@ -78,6 +87,7 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error {
 
 	apiVersionsForDiscovery := []metav1.GroupVersionForDiscovery{}
 	apiResourcesForDiscovery := []metav1.APIResource{}
+	aggregatedApiResourcesForDiscovery := []apidiscoveryv2beta1.APIResourceDiscovery{}
 	versionsForDiscoveryMap := map[metav1.GroupVersion]bool{}
 
 	crds, err := c.crdLister.List(labels.Everything())
@@ -146,6 +156,53 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error {
 		if err != nil {
 			return err
 		}
+
+		if c.resourceManager != nil {
+			var scope apidiscoveryv2beta1.ResourceScope
+			if crd.Spec.Scope == apiextensionsv1.NamespaceScoped {
+				scope = apidiscoveryv2beta1.ScopeNamespace
+			} else {
+				scope = apidiscoveryv2beta1.ScopeCluster
+			}
+			apiResourceDiscovery := apidiscoveryv2beta1.APIResourceDiscovery{
+				Resource:         crd.Status.AcceptedNames.Plural,
+				SingularResource: crd.Status.AcceptedNames.Singular,
+				Scope:            scope,
+				ResponseKind: &metav1.GroupVersionKind{
+					Group:   version.Group,
+					Version: version.Version,
+					Kind:    crd.Status.AcceptedNames.Kind,
+				},
+				Verbs:      verbs,
+				ShortNames: crd.Status.AcceptedNames.ShortNames,
+				Categories: crd.Status.AcceptedNames.Categories,
+			}
+			if subresources != nil && subresources.Status != nil {
+				apiResourceDiscovery.Subresources = append(apiResourceDiscovery.Subresources, apidiscoveryv2beta1.APISubresourceDiscovery{
+					Subresource: "status",
+					ResponseKind: &metav1.GroupVersionKind{
+						Group:   version.Group,
+						Version: version.Version,
+						Kind:    crd.Status.AcceptedNames.Kind,
+					},
+					Verbs: metav1.Verbs([]string{"get", "patch", "update"}),
+				})
+			}
+			if subresources != nil && subresources.Scale != nil {
+				apiResourceDiscovery.Subresources = append(apiResourceDiscovery.Subresources, apidiscoveryv2beta1.APISubresourceDiscovery{
+					Subresource: "scale",
+					ResponseKind: &metav1.GroupVersionKind{
+						Group:   autoscaling.GroupName,
+						Version: "v1",
+						Kind:    "Scale",
+					},
+					Verbs: metav1.Verbs([]string{"get", "patch", "update"}),
+				})
+
+			}
+			aggregatedApiResourcesForDiscovery = append(aggregatedApiResourcesForDiscovery, apiResourceDiscovery)
+		}
+
 		if subresources != nil && subresources.Status != nil {
 			apiResourcesForDiscovery = append(apiResourcesForDiscovery, metav1.APIResource{
 				Name:       crd.Status.AcceptedNames.Plural + "/status",
@@ -170,6 +227,10 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error {
 	if !foundGroup {
 		c.groupHandler.unsetDiscovery(version.Group)
 		c.versionHandler.unsetDiscovery(version)
+
+		if c.resourceManager != nil {
+			c.resourceManager.RemoveGroup(version.Group)
+		}
 		return nil
 	}
 
@@ -186,12 +247,30 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error {
 
 	if !foundVersion {
 		c.versionHandler.unsetDiscovery(version)
+
+		if c.resourceManager != nil {
+			c.resourceManager.RemoveGroupVersion(metav1.GroupVersion{
+				Group:   version.Group,
+				Version: version.Version,
+			})
+		}
 		return nil
 	}
 	c.versionHandler.setDiscovery(version, discovery.NewAPIVersionHandler(Codecs, version, discovery.APIResourceListerFunc(func() []metav1.APIResource {
 		return apiResourcesForDiscovery
 	})))
 
+	sort.Slice(aggregatedApiResourcesForDiscovery[:], func(i, j int) bool {
+		return aggregatedApiResourcesForDiscovery[i].Resource < aggregatedApiResourcesForDiscovery[j].Resource
+	})
+	if c.resourceManager != nil {
+		c.resourceManager.AddGroupVersion(version.Group, apidiscoveryv2beta1.APIVersionDiscovery{
+			Version:   version.Version,
+			Resources: aggregatedApiResourcesForDiscovery,
+		})
+		// Default priority for CRDs
+		c.resourceManager.SetGroupPriority(version.Group, 1000)
+	}
 	return nil
 }
 

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller_test.go

@@ -0,0 +1,408 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apiserver
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	apidiscoveryv2beta1 "k8s.io/api/apidiscovery/v2beta1"
+	v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake"
+	"k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apiserver/pkg/endpoints/discovery"
+	"k8s.io/apiserver/pkg/endpoints/discovery/aggregated"
+)
+
+var coolFooCRD = &v1.CustomResourceDefinition{
+	TypeMeta: metav1.TypeMeta{
+		APIVersion: "apiextensions.k8s.io/v1",
+		Kind:       "CustomResourceDefinition",
+	},
+	ObjectMeta: metav1.ObjectMeta{
+		Name: "coolfoo.stable.example.com",
+	},
+	Spec: v1.CustomResourceDefinitionSpec{
+		Group: "stable.example.com",
+		Names: v1.CustomResourceDefinitionNames{
+			Plural:     "coolfoos",
+			Singular:   "coolfoo",
+			ShortNames: []string{"foo"},
+			Kind:       "CoolFoo",
+			ListKind:   "CoolFooList",
+			Categories: []string{"cool"},
+		},
+		Scope: v1.ClusterScoped,
+		Versions: []v1.CustomResourceDefinitionVersion{
+			{
+				Name:       "v1",
+				Served:     true,
+				Storage:    true,
+				Deprecated: false,
+				Subresources: &v1.CustomResourceSubresources{
+					// This CRD has a /status subresource
+					Status: &v1.CustomResourceSubresourceStatus{},
+				},
+				Schema: &v1.CustomResourceValidation{
+					// Unused by discovery
+					OpenAPIV3Schema: &v1.JSONSchemaProps{},
+				},
+			},
+		},
+		Conversion:            &v1.CustomResourceConversion{},
+		PreserveUnknownFields: false,
+	},
+	Status: v1.CustomResourceDefinitionStatus{
+		Conditions: []v1.CustomResourceDefinitionCondition{
+			{
+				Type:   v1.Established,
+				Status: v1.ConditionTrue,
+			},
+		},
+	},
+}
+
+var coolBarCRD = &v1.CustomResourceDefinition{
+	TypeMeta: metav1.TypeMeta{
+		APIVersion: "apiextensions.k8s.io/v1",
+		Kind:       "CustomResourceDefinition",
+	},
+	ObjectMeta: metav1.ObjectMeta{
+		Name: "coolbar.stable.example.com",
+	},
+	Spec: v1.CustomResourceDefinitionSpec{
+		Group: "stable.example.com",
+		Names: v1.CustomResourceDefinitionNames{
+			Plural:     "coolbars",
+			Singular:   "coolbar",
+			ShortNames: []string{"bar"},
+			Kind:       "CoolBar",
+			ListKind:   "CoolBarList",
+			Categories: []string{"cool"},
+		},
+		Scope: v1.ClusterScoped,
+		Versions: []v1.CustomResourceDefinitionVersion{
+			{
+				Name:       "v1",
+				Served:     true,
+				Storage:    true,
+				Deprecated: false,
+				Schema: &v1.CustomResourceValidation{
+					// Unused by discovery
+					OpenAPIV3Schema: &v1.JSONSchemaProps{},
+				},
+			},
+		},
+		Conversion:            &v1.CustomResourceConversion{},
+		PreserveUnknownFields: false,
+	},
+	Status: v1.CustomResourceDefinitionStatus{
+		Conditions: []v1.CustomResourceDefinitionCondition{
+			{
+				Type:   v1.Established,
+				Status: v1.ConditionTrue,
+			},
+		},
+	},
+}
+
+var coolFooDiscovery apidiscoveryv2beta1.APIVersionDiscovery = apidiscoveryv2beta1.APIVersionDiscovery{
+	Version: "v1",
+	Resources: []apidiscoveryv2beta1.APIResourceDiscovery{
+		{
+			Resource:         "coolfoos",
+			Scope:            apidiscoveryv2beta1.ScopeCluster,
+			SingularResource: "coolfoo",
+			Verbs:            []string{"delete", "deletecollection", "get", "list", "patch", "create", "update", "watch"},
+			ShortNames:       []string{"foo"},
+			Categories:       []string{"cool"},
+			ResponseKind: &metav1.GroupVersionKind{
+				Group:   "stable.example.com",
+				Version: "v1",
+				Kind:    "CoolFoo",
+			},
+			Subresources: []apidiscoveryv2beta1.APISubresourceDiscovery{
+				{
+					Subresource:   "status",
+					Verbs:         []string{"get", "patch", "update"},
+					AcceptedTypes: nil, // is this correct?
+					ResponseKind: &metav1.GroupVersionKind{
+						Group:   "stable.example.com",
+						Version: "v1",
+						Kind:    "CoolFoo",
+					},
+				},
+			},
+		},
+	},
+}
+
+var mergedDiscovery apidiscoveryv2beta1.APIVersionDiscovery = apidiscoveryv2beta1.APIVersionDiscovery{
+	Version: "v1",
+	Resources: []apidiscoveryv2beta1.APIResourceDiscovery{
+		{
+			Resource:         "coolbars",
+			Scope:            apidiscoveryv2beta1.ScopeCluster,
+			SingularResource: "coolbar",
+			Verbs:            []string{"delete", "deletecollection", "get", "list", "patch", "create", "update", "watch"},
+			ShortNames:       []string{"bar"},
+			Categories:       []string{"cool"},
+			ResponseKind: &metav1.GroupVersionKind{
+				Group:   "stable.example.com",
+				Version: "v1",
+				Kind:    "CoolBar",
+			},
+		}, {
+			Resource:         "coolfoos",
+			Scope:            apidiscoveryv2beta1.ScopeCluster,
+			SingularResource: "coolfoo",
+			Verbs:            []string{"delete", "deletecollection", "get", "list", "patch", "create", "update", "watch"},
+			ShortNames:       []string{"foo"},
+			Categories:       []string{"cool"},
+			ResponseKind: &metav1.GroupVersionKind{
+				Group:   "stable.example.com",
+				Version: "v1",
+				Kind:    "CoolFoo",
+			},
+			Subresources: []apidiscoveryv2beta1.APISubresourceDiscovery{
+				{
+					Subresource:   "status",
+					Verbs:         []string{"get", "patch", "update"},
+					AcceptedTypes: nil, // is this correct?
+					ResponseKind: &metav1.GroupVersionKind{
+						Group:   "stable.example.com",
+						Version: "v1",
+						Kind:    "CoolFoo",
+					},
+				},
+			},
+		},
+	},
+}
+
+func init() {
+	// Not testing against an apiserver, so just assume names are accepted
+	coolFooCRD.Status.AcceptedNames = coolFooCRD.Spec.Names
+	coolBarCRD.Status.AcceptedNames = coolBarCRD.Spec.Names
+}
+
+// Provides an apiextensions-apiserver client
+type testEnvironment struct {
+	clientset.Interface
+
+	// Discovery test details
+	versionDiscoveryHandler
+	groupDiscoveryHandler
+
+	aggregated.FakeResourceManager
+}
+
+func (env *testEnvironment) Start(ctx context.Context) {
+	discoverySyncedCh := make(chan struct{})
+
+	factory := externalversions.NewSharedInformerFactoryWithOptions(
+		env.Interface, 30*time.Second)
+
+	discoveryController := NewDiscoveryController(
+		factory.Apiextensions().V1().CustomResourceDefinitions(),
+		&env.versionDiscoveryHandler,
+		&env.groupDiscoveryHandler,
+		env.FakeResourceManager,
+	)
+
+	factory.Start(ctx.Done())
+	go discoveryController.Run(ctx.Done(), discoverySyncedCh)
+
+	select {
+	case <-discoverySyncedCh:
+	case <-ctx.Done():
+	}
+}
+
+func setup() *testEnvironment {
+	env := &testEnvironment{
+		Interface:           fake.NewSimpleClientset(),
+		FakeResourceManager: aggregated.NewFakeResourceManager(),
+		versionDiscoveryHandler: versionDiscoveryHandler{
+			discovery: make(map[schema.GroupVersion]*discovery.APIVersionHandler),
+		},
+		groupDiscoveryHandler: groupDiscoveryHandler{
+			discovery: make(map[string]*discovery.APIGroupHandler),
+		},
+	}
+
+	return env
+}
+
+func TestResourceManagerExistingCRD(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	env := setup()
+	_, err := env.Interface.
+		ApiextensionsV1().
+		CustomResourceDefinitions().
+		Create(
+			ctx,
+			coolFooCRD,
+			metav1.CreateOptions{
+				FieldManager: "resource-manager-test",
+			},
+		)
+
+	require.NoError(t, err)
+
+	env.FakeResourceManager.Expect().
+		AddGroupVersion(coolFooCRD.Spec.Group, coolFooDiscovery)
+	env.FakeResourceManager.Expect().
+		SetGroupPriority(coolFooCRD.Spec.Group, 1000)
+
+	env.FakeResourceManager.Expect().
+		AddGroupVersion(coolFooCRD.Spec.Group, coolFooDiscovery)
+	env.FakeResourceManager.Expect().
+		SetGroupPriority(coolFooCRD.Spec.Group, 1000)
+
+	env.Start(ctx)
+	err = env.FakeResourceManager.WaitForActions(ctx, 1*time.Second)
+	require.NoError(t, err)
+}
+
+// Tests that if a CRD is added a runtime, the discovery controller will
+// put its information in the discovery document
+func TestResourceManagerAddedCRD(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	env := setup()
+	env.FakeResourceManager.Expect().
+		AddGroupVersion(coolFooCRD.Spec.Group, coolFooDiscovery)
+	env.FakeResourceManager.Expect().SetGroupPriority(coolFooCRD.Spec.Group, 1000)
+
+	env.Start(ctx)
+
+	// Create CRD after the controller has already started
+	_, err := env.Interface.
+		ApiextensionsV1().
+		CustomResourceDefinitions().
+		Create(
+			ctx,
+			coolFooCRD,
+			metav1.CreateOptions{
+				FieldManager: "resource-manager-test",
+			},
+		)
+
+	require.NoError(t, err)
+
+	err = env.FakeResourceManager.WaitForActions(ctx, 1*time.Second)
+	require.NoError(t, err)
+}
+
+// Test that having multiple CRDs in the same version will add both
+// versions to discovery.
+func TestMultipleCRDSameVersion(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	env := setup()
+	env.Start(ctx)
+
+	_, err := env.Interface.
+		ApiextensionsV1().
+		CustomResourceDefinitions().
+		Create(
+			ctx,
+			coolFooCRD,
+			metav1.CreateOptions{
+				FieldManager: "resource-manager-test",
+			},
+		)
+
+	require.NoError(t, err)
+	env.FakeResourceManager.Expect().
+		AddGroupVersion(coolFooCRD.Spec.Group, coolFooDiscovery)
+	env.FakeResourceManager.Expect().SetGroupPriority(coolFooCRD.Spec.Group, 1000)
+	err = env.FakeResourceManager.WaitForActions(ctx, 1*time.Second)
+	require.NoError(t, err)
+
+	_, err = env.Interface.
+		ApiextensionsV1().
+		CustomResourceDefinitions().
+		Create(
+			ctx,
+			coolBarCRD,
+			metav1.CreateOptions{
+				FieldManager: "resource-manager-test",
+			},
+		)
+	require.NoError(t, err)
+
+	env.FakeResourceManager.Expect().
+		AddGroupVersion(coolFooCRD.Spec.Group, mergedDiscovery)
+	env.FakeResourceManager.Expect().SetGroupPriority(coolFooCRD.Spec.Group, 1000)
+	err = env.FakeResourceManager.WaitForActions(ctx, 1*time.Second)
+	require.NoError(t, err)
+}
+
+// Tests that if a CRD is deleted at runtime, the discovery controller will
+// remove its information from its ResourceManager
+func TestDiscoveryControllerResourceManagerRemovedCRD(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	env := setup()
+	env.Start(ctx)
+
+	// Create CRD after the controller has already started
+	_, err := env.Interface.
+		ApiextensionsV1().
+		CustomResourceDefinitions().
+		Create(
+			ctx,
+			coolFooCRD,
+			metav1.CreateOptions{},
+		)
+
+	require.NoError(t, err)
+
+	// Wait for the Controller to pick up the Create event and add it to the
+	// Resource Manager
+	env.FakeResourceManager.Expect().
+		AddGroupVersion(coolFooCRD.Spec.Group, coolFooDiscovery)
+	env.FakeResourceManager.Expect().SetGroupPriority(coolFooCRD.Spec.Group, 1000)
+	err = env.FakeResourceManager.WaitForActions(ctx, 1*time.Second)
+	require.NoError(t, err)
+
+	err = env.Interface.
+		ApiextensionsV1().
+		CustomResourceDefinitions().
+		Delete(ctx, coolFooCRD.Name, metav1.DeleteOptions{})
+
+	require.NoError(t, err)
+
+	// Wait for the Controller to detect there are no more CRDs of this group
+	// and remove the entire group
+	env.FakeResourceManager.Expect().RemoveGroup(coolFooCRD.Spec.Group)
+
+	err = env.FakeResourceManager.WaitForActions(ctx, 1*time.Second)
+	require.NoError(t, err)
+}