From 93caf125b99647596fccb579b55e5498fb5b37b5 Mon Sep 17 00:00:00 2001 From: Alexander Zielenski Date: Thu, 10 Nov 2022 13:20:10 -0800 Subject: [PATCH] add discovery integration tests add crd integration tests \ --- .../apiserver/discovery/discovery_test.go | 747 ++++++++++++------ .../apiserver/discovery/framework.go | 478 +++++++++++ .../apiserver/discovery/service.go | 25 +- 3 files changed, 991 insertions(+), 259 deletions(-) create mode 100644 test/integration/apiserver/discovery/framework.go diff --git a/test/integration/apiserver/discovery/discovery_test.go b/test/integration/apiserver/discovery/discovery_test.go index 51604a105a0..c6b7caaa477 100644 --- a/test/integration/apiserver/discovery/discovery_test.go +++ b/test/integration/apiserver/discovery/discovery_test.go @@ -17,28 +17,25 @@ limitations under the License. package discovery import ( - "bytes" "context" - "encoding/json" "errors" + "fmt" "net/http" "reflect" "strings" "testing" - "time" + "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/require" apidiscoveryv2beta1 "k8s.io/api/apidiscovery/v2beta1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer" - "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/apimachinery/pkg/util/wait" discoveryendpoint "k8s.io/apiserver/pkg/endpoints/discovery/aggregated" genericfeatures "k8s.io/apiserver/pkg/features" utilfeature "k8s.io/apiserver/pkg/util/feature" @@ -66,6 +63,8 @@ type testClientSet struct { dynamicClientset } +var _ testClient = testClientSet{} + func (t testClientSet) Discovery() discovery.DiscoveryInterface { return t.kubeClientSet.Discovery() } @@ -94,6 +93,13 @@ var ( }, }, } + + stableGroup = "stable.example.com" + stableV1 = metav1.GroupVersion{Group: stableGroup, Version: "v1"} + stableV1alpha1 = metav1.GroupVersion{Group: stableGroup, Version: "v1alpha1"} + stableV1alpha2 = metav1.GroupVersion{Group: stableGroup, Version: "v1alpha2"} + stableV1beta1 = metav1.GroupVersion{Group: stableGroup, Version: "v1beta1"} + stableV2 = metav1.GroupVersion{Group: stableGroup, Version: "v2"} ) func init() { @@ -178,68 +184,6 @@ func unregisterAPIService(ctx context.Context, client aggregator.Interface, gv m return client.ApiregistrationV1().APIServices().Delete(ctx, gv.Version+"."+gv.Group, metav1.DeleteOptions{}) } -func WaitForGroupsAbsent(ctx context.Context, client testClientSet, groups ...string) error { - return WaitForResultWithCondition(ctx, client, func(groupList apidiscoveryv2beta1.APIGroupDiscoveryList) bool { - for _, searchGroup := range groups { - for _, docGroup := range groupList.Items { - if docGroup.Name == searchGroup { - return false - } - } - } - return true - }) - -} - -func WaitForGroups(ctx context.Context, client testClientSet, groups ...apidiscoveryv2beta1.APIGroupDiscovery) error { - return WaitForResultWithCondition(ctx, client, func(groupList apidiscoveryv2beta1.APIGroupDiscoveryList) bool { - for _, searchGroup := range groups { - for _, docGroup := range groupList.Items { - if reflect.DeepEqual(searchGroup, docGroup) { - return true - } - } - } - return false - }) -} - -func WaitForResultWithCondition(ctx context.Context, client testClientSet, condition func(result apidiscoveryv2beta1.APIGroupDiscoveryList) bool) error { - // Keep repeatedly fetching document from aggregator. - // Check to see if it contains our service within a reasonable amount of time - return wait.PollWithContext( - ctx, - 250*time.Millisecond, - 1*time.Second, - func(ctx context.Context) (done bool, err error) { - result, err := client. - Discovery(). - RESTClient(). - Get(). - AbsPath("/apis"). - SetHeader("Accept", "application/json;g=apidiscovery.k8s.io;v=v2beta1;as=APIGroupDiscoveryList"). - Do(ctx). - Raw() - - if err != nil { - return false, err - } - - groupList := apidiscoveryv2beta1.APIGroupDiscoveryList{} - err = json.Unmarshal(result, &groupList) - if err != nil { - panic(err) - } - - if condition(groupList) { - return true, nil - } - - return false, nil - }) -} - func TestAggregatedAPIServiceDiscovery(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.AggregatedDiscoveryEndpoint, true)() @@ -264,7 +208,10 @@ func TestAggregatedAPIServiceDiscovery(t *testing.T) { w.WriteHeader(http.StatusNotFound) } })) - service.Start(t, ctx) + go func() { + require.NoError(t, service.Run(ctx)) + }() + require.NoError(t, service.WaitForReady(ctx)) // For each groupversion served by our resourcemanager, create an APIService // object connected to our fake APIServer @@ -285,188 +232,498 @@ func TestAggregatedAPIServiceDiscovery(t *testing.T) { require.NoError(t, WaitForGroups(ctx, client, basicTestGroup)) } -// Shows that the following sequence is handled correctly: -// 1. Create an APIService -// - Check that API service is in discovery doc -// 2. Create CRD with the same GroupVersion as APIService -// 3. Delete APIService -// - Check that API service is removed from discovery -// 4. Update CRD -// - Check that CRD is in discovery document -func TestOverlappingCRDAndAPIService(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.AggregatedDiscoveryEndpoint, true)() +func runTestCases(t *testing.T, cases []testCase) { // Keep any goroutines spawned from running past the execution of this test ctx, client, cleanup := setup(t) defer cleanup() - // Create a resource manager whichs serves our GroupVersion - resourceManager := discoveryendpoint.NewResourceManager() - resourceManager.SetGroups([]apidiscoveryv2beta1.APIGroupDiscovery{basicTestGroup}) - - // Install our ResourceManager as an Aggregated APIService to the - // test server - service := NewFakeService("test-server", client, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/apis" { - resourceManager.ServeHTTP(w, r) - } else if strings.HasPrefix(r.URL.Path, "/apis/") { - // Return "valid" response so APIService can be marked as "available" - w.WriteHeader(http.StatusOK) - } else { - // reject openapi/v2, openapi/v3, apis// - w.WriteHeader(http.StatusNotFound) - } - })) - service.Start(t, ctx) - - // For each groupversion served by our resourcemanager, create an APIService - // object connected to our fake APIServer - for _, versionInfo := range basicTestGroup.Versions { - groupVersion := metav1.GroupVersion{ - Group: basicTestGroup.Name, - Version: versionInfo.Version, - } - - registerAPIService(ctx, client, groupVersion, service) - } - - // Keep repeatedly fetching document from aggregator. - // Check to see if it contains our service within a reasonable amount of time - require.NoError(t, WaitForGroups(ctx, client, basicTestGroup)) - - // Create a CRD - crd, err := client.ApiextensionsV1().CustomResourceDefinitions().Create(ctx, &apiextensionsv1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foos.stable.example.com", - }, - Spec: apiextensionsv1.CustomResourceDefinitionSpec{ - Group: "stable.example.com", - Names: apiextensionsv1.CustomResourceDefinitionNames{ - Singular: "foo", - Plural: "foos", - Kind: "Foo", - }, - Scope: apiextensionsv1.ClusterScoped, - Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ - { - Name: "v1", - Served: true, - Storage: true, - Schema: &apiextensionsv1.CustomResourceValidation{ - OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ - Type: "object", - Properties: map[string]apiextensionsv1.JSONSchemaProps{ - "stringMap": { - Description: "a map[string]string", - Type: "object", - AdditionalProperties: &apiextensionsv1.JSONSchemaPropsOrBool{ - Schema: &apiextensionsv1.JSONSchemaProps{ - Type: "string", - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, metav1.CreateOptions{ - FieldManager: "test-manager", - }) + // Fetch the original discovery information so we can wait for it to + // reset between tests + originalV1, err := FetchV1DiscoveryGroups(ctx, client) require.NoError(t, err) - // Create a CR for the CRD - // Keep trying until it succeeds (or should we try for discovery?) - require.NoError(t, wait.PollWithContext(ctx, 100*time.Millisecond, 1*time.Second, func(ctx context.Context) (done bool, err error) { - toCreate := &unstructured.Unstructured{} - toCreate.SetUnstructuredContent(map[string]any{ - "apiVersion": "stable.example.com/v1", - "kind": "Foo", - "key": "value", + originalV2, err := FetchV2Discovery(ctx, client) + require.NoError(t, err) + + for _, c := range cases { + t.Run(c.Name, func(t *testing.T) { + func() { + for _, a := range c.Actions { + if cleaning, ok := a.(cleaningAction); ok { + defer func() { + require.NoError(t, cleaning.Cleanup(ctx, client)) + }() + } + require.NoError(t, a.Do(ctx, client)) + } + }() + + var diff string + err := WaitForV1GroupsWithCondition(ctx, client, func(result metav1.APIGroupList) bool { + diff = cmp.Diff(originalV1, result) + return reflect.DeepEqual(result, originalV1) + }) + require.NoError(t, err, "v1 discovery must reset between tests: "+diff) + + err = WaitForResultWithCondition(ctx, client, func(result apidiscoveryv2beta1.APIGroupDiscoveryList) bool { + diff = cmp.Diff(originalV2, result) + return reflect.DeepEqual(result, originalV2) + }) + require.NoError(t, err, "v2 discovery must reset between tests: "+diff) }) - - _, err = client.dynamicClientset.Resource(schema.GroupVersionResource{ - Group: "stable.example.com", - Version: "v1", - Resource: "foos", - }).Create(ctx, toCreate, metav1.CreateOptions{ - FieldManager: "test-manager", - }) - return err != nil, nil - })) - - // For each groupversion served by our resourcemanager, delete an APIService - // object connected to our fake APIServer - for _, versionInfo := range basicTestGroup.Versions { - groupVersion := metav1.GroupVersion{ - Group: basicTestGroup.Name, - Version: versionInfo.Version, - } - - unregisterAPIService(ctx, client, groupVersion) } - - // Wait for the apiservice to be deleted from discovery - require.NoError(t, WaitForGroupsAbsent(ctx, client, "stable.example.com")) - - // Update the CRD with a minor change to show that reconciliation will - // eventually refresh the discovery group on resync - obj := &unstructured.Unstructured{} - obj.SetUnstructuredContent(map[string]interface{}{ - "apiVersion": "apiextensions.k8s.io/v1", - "kind": "CustomResourceDefinition", - "metadata": map[string]any{ - "name": crd.Name, - }, - "spec": map[string]interface{}{ - "names": map[string]any{ - "categories": []string{"all"}, - }, - }, - }) - - buf := bytes.NewBuffer(nil) - err = unstructured.UnstructuredJSONScheme.Encode(obj, buf) - require.NoError(t, err) - - //Is there a better way to force crd resync? - _, err = client.ApiextensionsV1().CustomResourceDefinitions().Patch( - ctx, - crd.Name, - types.ApplyPatchType, - buf.Bytes(), - metav1.PatchOptions{ - FieldManager: "test-manager", - }, - ) - require.NoError(t, err) - - // Wait until the crd appears in discovery - expectedDiscovery := apidiscoveryv2beta1.APIGroupDiscovery{ - ObjectMeta: metav1.ObjectMeta{ - Name: basicTestGroup.Name, - }, - Versions: []apidiscoveryv2beta1.APIVersionDiscovery{ - { - Version: "v1", - Resources: []apidiscoveryv2beta1.APIResourceDiscovery{ - { - Resource: "foos", - ResponseKind: &metav1.GroupVersionKind{ - Group: basicTestGroup.Name, - Version: "v1", - Kind: "Foo", - }, - Scope: apidiscoveryv2beta1.ScopeCluster, - SingularResource: crd.Spec.Names.Singular, - Verbs: []string{"delete", "deletecollection", "get", "list", "patch", "create", "update", "watch"}, - Categories: []string{"all"}, - }, - }, - //!TODO: set freshness of builtin/crds - Freshness: "", - }, - }, - } - require.NoError(t, WaitForGroups(ctx, client, expectedDiscovery)) +} + +// Declarative tests targeting CRD integration +func TestCRD(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.AggregatedDiscoveryEndpoint, true)() + + runTestCases(t, []testCase{ + { + // Show that when a CRD is added it gets included on the discovery doc + // within a reasonable amount of time + Name: "CRDInclusion", + Actions: []testAction{ + applyCRD(makeCRDSpec(stableGroup, "Foo", false, []string{"v1", "v1alpha1", "v1beta1", "v2"})), + waitForGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}), + waitForGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}), + }, + }, + { + // Show that a CRD added to the discovery doc can also be removed + Name: "CRDRemoval", + Actions: []testAction{ + applyCRD(makeCRDSpec(stableGroup, "Foo", false, []string{"v1", "v1alpha1", "v1beta1", "v2"})), + waitForGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}), + waitForGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}), + deleteObject{ + GroupVersionResource: metav1.GroupVersionResource(apiextensionsv1.SchemeGroupVersion.WithResource("customresourcedefinitions")), + Name: "foos.stable.example.com", + }, + waitForAbsentGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}), + waitForAbsentGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}), + }, + }, + { + // Show that if CRD and APIService share a groupversion, and the + // APIService is deleted, and CRD updated, the APIService remains in + // discovery. + // This test simulates a resync of CRD controler to show that eventually + // APIService is recreated + Name: "CRDAPIServiceOverlap", + Actions: []testAction{ + applyAPIService( + apiregistrationv1.APIServiceSpec{ + Group: stableGroup, + Version: "v1", + InsecureSkipTLSVerify: true, + GroupPriorityMinimum: int32(1000), + VersionPriority: int32(15), + Service: &apiregistrationv1.ServiceReference{ + Name: "unused", + Namespace: "default", + }, + }, + ), + + // Wait for GV to appear in both discovery documents + waitForGroupVersionsV1([]metav1.GroupVersion{stableV1}), + waitForGroupVersionsV2([]metav1.GroupVersion{stableV1}), + + applyCRD(makeCRDSpec(stableGroup, "Bar", false, []string{"v1", "v2"})), + + // only CRD has stable v2, this will show that CRD has been synced + waitForGroupVersionsV1([]metav1.GroupVersion{stableV2}), + waitForGroupVersionsV2([]metav1.GroupVersion{stableV2}), + + // Delete APIService shared by the aggregated apiservice and + // CRD + deleteObject{ + GroupVersionResource: metav1.GroupVersionResource(apiregistrationv1.SchemeGroupVersion.WithResource("apiservices")), + Name: "v1.stable.example.com", + }, + + // Update CRD to trigger a resync by adding a category and new groupversion + applyCRD(makeCRDSpec(stableGroup, "Bar", false, []string{"v1", "v2", "v1alpha1"}, "all")), + + // Show that the groupversion is re-added back + waitForGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV2, stableV1alpha1}), + waitForGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV2, stableV1alpha1}), + }, + }, + }) +} + +func TestFreshness(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.AggregatedDiscoveryEndpoint, true)() + + requireStaleGVs := func(gvs ...metav1.GroupVersion) inlineAction { + return inlineAction(func(ctx context.Context, client testClient) error { + document, err := FetchV2Discovery(ctx, client) + if err != nil { + return nil + } + + // Track the stale gvs in array for nice diff output upon test failure + staleGVs := []metav1.GroupVersion{} + + // Iterate through input so order does not matter + for _, targetGv := range gvs { + entry := FindGroupVersionV2(document, targetGv) + if entry == nil { + continue + } + + switch entry.Freshness { + case apidiscoveryv2beta1.DiscoveryFreshnessCurrent: + // Skip + case apidiscoveryv2beta1.DiscoveryFreshnessStale: + staleGVs = append(staleGVs, targetGv) + default: + return fmt.Errorf("unrecognized freshness '%v' on gv '%v'", entry.Freshness, targetGv) + } + } + + if !(len(staleGVs) == 0 && len(gvs) == 0) && !reflect.DeepEqual(staleGVs, gvs) { + diff := cmp.Diff(staleGVs, gvs) + return fmt.Errorf("expected sets of stale gvs to be equal:\n%v", diff) + } + + return nil + }) + } + + runTestCases(t, []testCase{ + { + Name: "BuiltinsFresh", + Actions: []testAction{ + // Wait for discovery ready + waitForGroupVersionsV2{metav1.GroupVersion(apiregistrationv1.SchemeGroupVersion)}, + // Require there are no stale groupversions and no unrecognized + // GVs + requireStaleGVs(), + }, + }, + { + // CRD freshness is always current + Name: "CRDFresh", + Actions: []testAction{ + // Add a CRD and wait for it to appear in discovery + applyCRD(makeCRDSpec(stableGroup, "Foo", false, []string{"v1", "v1alpha1", "v1beta1", "v2"})), + waitForGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}), + waitForGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}), + + // Test CRD is current by requiring there is nothing stale + requireStaleGVs(), + }, + }, + { + // Make an aggregated APIService that's unreachable and show + // that its groupversion is included in the discovery document as + // stale + Name: "AggregatedUnreachable", + Actions: []testAction{ + applyAPIService{ + Group: stableGroup, + Version: "v1", + GroupPriorityMinimum: 1000, + VersionPriority: 15, + Service: &apiregistrationv1.ServiceReference{ + Name: "doesnt-exist", + Namespace: "default", + }, + }, + waitForGroupVersionsV2([]metav1.GroupVersion{stableV1}), + // Require there is one and only one stale GV and it is stableV1 + requireStaleGVs(stableV1), + }, + }, + }) + +} + +// Shows a group for which multiple APIServices specify a GroupPriorityMinimum, +// it is sorted the same in both versions of discovery +func TestGroupPriorty(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.AggregatedDiscoveryEndpoint, true)() + + makeApiServiceSpec := func(gv metav1.GroupVersion, groupPriorityMin, versionPriority int) apiregistrationv1.APIServiceSpec { + return apiregistrationv1.APIServiceSpec{ + Group: gv.Group, + Version: gv.Version, + InsecureSkipTLSVerify: true, + GroupPriorityMinimum: int32(groupPriorityMin), + VersionPriority: int32(versionPriority), + Service: &apiregistrationv1.ServiceReference{ + Name: "unused", + Namespace: "default", + }, + } + } + + checkGVOrder := inlineAction(func(ctx context.Context, client testClient) (err error) { + // Fetch v1 document and v2 document, and ensure they have + // equal orderings of groupversions. and nothing missing or + // extra. + v1GroupsAndVersions, err := FetchV1DiscoveryGroups(ctx, client) + if err != nil { + return err + } + v2GroupsAndVersions, err := FetchV2Discovery(ctx, client) + if err != nil { + return err + } + + v1Gvs := []metav1.GroupVersion{} + v2Gvs := []metav1.GroupVersion{} + + for _, group := range v1GroupsAndVersions.Groups { + for _, version := range group.Versions { + v1Gvs = append(v1Gvs, metav1.GroupVersion{ + Group: group.Name, + Version: version.Version, + }) + } + } + + for _, group := range v2GroupsAndVersions.Items { + for _, version := range group.Versions { + v2Gvs = append(v2Gvs, metav1.GroupVersion{ + Group: group.Name, + Version: version.Version, + }) + } + } + + if !reflect.DeepEqual(v1Gvs, v2Gvs) { + return fmt.Errorf("expected equal orderings and lists of groupversions in both v1 and v2 discovery:\n%v", cmp.Diff(v1Gvs, v2Gvs)) + } + + return nil + }) + + runTestCases(t, []testCase{ + { + // Show that the legacy and aggregated discovery docs have the same + // set of builtin groupversions + Name: "BuiltinsAndOrdering", + Actions: []testAction{ + waitForGroupVersionsV1{metav1.GroupVersion(apiregistrationv1.SchemeGroupVersion)}, + waitForGroupVersionsV2{metav1.GroupVersion(apiregistrationv1.SchemeGroupVersion)}, + checkGVOrder, + }, + }, + { + // Show that a very high priority group is sorted first (below apiregistration v1) + // Also show the ordering is same for both v1 and v2 discovery apis + // Does not vary version priority + Name: "HighGroupPriority", + Actions: []testAction{ + // A VERY high priority which should take precedence + // 20000 is highest possible priority + applyAPIService(makeApiServiceSpec(stableV1, 20000, 15)), + // A VERY low priority which should be ignored + applyAPIService(makeApiServiceSpec(stableV1alpha1, 1, 15)), + // A medium-high priority (that conflicts with k8s) which should be ignored + applyAPIService(makeApiServiceSpec(stableV1alpha2, 17300, 15)), + // Wait for all the added group-versions to appear in both discovery documents + waitForGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1alpha2}), + waitForGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1alpha2}), + // Check that both v1 and v2 endpoints have exactly the same + // sets of groupversions + checkGVOrder, + // Check that the first group-version is the one with the highest + // priority + inlineAction(func(ctx context.Context, client testClient) error { + v2GroupsAndVersions, err := FetchV2Discovery(ctx, client) + if err != nil { + return err + } + + // First group should always be apiregistration.k8s.io + secondGV := metav1.GroupVersion{ + Group: v2GroupsAndVersions.Items[1].Name, + Version: v2GroupsAndVersions.Items[1].Versions[0].Version, + } + + if !reflect.DeepEqual(&stableV1, &secondGV) { + return fmt.Errorf("expected second group's first version to be %v, not %v", stableV1, secondGV) + } + + return nil + }), + }, + }, + { + // Show that a very low group priority is ordered last + Name: "LowGroupPriority", + Actions: []testAction{ + // A minimal priority + applyAPIService(makeApiServiceSpec(stableV1alpha1, 1, 15)), + // Wait for all the added group-versions to appear in v2 discovery + waitForGroupVersionsV2([]metav1.GroupVersion{stableV1alpha1}), + // Check that the last group-version is the one with the lowest + // priority + inlineAction(func(ctx context.Context, client testClient) error { + v2GroupsAndVersions, err := FetchV2Discovery(ctx, client) + if err != nil { + return err + } + lastGroup := v2GroupsAndVersions.Items[len(v2GroupsAndVersions.Items)-1] + + lastGV := metav1.GroupVersion{ + Group: lastGroup.Name, + Version: lastGroup.Versions[0].Version, + } + + if !reflect.DeepEqual(&stableV1alpha1, &lastGV) { + return fmt.Errorf("expected last group to be %v, not %v", stableV1alpha1, lastGV) + } + + return nil + }), + // Wait for all the added group-versions to appear in both discovery documents + waitForGroupVersionsV1([]metav1.GroupVersion{stableV1alpha1}), + // Check that both v1 and v2 endpoints have exactly the same + // sets of groupversions + checkGVOrder, + }, + }, + { + // Show that versions within a group are sorted by priority + Name: "VersionPriority", + Actions: []testAction{ + applyAPIService(makeApiServiceSpec(stableV1, 1000, 2)), + applyAPIService(makeApiServiceSpec(stableV1alpha1, 1000, 1)), + applyAPIService(makeApiServiceSpec(stableV1alpha2, 1000, 3)), + // Wait for all the added group-versions to appear in both discovery documents + waitForGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1alpha2}), + waitForGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1alpha2}), + // Check that both v1 and v2 endpoints have exactly the same + // sets of groupversions + checkGVOrder, + inlineAction(func(ctx context.Context, client testClient) error { + // Find the entry for stable.example.com + // and show the versions are ordered how we expect + v2GroupsAndVersions, err := FetchV2Discovery(ctx, client) + if err != nil { + return err + } + + // Should be ordered last for this test + group := v2GroupsAndVersions.Items[len(v2GroupsAndVersions.Items)-1] + if group.Name != stableGroup { + return fmt.Errorf("group is not where we expect: found %v, expected %v", group.Name, stableGroup) + } + + versionOrder := []string{} + for _, version := range group.Versions { + versionOrder = append(versionOrder, version.Version) + } + + expectedOrder := []string{ + stableV1alpha2.Version, + stableV1.Version, + stableV1alpha1.Version, + } + + if !reflect.DeepEqual(expectedOrder, versionOrder) { + return fmt.Errorf("version in wrong order: %v", cmp.Diff(expectedOrder, versionOrder)) + } + + return nil + }), + }, + }, + { + // Show that versions within a group are sorted by priority + // and that equal versions will be sorted by a kube-aware version + // comparator + Name: "VersionPriorityTiebreaker", + Actions: []testAction{ + applyAPIService(makeApiServiceSpec(stableV1, 1000, 15)), + applyAPIService(makeApiServiceSpec(stableV1alpha1, 1000, 15)), + applyAPIService(makeApiServiceSpec(stableV1alpha2, 1000, 15)), + applyAPIService(makeApiServiceSpec(stableV1beta1, 1000, 15)), + applyAPIService(makeApiServiceSpec(stableV2, 1000, 15)), + // Wait for all the added group-versions to appear in both discovery documents + waitForGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1alpha2, stableV1beta1, stableV2}), + waitForGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1alpha2, stableV1beta1, stableV2}), + // Check that both v1 and v2 endpoints have exactly the same + // sets of groupversions + checkGVOrder, + inlineAction(func(ctx context.Context, client testClient) error { + // Find the entry for stable.example.com + // and show the versions are ordered how we expect + v2GroupsAndVersions, err := FetchV2Discovery(ctx, client) + if err != nil { + return err + } + + // Should be ordered last for this test + group := v2GroupsAndVersions.Items[len(v2GroupsAndVersions.Items)-1] + if group.Name != stableGroup { + return fmt.Errorf("group is not where we expect: found %v, expected %v", group.Name, stableGroup) + } + + versionOrder := []string{} + for _, version := range group.Versions { + versionOrder = append(versionOrder, version.Version) + } + + expectedOrder := []string{ + stableV2.Version, + stableV1.Version, + stableV1beta1.Version, + stableV1alpha2.Version, + stableV1alpha1.Version, + } + + if !reflect.DeepEqual(expectedOrder, versionOrder) { + return fmt.Errorf("version in wrong order: %v", cmp.Diff(expectedOrder, versionOrder)) + } + + return nil + }), + }, + }, + }) +} + +func makeCRDSpec(group string, kind string, namespaced bool, versions []string, categories ...string) apiextensionsv1.CustomResourceDefinitionSpec { + scope := apiextensionsv1.NamespaceScoped + if !namespaced { + scope = apiextensionsv1.ClusterScoped + } + + plural, singular := meta.UnsafeGuessKindToResource(schema.GroupVersionKind{Kind: kind}) + res := apiextensionsv1.CustomResourceDefinitionSpec{ + Group: group, + Scope: scope, + Names: apiextensionsv1.CustomResourceDefinitionNames{ + Plural: plural.Resource, + Singular: singular.Resource, + Kind: kind, + Categories: categories, + }, + } + + for i, version := range versions { + res.Versions = append(res.Versions, apiextensionsv1.CustomResourceDefinitionVersion{ + Name: version, + Served: true, + Storage: i == 0, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensionsv1.JSONSchemaProps{ + "data": { + Type: "string", + }, + }, + }, + }, + }) + } + return res } diff --git a/test/integration/apiserver/discovery/framework.go b/test/integration/apiserver/discovery/framework.go new file mode 100644 index 00000000000..8d624284eed --- /dev/null +++ b/test/integration/apiserver/discovery/framework.go @@ -0,0 +1,478 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package discovery + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "time" + + apiextensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + aggregator "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" + + apidiscoveryv2beta1 "k8s.io/api/apidiscovery/v2beta1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" +) + +const acceptV1JSON = "application/json" +const acceptV2JSON = "application/json;g=apidiscovery.k8s.io;v=v2beta1;as=APIGroupDiscoveryList" + +type testClient interface { + kubernetes.Interface + aggregator.Interface + apiextensions.Interface + dynamic.Interface +} + +// declarative framework for discovery integration tests +// each test has metadata and a list of actions which each must pass for the +// test to pass +type testCase struct { + Name string + Actions []testAction +} + +// interface defining a function that does something with the integration test +// api server and returns an error. the test fails if the error is non nil +type testAction interface { + Do(ctx context.Context, client testClient) error +} + +type cleaningAction interface { + testAction + Cleanup(ctx context.Context, client testClient) error +} + +// apply an apiservice to the cluster +type applyAPIService apiregistrationv1.APIServiceSpec + +type applyCRD apiextensionsv1.CustomResourceDefinitionSpec + +type deleteObject struct { + metav1.GroupVersionResource + Namespace string + Name string +} + +// Wait for groupversions to appear in v1 discovery +type waitForGroupVersionsV1 []metav1.GroupVersion + +// Wait for groupversions to disappear from v2 discovery +type waitForAbsentGroupVersionsV1 []metav1.GroupVersion + +// Wait for groupversions to appear in v2 discovery +type waitForGroupVersionsV2 []metav1.GroupVersion + +// Wait for groupversions to disappear from v2 discovery +type waitForAbsentGroupVersionsV2 []metav1.GroupVersion + +// Assert something about the current state of v2 discovery +type inlineAction func(ctx context.Context, client testClient) error + +func (a applyAPIService) Do(ctx context.Context, client testClient) error { + // using dynamic client since the typed client does not support `Apply` + // operation? + obj := &apiregistrationv1.APIService{ + ObjectMeta: metav1.ObjectMeta{ + Name: a.Version + "." + a.Group, + }, + Spec: apiregistrationv1.APIServiceSpec(a), + } + + unstructuredContent, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) + if err != nil { + return err + } + + unstructedObject := &unstructured.Unstructured{} + unstructedObject.SetUnstructuredContent(unstructuredContent) + unstructedObject.SetGroupVersionKind(apiregistrationv1.SchemeGroupVersion.WithKind("APIService")) + + _, err = client. + Resource(apiregistrationv1.SchemeGroupVersion.WithResource("apiservices")). + Apply(ctx, obj.Name, unstructedObject, metav1.ApplyOptions{ + FieldManager: "test-manager", + }) + + return err +} + +func (a applyAPIService) Cleanup(ctx context.Context, client testClient) error { + name := a.Version + "." + a.Group + err := client.ApiregistrationV1().APIServices().Delete(ctx, name, metav1.DeleteOptions{}) + + if !errors.IsNotFound(err) { + return err + } + + err = wait.PollWithContext( + ctx, + 250*time.Millisecond, + 1*time.Second, + func(ctx context.Context) (done bool, err error) { + _, err = client.ApiregistrationV1().APIServices().Get(ctx, name, metav1.GetOptions{}) + if err == nil { + return false, nil + } + + if !errors.IsNotFound(err) { + return false, err + } + return true, nil + }, + ) + + if err != nil { + return fmt.Errorf("error waiting for APIService %v to clean up: %w", name, err) + } + + return nil +} + +func (a applyCRD) Do(ctx context.Context, client testClient) error { + // using dynamic client since the typed client does not support `Apply` + // operation? + name := a.Names.Plural + "." + a.Group + obj := &apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: apiextensionsv1.CustomResourceDefinitionSpec(a), + } + + unstructuredContent, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) + if err != nil { + return err + } + + unstructedObject := &unstructured.Unstructured{} + unstructedObject.SetUnstructuredContent(unstructuredContent) + unstructedObject.SetGroupVersionKind(apiextensionsv1.SchemeGroupVersion.WithKind("CustomResourceDefinition")) + + _, err = client. + Resource(apiextensionsv1.SchemeGroupVersion.WithResource("customresourcedefinitions")). + Apply(ctx, obj.Name, unstructedObject, metav1.ApplyOptions{ + FieldManager: "test-manager", + }) + + return err +} + +func (a applyCRD) Cleanup(ctx context.Context, client testClient) error { + name := a.Names.Plural + "." + a.Group + err := client.ApiextensionsV1().CustomResourceDefinitions().Delete(ctx, name, metav1.DeleteOptions{}) + + if !errors.IsNotFound(err) { + return err + } + + err = wait.PollWithContext( + ctx, + 250*time.Millisecond, + 1*time.Second, + func(ctx context.Context) (done bool, err error) { + _, err = client.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, name, metav1.GetOptions{}) + if err == nil { + return false, nil + } + + if !errors.IsNotFound(err) { + return false, err + } + return true, nil + }, + ) + + if err != nil { + return fmt.Errorf("error waiting for CRD %v to clean up: %w", name, err) + } + + return nil +} + +func (d deleteObject) Do(ctx context.Context, client testClient) error { + if d.Namespace == "" { + return client.Resource(schema.GroupVersionResource(d.GroupVersionResource)). + Delete(ctx, d.Name, metav1.DeleteOptions{}) + } else { + return client.Resource(schema.GroupVersionResource(d.GroupVersionResource)). + Namespace(d.Namespace). + Delete(ctx, d.Name, metav1.DeleteOptions{}) + } +} + +func (w waitForGroupVersionsV2) Do(ctx context.Context, client testClient) error { + err := WaitForResultWithCondition(ctx, client, func(result apidiscoveryv2beta1.APIGroupDiscoveryList) bool { + for _, gv := range w { + if FindGroupVersionV2(result, gv) == nil { + return false + } + } + + return true + }) + + if err != nil { + return fmt.Errorf("waiting for groupversions v2 (%v): %w", w, err) + } + return nil +} + +func (w waitForAbsentGroupVersionsV2) Do(ctx context.Context, client testClient) error { + err := WaitForResultWithCondition(ctx, client, func(result apidiscoveryv2beta1.APIGroupDiscoveryList) bool { + for _, gv := range w { + if FindGroupVersionV2(result, gv) != nil { + return false + } + } + + return true + }) + + if err != nil { + return fmt.Errorf("waiting for absent groupversions v2 (%v): %w", w, err) + } + return nil +} + +func (w waitForGroupVersionsV1) Do(ctx context.Context, client testClient) error { + err := WaitForV1GroupsWithCondition(ctx, client, func(result metav1.APIGroupList) bool { + for _, gv := range w { + if !FindGroupVersionV1(result, gv) { + return false + } + } + + return true + }) + + if err != nil { + return fmt.Errorf("waiting for groupversions v1 (%v): %w", w, err) + } + return nil +} + +func (w waitForAbsentGroupVersionsV1) Do(ctx context.Context, client testClient) error { + err := WaitForV1GroupsWithCondition(ctx, client, func(result metav1.APIGroupList) bool { + for _, gv := range w { + if FindGroupVersionV1(result, gv) { + return false + } + } + + return true + }) + + if err != nil { + return fmt.Errorf("waiting for absent groupversions v1 (%v): %w", w, err) + } + return nil +} + +func (i inlineAction) Do(ctx context.Context, client testClient) error { + return i(ctx, client) +} + +func FetchV2Discovery(ctx context.Context, client testClient) (apidiscoveryv2beta1.APIGroupDiscoveryList, error) { + result, err := client. + Discovery(). + RESTClient(). + Get(). + AbsPath("/apis"). + SetHeader("Accept", acceptV2JSON). + Do(ctx). + Raw() + + if err != nil { + return apidiscoveryv2beta1.APIGroupDiscoveryList{}, fmt.Errorf("failed to fetch v2 discovery: %w", err) + } + + groupList := apidiscoveryv2beta1.APIGroupDiscoveryList{} + err = json.Unmarshal(result, &groupList) + if err != nil { + return apidiscoveryv2beta1.APIGroupDiscoveryList{}, fmt.Errorf("failed to parse v2 discovery: %w", err) + } + + return groupList, nil +} + +func FetchV1DiscoveryGroups(ctx context.Context, client testClient) (metav1.APIGroupList, error) { + return FetchV1DiscoveryGroupsAtPath(ctx, client, "/apis") +} + +func FetchV1DiscoveryLegacyGroups(ctx context.Context, client testClient) (metav1.APIGroupList, error) { + return FetchV1DiscoveryGroupsAtPath(ctx, client, "/api") +} + +func FetchV1DiscoveryGroupsAtPath(ctx context.Context, client testClient, path string) (metav1.APIGroupList, error) { + result, err := client. + Discovery(). + RESTClient(). + Get(). + AbsPath("/apis"). + SetHeader("Accept", acceptV1JSON). + Do(ctx). + Raw() + + if err != nil { + return metav1.APIGroupList{}, fmt.Errorf("failed to fetch v1 discovery at %v: %w", path, err) + } + + groupList := metav1.APIGroupList{} + err = json.Unmarshal(result, &groupList) + if err != nil { + return metav1.APIGroupList{}, fmt.Errorf("failed to parse v1 discovery at %v: %w", path, err) + } + + return groupList, nil +} + +func FetchV1DiscoveryResource(ctx context.Context, client testClient, gv metav1.GroupVersion) (metav1.APIGroupList, error) { + result, err := client. + Discovery(). + RESTClient(). + Get(). + AbsPath("/apis/"+gv.Group+"/"+gv.Version). + SetHeader("Accept", acceptV1JSON). + Do(ctx). + Raw() + + if err != nil { + return metav1.APIGroupList{}, err + } + + groupList := metav1.APIGroupList{} + err = json.Unmarshal(result, &groupList) + if err != nil { + return metav1.APIGroupList{}, err + } + + return groupList, nil +} + +func WaitForGroupsAbsent(ctx context.Context, client testClient, groups ...string) error { + return WaitForResultWithCondition(ctx, client, func(groupList apidiscoveryv2beta1.APIGroupDiscoveryList) bool { + for _, searchGroup := range groups { + for _, docGroup := range groupList.Items { + if docGroup.Name == searchGroup { + return false + } + } + } + return true + }) + +} + +func WaitForGroups(ctx context.Context, client testClient, groups ...apidiscoveryv2beta1.APIGroupDiscovery) error { + return WaitForResultWithCondition(ctx, client, func(groupList apidiscoveryv2beta1.APIGroupDiscoveryList) bool { + for _, searchGroup := range groups { + for _, docGroup := range groupList.Items { + if reflect.DeepEqual(searchGroup, docGroup) { + return true + } + } + } + return false + }) +} + +func WaitForResultWithCondition(ctx context.Context, client testClient, condition func(result apidiscoveryv2beta1.APIGroupDiscoveryList) bool) error { + // Keep repeatedly fetching document from aggregator. + // Check to see if it contains our service within a reasonable amount of time + return wait.PollWithContext( + ctx, + 250*time.Millisecond, + 1*time.Second, + func(ctx context.Context) (done bool, err error) { + groupList, err := FetchV2Discovery(ctx, client) + if err != nil { + return false, err + } + + if condition(groupList) { + return true, nil + } + + return false, nil + }) +} + +func WaitForV1GroupsWithCondition(ctx context.Context, client testClient, condition func(result metav1.APIGroupList) bool) error { + // Keep repeatedly fetching document from aggregator. + // Check to see if it contains our service within a reasonable amount of time + return wait.PollWithContext( + ctx, + 250*time.Millisecond, + 1*time.Second, + func(ctx context.Context) (done bool, err error) { + groupList, err := FetchV1DiscoveryGroups(ctx, client) + + if err != nil { + return false, err + } + + if condition(groupList) { + return true, nil + } + + return false, nil + }) +} + +func FindGroupVersionV1(discovery metav1.APIGroupList, gv metav1.GroupVersion) bool { + for _, documentGroup := range discovery.Groups { + if documentGroup.Name != gv.Group { + continue + } + + for _, documentVersion := range documentGroup.Versions { + if documentVersion.Version == gv.Version { + return true + } + } + } + + return false +} + +func FindGroupVersionV2(discovery apidiscoveryv2beta1.APIGroupDiscoveryList, gv metav1.GroupVersion) *apidiscoveryv2beta1.APIVersionDiscovery { + for _, documentGroup := range discovery.Items { + if documentGroup.Name != gv.Group { + continue + } + + for _, documentVersion := range documentGroup.Versions { + if documentVersion.Version == gv.Version { + return &documentVersion + } + } + } + + return nil +} diff --git a/test/integration/apiserver/discovery/service.go b/test/integration/apiserver/discovery/service.go index de3d1854943..c875f83a607 100644 --- a/test/integration/apiserver/discovery/service.go +++ b/test/integration/apiserver/discovery/service.go @@ -19,15 +19,14 @@ package discovery import ( "context" "errors" + "fmt" "net/http" "net/http/httptest" "net/url" "strconv" "sync" - "testing" "time" - "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" @@ -38,7 +37,7 @@ import ( ) type FakeService interface { - Start(t *testing.T, ctx context.Context) + Run(ctx context.Context) error Port() *int32 Name() string } @@ -62,7 +61,7 @@ func NewFakeService(name string, client kubernetes.Interface, handler http.Handl } } -func (f *fakeService) run(ctx context.Context) error { +func (f *fakeService) Run(ctx context.Context) error { aggregatedServer := httptest.NewUnstartedServer(f.handler) aggregatedServer.StartTLS() defer aggregatedServer.Close() @@ -111,26 +110,24 @@ func (f *fakeService) run(ctx context.Context) error { // Uninstall service from the cluser err = f.client.CoreV1().Services("default").Delete(ctx, service.Name, metav1.DeleteOptions{}) + if errors.Is(err, context.Canceled) { + err = nil + } return err } -func (f *fakeService) Start(t *testing.T, ctx context.Context) { - go func() { - err := f.run(ctx) - if errors.Is(err, context.Canceled) { - err = nil - } - require.NoError(t, err) - }() - +func (f *fakeService) WaitForReady(ctx context.Context) error { err := wait.PollWithContext(ctx, 1*time.Second, 200*time.Millisecond, func(ctx context.Context) (done bool, err error) { return f.Port() != nil, nil }) if errors.Is(err, context.Canceled) { err = nil + } else if err != nil { + err = fmt.Errorf("service should have come alive in a reasonable amount of time: %w", err) } - require.NoError(t, err, "service should have come alive in a reasonable amount of time") + + return err } func (f *fakeService) Port() *int32 {