add discovery integration tests

add crd integration tests
\

test/integration/apiserver/discovery/discovery_test.go

@@ -17,28 +17,25 @@ limitations under the License.
 package discovery
 
 import (
-	"bytes"
 	"context"
-	"encoding/json"
 	"errors"
+	"fmt"
 	"net/http"
 	"reflect"
 	"strings"
 	"testing"
-	"time"
 
+	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/require"
 	apidiscoveryv2beta1 "k8s.io/api/apidiscovery/v2beta1"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apiextensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer"
-	"k8s.io/apimachinery/pkg/types"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
-	"k8s.io/apimachinery/pkg/util/wait"
 	discoveryendpoint "k8s.io/apiserver/pkg/endpoints/discovery/aggregated"
 	genericfeatures "k8s.io/apiserver/pkg/features"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
@@ -66,6 +63,8 @@ type testClientSet struct {
 	dynamicClientset
 }
 
+var _ testClient = testClientSet{}
+
 func (t testClientSet) Discovery() discovery.DiscoveryInterface {
 	return t.kubeClientSet.Discovery()
 }
@@ -94,6 +93,13 @@ var (
 			},
 		},
 	}
+
+	stableGroup    = "stable.example.com"
+	stableV1       = metav1.GroupVersion{Group: stableGroup, Version: "v1"}
+	stableV1alpha1 = metav1.GroupVersion{Group: stableGroup, Version: "v1alpha1"}
+	stableV1alpha2 = metav1.GroupVersion{Group: stableGroup, Version: "v1alpha2"}
+	stableV1beta1  = metav1.GroupVersion{Group: stableGroup, Version: "v1beta1"}
+	stableV2       = metav1.GroupVersion{Group: stableGroup, Version: "v2"}
 )
 
 func init() {
@@ -178,68 +184,6 @@ func unregisterAPIService(ctx context.Context, client aggregator.Interface, gv m
 	return client.ApiregistrationV1().APIServices().Delete(ctx, gv.Version+"."+gv.Group, metav1.DeleteOptions{})
 }
 
-func WaitForGroupsAbsent(ctx context.Context, client testClientSet, groups ...string) error {
-	return WaitForResultWithCondition(ctx, client, func(groupList apidiscoveryv2beta1.APIGroupDiscoveryList) bool {
-		for _, searchGroup := range groups {
-			for _, docGroup := range groupList.Items {
-				if docGroup.Name == searchGroup {
-					return false
-				}
-			}
-		}
-		return true
-	})
-
-}
-
-func WaitForGroups(ctx context.Context, client testClientSet, groups ...apidiscoveryv2beta1.APIGroupDiscovery) error {
-	return WaitForResultWithCondition(ctx, client, func(groupList apidiscoveryv2beta1.APIGroupDiscoveryList) bool {
-		for _, searchGroup := range groups {
-			for _, docGroup := range groupList.Items {
-				if reflect.DeepEqual(searchGroup, docGroup) {
-					return true
-				}
-			}
-		}
-		return false
-	})
-}
-
-func WaitForResultWithCondition(ctx context.Context, client testClientSet, condition func(result apidiscoveryv2beta1.APIGroupDiscoveryList) bool) error {
-	// Keep repeatedly fetching document from aggregator.
-	// Check to see if it contains our service within a reasonable amount of time
-	return wait.PollWithContext(
-		ctx,
-		250*time.Millisecond,
-		1*time.Second,
-		func(ctx context.Context) (done bool, err error) {
-			result, err := client.
-				Discovery().
-				RESTClient().
-				Get().
-				AbsPath("/apis").
-				SetHeader("Accept", "application/json;g=apidiscovery.k8s.io;v=v2beta1;as=APIGroupDiscoveryList").
-				Do(ctx).
-				Raw()
-
-			if err != nil {
-				return false, err
-			}
-
-			groupList := apidiscoveryv2beta1.APIGroupDiscoveryList{}
-			err = json.Unmarshal(result, &groupList)
-			if err != nil {
-				panic(err)
-			}
-
-			if condition(groupList) {
-				return true, nil
-			}
-
-			return false, nil
-		})
-}
-
 func TestAggregatedAPIServiceDiscovery(t *testing.T) {
 	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.AggregatedDiscoveryEndpoint, true)()
 
@@ -264,7 +208,10 @@ func TestAggregatedAPIServiceDiscovery(t *testing.T) {
 			w.WriteHeader(http.StatusNotFound)
 		}
 	}))
-	service.Start(t, ctx)
+	go func() {
+		require.NoError(t, service.Run(ctx))
+	}()
+	require.NoError(t, service.WaitForReady(ctx))
 
 	// For each groupversion served by our resourcemanager, create an APIService
 	// object connected to our fake APIServer
@@ -285,188 +232,498 @@ func TestAggregatedAPIServiceDiscovery(t *testing.T) {
 	require.NoError(t, WaitForGroups(ctx, client, basicTestGroup))
 }
 
-// Shows that the following sequence is handled correctly:
+func runTestCases(t *testing.T, cases []testCase) {
-// 1. Create an APIService
-// - Check that API service is in discovery doc
-// 2. Create CRD with the same GroupVersion as APIService
-// 3. Delete APIService
-// - Check that API service is removed from discovery
-// 4. Update CRD
-// -  Check that CRD is in discovery document
-func TestOverlappingCRDAndAPIService(t *testing.T) {
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.AggregatedDiscoveryEndpoint, true)()
 	// Keep any goroutines spawned from running past the execution of this test
 	ctx, client, cleanup := setup(t)
 	defer cleanup()
 
-	// Create a resource manager whichs serves our GroupVersion
+	// Fetch the original discovery information so we can wait for it to
-	resourceManager := discoveryendpoint.NewResourceManager()
+	// reset between tests
-	resourceManager.SetGroups([]apidiscoveryv2beta1.APIGroupDiscovery{basicTestGroup})
+	originalV1, err := FetchV1DiscoveryGroups(ctx, client)
-
-	// Install our ResourceManager as an Aggregated APIService to the
-	// test server
-	service := NewFakeService("test-server", client, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/apis" {
-			resourceManager.ServeHTTP(w, r)
-		} else if strings.HasPrefix(r.URL.Path, "/apis/") {
-			// Return "valid" response so APIService can be marked as "available"
-			w.WriteHeader(http.StatusOK)
-		} else {
-			// reject openapi/v2, openapi/v3, apis/<group>/<version>
-			w.WriteHeader(http.StatusNotFound)
-		}
-	}))
-	service.Start(t, ctx)
-
-	// For each groupversion served by our resourcemanager, create an APIService
-	// object connected to our fake APIServer
-	for _, versionInfo := range basicTestGroup.Versions {
-		groupVersion := metav1.GroupVersion{
-			Group:   basicTestGroup.Name,
-			Version: versionInfo.Version,
-		}
-
-		registerAPIService(ctx, client, groupVersion, service)
-	}
-
-	// Keep repeatedly fetching document from aggregator.
-	// Check to see if it contains our service within a reasonable amount of time
-	require.NoError(t, WaitForGroups(ctx, client, basicTestGroup))
-
-	// Create a CRD
-	crd, err := client.ApiextensionsV1().CustomResourceDefinitions().Create(ctx, &apiextensionsv1.CustomResourceDefinition{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: "foos.stable.example.com",
-		},
-		Spec: apiextensionsv1.CustomResourceDefinitionSpec{
-			Group: "stable.example.com",
-			Names: apiextensionsv1.CustomResourceDefinitionNames{
-				Singular: "foo",
-				Plural:   "foos",
-				Kind:     "Foo",
-			},
-			Scope: apiextensionsv1.ClusterScoped,
-			Versions: []apiextensionsv1.CustomResourceDefinitionVersion{
-				{
-					Name:    "v1",
-					Served:  true,
-					Storage: true,
-					Schema: &apiextensionsv1.CustomResourceValidation{
-						OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
-							Type: "object",
-							Properties: map[string]apiextensionsv1.JSONSchemaProps{
-								"stringMap": {
-									Description: "a map[string]string",
-									Type:        "object",
-									AdditionalProperties: &apiextensionsv1.JSONSchemaPropsOrBool{
-										Schema: &apiextensionsv1.JSONSchemaProps{
-											Type: "string",
-										},
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-		},
-	}, metav1.CreateOptions{
-		FieldManager: "test-manager",
-	})
 	require.NoError(t, err)
 
-	// Create a CR for the CRD
+	originalV2, err := FetchV2Discovery(ctx, client)
-	// Keep trying until it succeeds (or should we try for discovery?)
+	require.NoError(t, err)
-	require.NoError(t, wait.PollWithContext(ctx, 100*time.Millisecond, 1*time.Second, func(ctx context.Context) (done bool, err error) {
+
-		toCreate := &unstructured.Unstructured{}
+	for _, c := range cases {
-		toCreate.SetUnstructuredContent(map[string]any{
+		t.Run(c.Name, func(t *testing.T) {
-			"apiVersion": "stable.example.com/v1",
+			func() {
-			"kind":       "Foo",
+				for _, a := range c.Actions {
-			"key":        "value",
+					if cleaning, ok := a.(cleaningAction); ok {
+						defer func() {
+							require.NoError(t, cleaning.Cleanup(ctx, client))
+						}()
+					}
+					require.NoError(t, a.Do(ctx, client))
+				}
+			}()
+
+			var diff string
+			err := WaitForV1GroupsWithCondition(ctx, client, func(result metav1.APIGroupList) bool {
+				diff = cmp.Diff(originalV1, result)
+				return reflect.DeepEqual(result, originalV1)
+			})
+			require.NoError(t, err, "v1 discovery must reset between tests: "+diff)
+
+			err = WaitForResultWithCondition(ctx, client, func(result apidiscoveryv2beta1.APIGroupDiscoveryList) bool {
+				diff = cmp.Diff(originalV2, result)
+				return reflect.DeepEqual(result, originalV2)
+			})
+			require.NoError(t, err, "v2 discovery must reset between tests: "+diff)
 		})
-
-		_, err = client.dynamicClientset.Resource(schema.GroupVersionResource{
-			Group:    "stable.example.com",
-			Version:  "v1",
-			Resource: "foos",
-		}).Create(ctx, toCreate, metav1.CreateOptions{
-			FieldManager: "test-manager",
-		})
-		return err != nil, nil
-	}))
-
-	// For each groupversion served by our resourcemanager, delete an APIService
-	// object connected to our fake APIServer
-	for _, versionInfo := range basicTestGroup.Versions {
-		groupVersion := metav1.GroupVersion{
-			Group:   basicTestGroup.Name,
-			Version: versionInfo.Version,
-		}
-
-		unregisterAPIService(ctx, client, groupVersion)
 	}
-
+}
-	// Wait for the apiservice to be deleted from discovery
+
-	require.NoError(t, WaitForGroupsAbsent(ctx, client, "stable.example.com"))
+// Declarative tests targeting CRD integration
-
+func TestCRD(t *testing.T) {
-	// Update the CRD with a minor change to show that reconciliation will
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.AggregatedDiscoveryEndpoint, true)()
-	// eventually refresh the discovery group on resync
+
-	obj := &unstructured.Unstructured{}
+	runTestCases(t, []testCase{
-	obj.SetUnstructuredContent(map[string]interface{}{
+		{
-		"apiVersion": "apiextensions.k8s.io/v1",
+			// Show that when a CRD is added it gets included on the discovery doc
-		"kind":       "CustomResourceDefinition",
+			// within a reasonable amount of time
-		"metadata": map[string]any{
+			Name: "CRDInclusion",
-			"name": crd.Name,
+			Actions: []testAction{
-		},
+				applyCRD(makeCRDSpec(stableGroup, "Foo", false, []string{"v1", "v1alpha1", "v1beta1", "v2"})),
-		"spec": map[string]interface{}{
+				waitForGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}),
-			"names": map[string]any{
+				waitForGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}),
-				"categories": []string{"all"},
+			},
-			},
+		},
-		},
+		{
-	})
+			// Show that a CRD added to the discovery doc can also be removed
-
+			Name: "CRDRemoval",
-	buf := bytes.NewBuffer(nil)
+			Actions: []testAction{
-	err = unstructured.UnstructuredJSONScheme.Encode(obj, buf)
+				applyCRD(makeCRDSpec(stableGroup, "Foo", false, []string{"v1", "v1alpha1", "v1beta1", "v2"})),
-	require.NoError(t, err)
+				waitForGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}),
-
+				waitForGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}),
-	//Is there a better way to force crd resync?
+				deleteObject{
-	_, err = client.ApiextensionsV1().CustomResourceDefinitions().Patch(
+					GroupVersionResource: metav1.GroupVersionResource(apiextensionsv1.SchemeGroupVersion.WithResource("customresourcedefinitions")),
-		ctx,
+					Name:                 "foos.stable.example.com",
-		crd.Name,
+				},
-		types.ApplyPatchType,
+				waitForAbsentGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}),
-		buf.Bytes(),
+				waitForAbsentGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}),
-		metav1.PatchOptions{
+			},
-			FieldManager: "test-manager",
+		},
-		},
+		{
-	)
+			// Show that if CRD and APIService share a groupversion, and the
-	require.NoError(t, err)
+			// APIService is deleted, and CRD updated, the APIService remains in
-
+			// discovery.
-	// Wait until the crd appears in discovery
+			// This test simulates a resync of CRD controler to show that eventually
-	expectedDiscovery := apidiscoveryv2beta1.APIGroupDiscovery{
+			// APIService is recreated
-		ObjectMeta: metav1.ObjectMeta{
+			Name: "CRDAPIServiceOverlap",
-			Name: basicTestGroup.Name,
+			Actions: []testAction{
-		},
+				applyAPIService(
-		Versions: []apidiscoveryv2beta1.APIVersionDiscovery{
+					apiregistrationv1.APIServiceSpec{
-			{
+						Group:                 stableGroup,
-				Version: "v1",
+						Version:               "v1",
-				Resources: []apidiscoveryv2beta1.APIResourceDiscovery{
+						InsecureSkipTLSVerify: true,
-					{
+						GroupPriorityMinimum:  int32(1000),
-						Resource: "foos",
+						VersionPriority:       int32(15),
-						ResponseKind: &metav1.GroupVersionKind{
+						Service: &apiregistrationv1.ServiceReference{
-							Group:   basicTestGroup.Name,
+							Name:      "unused",
-							Version: "v1",
+							Namespace: "default",
-							Kind:    "Foo",
+						},
-						},
+					},
-						Scope:            apidiscoveryv2beta1.ScopeCluster,
+				),
-						SingularResource: crd.Spec.Names.Singular,
+
-						Verbs:            []string{"delete", "deletecollection", "get", "list", "patch", "create", "update", "watch"},
+				// Wait for GV to appear in both discovery documents
-						Categories:       []string{"all"},
+				waitForGroupVersionsV1([]metav1.GroupVersion{stableV1}),
-					},
+				waitForGroupVersionsV2([]metav1.GroupVersion{stableV1}),
-				},
+
-				//!TODO: set freshness of builtin/crds
+				applyCRD(makeCRDSpec(stableGroup, "Bar", false, []string{"v1", "v2"})),
-				Freshness: "",
+
-			},
+				// only CRD has stable v2,  this will show that CRD has been synced
-		},
+				waitForGroupVersionsV1([]metav1.GroupVersion{stableV2}),
-	}
+				waitForGroupVersionsV2([]metav1.GroupVersion{stableV2}),
-	require.NoError(t, WaitForGroups(ctx, client, expectedDiscovery))
+
+				// Delete APIService shared by the aggregated apiservice and
+				// CRD
+				deleteObject{
+					GroupVersionResource: metav1.GroupVersionResource(apiregistrationv1.SchemeGroupVersion.WithResource("apiservices")),
+					Name:                 "v1.stable.example.com",
+				},
+
+				// Update CRD to trigger a resync by adding a category and new groupversion
+				applyCRD(makeCRDSpec(stableGroup, "Bar", false, []string{"v1", "v2", "v1alpha1"}, "all")),
+
+				// Show that the groupversion is re-added back
+				waitForGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV2, stableV1alpha1}),
+				waitForGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV2, stableV1alpha1}),
+			},
+		},
+	})
+}
+
+func TestFreshness(t *testing.T) {
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.AggregatedDiscoveryEndpoint, true)()
+
+	requireStaleGVs := func(gvs ...metav1.GroupVersion) inlineAction {
+		return inlineAction(func(ctx context.Context, client testClient) error {
+			document, err := FetchV2Discovery(ctx, client)
+			if err != nil {
+				return nil
+			}
+
+			// Track the stale gvs in array for nice diff output upon test failure
+			staleGVs := []metav1.GroupVersion{}
+
+			// Iterate through input so order does not matter
+			for _, targetGv := range gvs {
+				entry := FindGroupVersionV2(document, targetGv)
+				if entry == nil {
+					continue
+				}
+
+				switch entry.Freshness {
+				case apidiscoveryv2beta1.DiscoveryFreshnessCurrent:
+					// Skip
+				case apidiscoveryv2beta1.DiscoveryFreshnessStale:
+					staleGVs = append(staleGVs, targetGv)
+				default:
+					return fmt.Errorf("unrecognized freshness '%v' on gv '%v'", entry.Freshness, targetGv)
+				}
+			}
+
+			if !(len(staleGVs) == 0 && len(gvs) == 0) && !reflect.DeepEqual(staleGVs, gvs) {
+				diff := cmp.Diff(staleGVs, gvs)
+				return fmt.Errorf("expected sets of stale gvs to be equal:\n%v", diff)
+			}
+
+			return nil
+		})
+	}
+
+	runTestCases(t, []testCase{
+		{
+			Name: "BuiltinsFresh",
+			Actions: []testAction{
+				// Wait for discovery ready
+				waitForGroupVersionsV2{metav1.GroupVersion(apiregistrationv1.SchemeGroupVersion)},
+				// Require there are no stale groupversions and no unrecognized
+				// GVs
+				requireStaleGVs(),
+			},
+		},
+		{
+			// CRD freshness is always current
+			Name: "CRDFresh",
+			Actions: []testAction{
+				// Add a CRD and wait for it to appear in discovery
+				applyCRD(makeCRDSpec(stableGroup, "Foo", false, []string{"v1", "v1alpha1", "v1beta1", "v2"})),
+				waitForGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}),
+				waitForGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}),
+
+				// Test CRD is current by requiring there is nothing stale
+				requireStaleGVs(),
+			},
+		},
+		{
+			// Make an aggregated APIService that's unreachable and show
+			// that its groupversion is included in the discovery document as
+			// stale
+			Name: "AggregatedUnreachable",
+			Actions: []testAction{
+				applyAPIService{
+					Group:                stableGroup,
+					Version:              "v1",
+					GroupPriorityMinimum: 1000,
+					VersionPriority:      15,
+					Service: &apiregistrationv1.ServiceReference{
+						Name:      "doesnt-exist",
+						Namespace: "default",
+					},
+				},
+				waitForGroupVersionsV2([]metav1.GroupVersion{stableV1}),
+				// Require there is one and only one stale GV and it is stableV1
+				requireStaleGVs(stableV1),
+			},
+		},
+	})
+
+}
+
+// Shows a group for which multiple APIServices specify a GroupPriorityMinimum,
+// it is sorted the same in both versions of discovery
+func TestGroupPriorty(t *testing.T) {
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.AggregatedDiscoveryEndpoint, true)()
+
+	makeApiServiceSpec := func(gv metav1.GroupVersion, groupPriorityMin, versionPriority int) apiregistrationv1.APIServiceSpec {
+		return apiregistrationv1.APIServiceSpec{
+			Group:                 gv.Group,
+			Version:               gv.Version,
+			InsecureSkipTLSVerify: true,
+			GroupPriorityMinimum:  int32(groupPriorityMin),
+			VersionPriority:       int32(versionPriority),
+			Service: &apiregistrationv1.ServiceReference{
+				Name:      "unused",
+				Namespace: "default",
+			},
+		}
+	}
+
+	checkGVOrder := inlineAction(func(ctx context.Context, client testClient) (err error) {
+		// Fetch v1 document and v2 document, and ensure they have
+		// equal orderings of groupversions. and nothing missing or
+		// extra.
+		v1GroupsAndVersions, err := FetchV1DiscoveryGroups(ctx, client)
+		if err != nil {
+			return err
+		}
+		v2GroupsAndVersions, err := FetchV2Discovery(ctx, client)
+		if err != nil {
+			return err
+		}
+
+		v1Gvs := []metav1.GroupVersion{}
+		v2Gvs := []metav1.GroupVersion{}
+
+		for _, group := range v1GroupsAndVersions.Groups {
+			for _, version := range group.Versions {
+				v1Gvs = append(v1Gvs, metav1.GroupVersion{
+					Group:   group.Name,
+					Version: version.Version,
+				})
+			}
+		}
+
+		for _, group := range v2GroupsAndVersions.Items {
+			for _, version := range group.Versions {
+				v2Gvs = append(v2Gvs, metav1.GroupVersion{
+					Group:   group.Name,
+					Version: version.Version,
+				})
+			}
+		}
+
+		if !reflect.DeepEqual(v1Gvs, v2Gvs) {
+			return fmt.Errorf("expected equal orderings and lists of groupversions in both v1 and v2 discovery:\n%v", cmp.Diff(v1Gvs, v2Gvs))
+		}
+
+		return nil
+	})
+
+	runTestCases(t, []testCase{
+		{
+			// Show that the legacy and aggregated discovery docs have the same
+			// set of builtin groupversions
+			Name: "BuiltinsAndOrdering",
+			Actions: []testAction{
+				waitForGroupVersionsV1{metav1.GroupVersion(apiregistrationv1.SchemeGroupVersion)},
+				waitForGroupVersionsV2{metav1.GroupVersion(apiregistrationv1.SchemeGroupVersion)},
+				checkGVOrder,
+			},
+		},
+		{
+			// Show that a very high priority group is sorted first (below apiregistration v1)
+			// Also show the ordering is same for both v1 and v2 discovery apis
+			// Does not vary version priority
+			Name: "HighGroupPriority",
+			Actions: []testAction{
+				// A VERY high priority which should take precedence
+				// 20000 is highest possible priority
+				applyAPIService(makeApiServiceSpec(stableV1, 20000, 15)),
+				// A VERY low priority which should be ignored
+				applyAPIService(makeApiServiceSpec(stableV1alpha1, 1, 15)),
+				// A medium-high priority (that conflicts with k8s) which should be ignored
+				applyAPIService(makeApiServiceSpec(stableV1alpha2, 17300, 15)),
+				// Wait for all the added group-versions to appear in both discovery documents
+				waitForGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1alpha2}),
+				waitForGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1alpha2}),
+				// Check that both v1 and v2 endpoints have exactly the same
+				// sets of groupversions
+				checkGVOrder,
+				// Check that the first group-version is the one with the highest
+				// priority
+				inlineAction(func(ctx context.Context, client testClient) error {
+					v2GroupsAndVersions, err := FetchV2Discovery(ctx, client)
+					if err != nil {
+						return err
+					}
+
+					// First group should always be apiregistration.k8s.io
+					secondGV := metav1.GroupVersion{
+						Group:   v2GroupsAndVersions.Items[1].Name,
+						Version: v2GroupsAndVersions.Items[1].Versions[0].Version,
+					}
+
+					if !reflect.DeepEqual(&stableV1, &secondGV) {
+						return fmt.Errorf("expected second group's first version to be %v, not %v", stableV1, secondGV)
+					}
+
+					return nil
+				}),
+			},
+		},
+		{
+			// Show that a very low group priority is ordered last
+			Name: "LowGroupPriority",
+			Actions: []testAction{
+				// A minimal priority
+				applyAPIService(makeApiServiceSpec(stableV1alpha1, 1, 15)),
+				// Wait for all the added group-versions to appear in v2 discovery
+				waitForGroupVersionsV2([]metav1.GroupVersion{stableV1alpha1}),
+				// Check that the last group-version is the one with the lowest
+				// priority
+				inlineAction(func(ctx context.Context, client testClient) error {
+					v2GroupsAndVersions, err := FetchV2Discovery(ctx, client)
+					if err != nil {
+						return err
+					}
+					lastGroup := v2GroupsAndVersions.Items[len(v2GroupsAndVersions.Items)-1]
+
+					lastGV := metav1.GroupVersion{
+						Group:   lastGroup.Name,
+						Version: lastGroup.Versions[0].Version,
+					}
+
+					if !reflect.DeepEqual(&stableV1alpha1, &lastGV) {
+						return fmt.Errorf("expected last group to be %v, not %v", stableV1alpha1, lastGV)
+					}
+
+					return nil
+				}),
+				// Wait for all the added group-versions to appear in both discovery documents
+				waitForGroupVersionsV1([]metav1.GroupVersion{stableV1alpha1}),
+				// Check that both v1 and v2 endpoints have exactly the same
+				// sets of groupversions
+				checkGVOrder,
+			},
+		},
+		{
+			// Show that versions within a group are sorted by priority
+			Name: "VersionPriority",
+			Actions: []testAction{
+				applyAPIService(makeApiServiceSpec(stableV1, 1000, 2)),
+				applyAPIService(makeApiServiceSpec(stableV1alpha1, 1000, 1)),
+				applyAPIService(makeApiServiceSpec(stableV1alpha2, 1000, 3)),
+				// Wait for all the added group-versions to appear in both discovery documents
+				waitForGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1alpha2}),
+				waitForGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1alpha2}),
+				// Check that both v1 and v2 endpoints have exactly the same
+				// sets of groupversions
+				checkGVOrder,
+				inlineAction(func(ctx context.Context, client testClient) error {
+					// Find the entry for stable.example.com
+					// and show the versions are ordered how we expect
+					v2GroupsAndVersions, err := FetchV2Discovery(ctx, client)
+					if err != nil {
+						return err
+					}
+
+					// Should be ordered last for this test
+					group := v2GroupsAndVersions.Items[len(v2GroupsAndVersions.Items)-1]
+					if group.Name != stableGroup {
+						return fmt.Errorf("group is not where we expect: found %v, expected %v", group.Name, stableGroup)
+					}
+
+					versionOrder := []string{}
+					for _, version := range group.Versions {
+						versionOrder = append(versionOrder, version.Version)
+					}
+
+					expectedOrder := []string{
+						stableV1alpha2.Version,
+						stableV1.Version,
+						stableV1alpha1.Version,
+					}
+
+					if !reflect.DeepEqual(expectedOrder, versionOrder) {
+						return fmt.Errorf("version in wrong order: %v", cmp.Diff(expectedOrder, versionOrder))
+					}
+
+					return nil
+				}),
+			},
+		},
+		{
+			// Show that versions within a group are sorted by priority
+			// and that equal versions will be sorted by a kube-aware version
+			// comparator
+			Name: "VersionPriorityTiebreaker",
+			Actions: []testAction{
+				applyAPIService(makeApiServiceSpec(stableV1, 1000, 15)),
+				applyAPIService(makeApiServiceSpec(stableV1alpha1, 1000, 15)),
+				applyAPIService(makeApiServiceSpec(stableV1alpha2, 1000, 15)),
+				applyAPIService(makeApiServiceSpec(stableV1beta1, 1000, 15)),
+				applyAPIService(makeApiServiceSpec(stableV2, 1000, 15)),
+				// Wait for all the added group-versions to appear in both discovery documents
+				waitForGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1alpha2, stableV1beta1, stableV2}),
+				waitForGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1alpha2, stableV1beta1, stableV2}),
+				// Check that both v1 and v2 endpoints have exactly the same
+				// sets of groupversions
+				checkGVOrder,
+				inlineAction(func(ctx context.Context, client testClient) error {
+					// Find the entry for stable.example.com
+					// and show the versions are ordered how we expect
+					v2GroupsAndVersions, err := FetchV2Discovery(ctx, client)
+					if err != nil {
+						return err
+					}
+
+					// Should be ordered last for this test
+					group := v2GroupsAndVersions.Items[len(v2GroupsAndVersions.Items)-1]
+					if group.Name != stableGroup {
+						return fmt.Errorf("group is not where we expect: found %v, expected %v", group.Name, stableGroup)
+					}
+
+					versionOrder := []string{}
+					for _, version := range group.Versions {
+						versionOrder = append(versionOrder, version.Version)
+					}
+
+					expectedOrder := []string{
+						stableV2.Version,
+						stableV1.Version,
+						stableV1beta1.Version,
+						stableV1alpha2.Version,
+						stableV1alpha1.Version,
+					}
+
+					if !reflect.DeepEqual(expectedOrder, versionOrder) {
+						return fmt.Errorf("version in wrong order: %v", cmp.Diff(expectedOrder, versionOrder))
+					}
+
+					return nil
+				}),
+			},
+		},
+	})
+}
+
+func makeCRDSpec(group string, kind string, namespaced bool, versions []string, categories ...string) apiextensionsv1.CustomResourceDefinitionSpec {
+	scope := apiextensionsv1.NamespaceScoped
+	if !namespaced {
+		scope = apiextensionsv1.ClusterScoped
+	}
+
+	plural, singular := meta.UnsafeGuessKindToResource(schema.GroupVersionKind{Kind: kind})
+	res := apiextensionsv1.CustomResourceDefinitionSpec{
+		Group: group,
+		Scope: scope,
+		Names: apiextensionsv1.CustomResourceDefinitionNames{
+			Plural:     plural.Resource,
+			Singular:   singular.Resource,
+			Kind:       kind,
+			Categories: categories,
+		},
+	}
+
+	for i, version := range versions {
+		res.Versions = append(res.Versions, apiextensionsv1.CustomResourceDefinitionVersion{
+			Name:    version,
+			Served:  true,
+			Storage: i == 0,
+			Schema: &apiextensionsv1.CustomResourceValidation{
+				OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
+					Type: "object",
+					Properties: map[string]apiextensionsv1.JSONSchemaProps{
+						"data": {
+							Type: "string",
+						},
+					},
+				},
+			},
+		})
+	}
+	return res
 }

test/integration/apiserver/discovery/framework.go

@@ -0,0 +1,478 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package discovery
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"reflect"
+	"time"
+
+	apiextensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/kubernetes"
+	aggregator "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
+
+	apidiscoveryv2beta1 "k8s.io/api/apidiscovery/v2beta1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
+)
+
+const acceptV1JSON = "application/json"
+const acceptV2JSON = "application/json;g=apidiscovery.k8s.io;v=v2beta1;as=APIGroupDiscoveryList"
+
+type testClient interface {
+	kubernetes.Interface
+	aggregator.Interface
+	apiextensions.Interface
+	dynamic.Interface
+}
+
+// declarative framework for discovery integration tests
+// each test has metadata and a list of actions which each must pass for the
+// test to pass
+type testCase struct {
+	Name    string
+	Actions []testAction
+}
+
+// interface defining a function that does something with the integration test
+// api server and returns an error. the test fails if the error is non nil
+type testAction interface {
+	Do(ctx context.Context, client testClient) error
+}
+
+type cleaningAction interface {
+	testAction
+	Cleanup(ctx context.Context, client testClient) error
+}
+
+// apply an apiservice to the cluster
+type applyAPIService apiregistrationv1.APIServiceSpec
+
+type applyCRD apiextensionsv1.CustomResourceDefinitionSpec
+
+type deleteObject struct {
+	metav1.GroupVersionResource
+	Namespace string
+	Name      string
+}
+
+// Wait for groupversions to appear in v1 discovery
+type waitForGroupVersionsV1 []metav1.GroupVersion
+
+// Wait for groupversions to disappear from v2 discovery
+type waitForAbsentGroupVersionsV1 []metav1.GroupVersion
+
+// Wait for groupversions to appear in v2 discovery
+type waitForGroupVersionsV2 []metav1.GroupVersion
+
+// Wait for groupversions to disappear from v2 discovery
+type waitForAbsentGroupVersionsV2 []metav1.GroupVersion
+
+// Assert something about the current state of v2 discovery
+type inlineAction func(ctx context.Context, client testClient) error
+
+func (a applyAPIService) Do(ctx context.Context, client testClient) error {
+	// using dynamic client since the typed client does not support `Apply`
+	// operation?
+	obj := &apiregistrationv1.APIService{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: a.Version + "." + a.Group,
+		},
+		Spec: apiregistrationv1.APIServiceSpec(a),
+	}
+
+	unstructuredContent, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj)
+	if err != nil {
+		return err
+	}
+
+	unstructedObject := &unstructured.Unstructured{}
+	unstructedObject.SetUnstructuredContent(unstructuredContent)
+	unstructedObject.SetGroupVersionKind(apiregistrationv1.SchemeGroupVersion.WithKind("APIService"))
+
+	_, err = client.
+		Resource(apiregistrationv1.SchemeGroupVersion.WithResource("apiservices")).
+		Apply(ctx, obj.Name, unstructedObject, metav1.ApplyOptions{
+			FieldManager: "test-manager",
+		})
+
+	return err
+}
+
+func (a applyAPIService) Cleanup(ctx context.Context, client testClient) error {
+	name := a.Version + "." + a.Group
+	err := client.ApiregistrationV1().APIServices().Delete(ctx, name, metav1.DeleteOptions{})
+
+	if !errors.IsNotFound(err) {
+		return err
+	}
+
+	err = wait.PollWithContext(
+		ctx,
+		250*time.Millisecond,
+		1*time.Second,
+		func(ctx context.Context) (done bool, err error) {
+			_, err = client.ApiregistrationV1().APIServices().Get(ctx, name, metav1.GetOptions{})
+			if err == nil {
+				return false, nil
+			}
+
+			if !errors.IsNotFound(err) {
+				return false, err
+			}
+			return true, nil
+		},
+	)
+
+	if err != nil {
+		return fmt.Errorf("error waiting for APIService %v to clean up: %w", name, err)
+	}
+
+	return nil
+}
+
+func (a applyCRD) Do(ctx context.Context, client testClient) error {
+	// using dynamic client since the typed client does not support `Apply`
+	// operation?
+	name := a.Names.Plural + "." + a.Group
+	obj := &apiextensionsv1.CustomResourceDefinition{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+		Spec: apiextensionsv1.CustomResourceDefinitionSpec(a),
+	}
+
+	unstructuredContent, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj)
+	if err != nil {
+		return err
+	}
+
+	unstructedObject := &unstructured.Unstructured{}
+	unstructedObject.SetUnstructuredContent(unstructuredContent)
+	unstructedObject.SetGroupVersionKind(apiextensionsv1.SchemeGroupVersion.WithKind("CustomResourceDefinition"))
+
+	_, err = client.
+		Resource(apiextensionsv1.SchemeGroupVersion.WithResource("customresourcedefinitions")).
+		Apply(ctx, obj.Name, unstructedObject, metav1.ApplyOptions{
+			FieldManager: "test-manager",
+		})
+
+	return err
+}
+
+func (a applyCRD) Cleanup(ctx context.Context, client testClient) error {
+	name := a.Names.Plural + "." + a.Group
+	err := client.ApiextensionsV1().CustomResourceDefinitions().Delete(ctx, name, metav1.DeleteOptions{})
+
+	if !errors.IsNotFound(err) {
+		return err
+	}
+
+	err = wait.PollWithContext(
+		ctx,
+		250*time.Millisecond,
+		1*time.Second,
+		func(ctx context.Context) (done bool, err error) {
+			_, err = client.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, name, metav1.GetOptions{})
+			if err == nil {
+				return false, nil
+			}
+
+			if !errors.IsNotFound(err) {
+				return false, err
+			}
+			return true, nil
+		},
+	)
+
+	if err != nil {
+		return fmt.Errorf("error waiting for CRD %v to clean up: %w", name, err)
+	}
+
+	return nil
+}
+
+func (d deleteObject) Do(ctx context.Context, client testClient) error {
+	if d.Namespace == "" {
+		return client.Resource(schema.GroupVersionResource(d.GroupVersionResource)).
+			Delete(ctx, d.Name, metav1.DeleteOptions{})
+	} else {
+		return client.Resource(schema.GroupVersionResource(d.GroupVersionResource)).
+			Namespace(d.Namespace).
+			Delete(ctx, d.Name, metav1.DeleteOptions{})
+	}
+}
+
+func (w waitForGroupVersionsV2) Do(ctx context.Context, client testClient) error {
+	err := WaitForResultWithCondition(ctx, client, func(result apidiscoveryv2beta1.APIGroupDiscoveryList) bool {
+		for _, gv := range w {
+			if FindGroupVersionV2(result, gv) == nil {
+				return false
+			}
+		}
+
+		return true
+	})
+
+	if err != nil {
+		return fmt.Errorf("waiting for groupversions v2 (%v): %w", w, err)
+	}
+	return nil
+}
+
+func (w waitForAbsentGroupVersionsV2) Do(ctx context.Context, client testClient) error {
+	err := WaitForResultWithCondition(ctx, client, func(result apidiscoveryv2beta1.APIGroupDiscoveryList) bool {
+		for _, gv := range w {
+			if FindGroupVersionV2(result, gv) != nil {
+				return false
+			}
+		}
+
+		return true
+	})
+
+	if err != nil {
+		return fmt.Errorf("waiting for absent groupversions v2 (%v): %w", w, err)
+	}
+	return nil
+}
+
+func (w waitForGroupVersionsV1) Do(ctx context.Context, client testClient) error {
+	err := WaitForV1GroupsWithCondition(ctx, client, func(result metav1.APIGroupList) bool {
+		for _, gv := range w {
+			if !FindGroupVersionV1(result, gv) {
+				return false
+			}
+		}
+
+		return true
+	})
+
+	if err != nil {
+		return fmt.Errorf("waiting for groupversions v1 (%v): %w", w, err)
+	}
+	return nil
+}
+
+func (w waitForAbsentGroupVersionsV1) Do(ctx context.Context, client testClient) error {
+	err := WaitForV1GroupsWithCondition(ctx, client, func(result metav1.APIGroupList) bool {
+		for _, gv := range w {
+			if FindGroupVersionV1(result, gv) {
+				return false
+			}
+		}
+
+		return true
+	})
+
+	if err != nil {
+		return fmt.Errorf("waiting for absent groupversions v1 (%v): %w", w, err)
+	}
+	return nil
+}
+
+func (i inlineAction) Do(ctx context.Context, client testClient) error {
+	return i(ctx, client)
+}
+
+func FetchV2Discovery(ctx context.Context, client testClient) (apidiscoveryv2beta1.APIGroupDiscoveryList, error) {
+	result, err := client.
+		Discovery().
+		RESTClient().
+		Get().
+		AbsPath("/apis").
+		SetHeader("Accept", acceptV2JSON).
+		Do(ctx).
+		Raw()
+
+	if err != nil {
+		return apidiscoveryv2beta1.APIGroupDiscoveryList{}, fmt.Errorf("failed to fetch v2 discovery: %w", err)
+	}
+
+	groupList := apidiscoveryv2beta1.APIGroupDiscoveryList{}
+	err = json.Unmarshal(result, &groupList)
+	if err != nil {
+		return apidiscoveryv2beta1.APIGroupDiscoveryList{}, fmt.Errorf("failed to parse v2 discovery: %w", err)
+	}
+
+	return groupList, nil
+}
+
+func FetchV1DiscoveryGroups(ctx context.Context, client testClient) (metav1.APIGroupList, error) {
+	return FetchV1DiscoveryGroupsAtPath(ctx, client, "/apis")
+}
+
+func FetchV1DiscoveryLegacyGroups(ctx context.Context, client testClient) (metav1.APIGroupList, error) {
+	return FetchV1DiscoveryGroupsAtPath(ctx, client, "/api")
+}
+
+func FetchV1DiscoveryGroupsAtPath(ctx context.Context, client testClient, path string) (metav1.APIGroupList, error) {
+	result, err := client.
+		Discovery().
+		RESTClient().
+		Get().
+		AbsPath("/apis").
+		SetHeader("Accept", acceptV1JSON).
+		Do(ctx).
+		Raw()
+
+	if err != nil {
+		return metav1.APIGroupList{}, fmt.Errorf("failed to fetch v1 discovery at %v: %w", path, err)
+	}
+
+	groupList := metav1.APIGroupList{}
+	err = json.Unmarshal(result, &groupList)
+	if err != nil {
+		return metav1.APIGroupList{}, fmt.Errorf("failed to parse v1 discovery at %v: %w", path, err)
+	}
+
+	return groupList, nil
+}
+
+func FetchV1DiscoveryResource(ctx context.Context, client testClient, gv metav1.GroupVersion) (metav1.APIGroupList, error) {
+	result, err := client.
+		Discovery().
+		RESTClient().
+		Get().
+		AbsPath("/apis/"+gv.Group+"/"+gv.Version).
+		SetHeader("Accept", acceptV1JSON).
+		Do(ctx).
+		Raw()
+
+	if err != nil {
+		return metav1.APIGroupList{}, err
+	}
+
+	groupList := metav1.APIGroupList{}
+	err = json.Unmarshal(result, &groupList)
+	if err != nil {
+		return metav1.APIGroupList{}, err
+	}
+
+	return groupList, nil
+}
+
+func WaitForGroupsAbsent(ctx context.Context, client testClient, groups ...string) error {
+	return WaitForResultWithCondition(ctx, client, func(groupList apidiscoveryv2beta1.APIGroupDiscoveryList) bool {
+		for _, searchGroup := range groups {
+			for _, docGroup := range groupList.Items {
+				if docGroup.Name == searchGroup {
+					return false
+				}
+			}
+		}
+		return true
+	})
+
+}
+
+func WaitForGroups(ctx context.Context, client testClient, groups ...apidiscoveryv2beta1.APIGroupDiscovery) error {
+	return WaitForResultWithCondition(ctx, client, func(groupList apidiscoveryv2beta1.APIGroupDiscoveryList) bool {
+		for _, searchGroup := range groups {
+			for _, docGroup := range groupList.Items {
+				if reflect.DeepEqual(searchGroup, docGroup) {
+					return true
+				}
+			}
+		}
+		return false
+	})
+}
+
+func WaitForResultWithCondition(ctx context.Context, client testClient, condition func(result apidiscoveryv2beta1.APIGroupDiscoveryList) bool) error {
+	// Keep repeatedly fetching document from aggregator.
+	// Check to see if it contains our service within a reasonable amount of time
+	return wait.PollWithContext(
+		ctx,
+		250*time.Millisecond,
+		1*time.Second,
+		func(ctx context.Context) (done bool, err error) {
+			groupList, err := FetchV2Discovery(ctx, client)
+			if err != nil {
+				return false, err
+			}
+
+			if condition(groupList) {
+				return true, nil
+			}
+
+			return false, nil
+		})
+}
+
+func WaitForV1GroupsWithCondition(ctx context.Context, client testClient, condition func(result metav1.APIGroupList) bool) error {
+	// Keep repeatedly fetching document from aggregator.
+	// Check to see if it contains our service within a reasonable amount of time
+	return wait.PollWithContext(
+		ctx,
+		250*time.Millisecond,
+		1*time.Second,
+		func(ctx context.Context) (done bool, err error) {
+			groupList, err := FetchV1DiscoveryGroups(ctx, client)
+
+			if err != nil {
+				return false, err
+			}
+
+			if condition(groupList) {
+				return true, nil
+			}
+
+			return false, nil
+		})
+}
+
+func FindGroupVersionV1(discovery metav1.APIGroupList, gv metav1.GroupVersion) bool {
+	for _, documentGroup := range discovery.Groups {
+		if documentGroup.Name != gv.Group {
+			continue
+		}
+
+		for _, documentVersion := range documentGroup.Versions {
+			if documentVersion.Version == gv.Version {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
+func FindGroupVersionV2(discovery apidiscoveryv2beta1.APIGroupDiscoveryList, gv metav1.GroupVersion) *apidiscoveryv2beta1.APIVersionDiscovery {
+	for _, documentGroup := range discovery.Items {
+		if documentGroup.Name != gv.Group {
+			continue
+		}
+
+		for _, documentVersion := range documentGroup.Versions {
+			if documentVersion.Version == gv.Version {
+				return &documentVersion
+			}
+		}
+	}
+
+	return nil
+}

test/integration/apiserver/discovery/service.go

@@ -19,15 +19,14 @@ package discovery
 import (
 	"context"
 	"errors"
+	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
 	"strconv"
 	"sync"
-	"testing"
 	"time"
 
-	"github.com/stretchr/testify/require"
 	"k8s.io/apimachinery/pkg/util/wait"
 
 	"k8s.io/client-go/kubernetes"
@@ -38,7 +37,7 @@ import (
 )
 
 type FakeService interface {
-	Start(t *testing.T, ctx context.Context)
+	Run(ctx context.Context) error
 	Port() *int32
 	Name() string
 }
@@ -62,7 +61,7 @@ func NewFakeService(name string, client kubernetes.Interface, handler http.Handl
 	}
 }
 
-func (f *fakeService) run(ctx context.Context) error {
+func (f *fakeService) Run(ctx context.Context) error {
 	aggregatedServer := httptest.NewUnstartedServer(f.handler)
 	aggregatedServer.StartTLS()
 	defer aggregatedServer.Close()
@@ -111,26 +110,24 @@ func (f *fakeService) run(ctx context.Context) error {
 
 	// Uninstall service from the cluser
 	err = f.client.CoreV1().Services("default").Delete(ctx, service.Name, metav1.DeleteOptions{})
+	if errors.Is(err, context.Canceled) {
+		err = nil
+	}
 	return err
 }
 
-func (f *fakeService) Start(t *testing.T, ctx context.Context) {
+func (f *fakeService) WaitForReady(ctx context.Context) error {
-	go func() {
-		err := f.run(ctx)
-		if errors.Is(err, context.Canceled) {
-			err = nil
-		}
-		require.NoError(t, err)
-	}()
-
 	err := wait.PollWithContext(ctx, 1*time.Second, 200*time.Millisecond, func(ctx context.Context) (done bool, err error) {
 		return f.Port() != nil, nil
 	})
 
 	if errors.Is(err, context.Canceled) {
 		err = nil
+	} else if err != nil {
+		err = fmt.Errorf("service should have come alive in a reasonable amount of time: %w", err)
 	}
-	require.NoError(t, err, "service should have come alive in a reasonable amount of time")
+
+	return err
 }
 
 func (f *fakeService) Port() *int32 {