github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-20 18:31:15 +00:00
Code Issues Projects Releases Wiki Activity

add discovery integration tests

Browse Source 
add crd integration tests
\
This commit is contained in:
Alexander Zielenski 2022-11-10 13:20:10 -08:00
parent 3c68fe6596
commit 93caf125b9
3 changed files with 991 additions and 259 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

711
test/integration/apiserver/discovery/discovery_test.go
View File

@ -17,28 +17,25 @@ limitations under the License.
package discovery
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"reflect"
"strings"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/require"
apidiscoveryv2beta1 "k8s.io/api/apidiscovery/v2beta1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apiextensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/types"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
discoveryendpoint "k8s.io/apiserver/pkg/endpoints/discovery/aggregated"
genericfeatures "k8s.io/apiserver/pkg/features"
utilfeature "k8s.io/apiserver/pkg/util/feature"
@ -66,6 +63,8 @@ type testClientSet struct {
dynamicClientset
}
var _ testClient = testClientSet{}
func (t testClientSet) Discovery() discovery.DiscoveryInterface {
return t.kubeClientSet.Discovery()
}
@ -94,6 +93,13 @@ var (
},
},
}
stableGroup = "stable.example.com"
stableV1 = metav1.GroupVersion{Group: stableGroup, Version: "v1"}
stableV1alpha1 = metav1.GroupVersion{Group: stableGroup, Version: "v1alpha1"}
stableV1alpha2 = metav1.GroupVersion{Group: stableGroup, Version: "v1alpha2"}
stableV1beta1 = metav1.GroupVersion{Group: stableGroup, Version: "v1beta1"}
stableV2 = metav1.GroupVersion{Group: stableGroup, Version: "v2"}
)
func init() {
@ -178,68 +184,6 @@ func unregisterAPIService(ctx context.Context, client aggregator.Interface, gv m
return client.ApiregistrationV1().APIServices().Delete(ctx, gv.Version+"."+gv.Group, metav1.DeleteOptions{})
}
func WaitForGroupsAbsent(ctx context.Context, client testClientSet, groups ...string) error {
return WaitForResultWithCondition(ctx, client, func(groupList apidiscoveryv2beta1.APIGroupDiscoveryList) bool {
for _, searchGroup := range groups {
for _, docGroup := range groupList.Items {
if docGroup.Name == searchGroup {
return false
}
}
}
return true
})
}
func WaitForGroups(ctx context.Context, client testClientSet, groups ...apidiscoveryv2beta1.APIGroupDiscovery) error {
return WaitForResultWithCondition(ctx, client, func(groupList apidiscoveryv2beta1.APIGroupDiscoveryList) bool {
for _, searchGroup := range groups {
for _, docGroup := range groupList.Items {
if reflect.DeepEqual(searchGroup, docGroup) {
return true
}
}
}
return false
})
}
func WaitForResultWithCondition(ctx context.Context, client testClientSet, condition func(result apidiscoveryv2beta1.APIGroupDiscoveryList) bool) error {
// Keep repeatedly fetching document from aggregator.
// Check to see if it contains our service within a reasonable amount of time
return wait.PollWithContext(
ctx,
250*time.Millisecond,
1*time.Second,
func(ctx context.Context) (done bool, err error) {
result, err := client.
Discovery().
RESTClient().
Get().
AbsPath("/apis").
SetHeader("Accept", "application/json;g=apidiscovery.k8s.io;v=v2beta1;as=APIGroupDiscoveryList").
Do(ctx).
Raw()
if err != nil {
return false, err
}
groupList := apidiscoveryv2beta1.APIGroupDiscoveryList{}
err = json.Unmarshal(result, &groupList)
if err != nil {
panic(err)
}
if condition(groupList) {
return true, nil
}
return false, nil
})
}
func TestAggregatedAPIServiceDiscovery(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.AggregatedDiscoveryEndpoint, true)()
@ -264,7 +208,10 @@ func TestAggregatedAPIServiceDiscovery(t *testing.T) {
w.WriteHeader(http.StatusNotFound)
}
}))
service.Start(t, ctx)
go func() {
require.NoError(t, service.Run(ctx))
}()
require.NoError(t, service.WaitForReady(ctx))
// For each groupversion served by our resourcemanager, create an APIService
// object connected to our fake APIServer
@ -285,188 +232,498 @@ func TestAggregatedAPIServiceDiscovery(t *testing.T) {
require.NoError(t, WaitForGroups(ctx, client, basicTestGroup))
}
// Shows that the following sequence is handled correctly:
// 1. Create an APIService
// - Check that API service is in discovery doc
// 2. Create CRD with the same GroupVersion as APIService
// 3. Delete APIService
// - Check that API service is removed from discovery
// 4. Update CRD
// - Check that CRD is in discovery document
func TestOverlappingCRDAndAPIService(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.AggregatedDiscoveryEndpoint, true)()
func runTestCases(t *testing.T, cases []testCase) {
// Keep any goroutines spawned from running past the execution of this test
ctx, client, cleanup := setup(t)
defer cleanup()
// Create a resource manager whichs serves our GroupVersion
resourceManager := discoveryendpoint.NewResourceManager()
resourceManager.SetGroups([]apidiscoveryv2beta1.APIGroupDiscovery{basicTestGroup})
// Fetch the original discovery information so we can wait for it to
// reset between tests
originalV1, err := FetchV1DiscoveryGroups(ctx, client)
require.NoError(t, err)
// Install our ResourceManager as an Aggregated APIService to the
// test server
service := NewFakeService("test-server", client, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/apis" {
resourceManager.ServeHTTP(w, r)
} else if strings.HasPrefix(r.URL.Path, "/apis/") {
// Return "valid" response so APIService can be marked as "available"
w.WriteHeader(http.StatusOK)
} else {
// reject openapi/v2, openapi/v3, apis/<group>/<version>
w.WriteHeader(http.StatusNotFound)
originalV2, err := FetchV2Discovery(ctx, client)
require.NoError(t, err)
for _, c := range cases {
t.Run(c.Name, func(t *testing.T) {
func() {
for _, a := range c.Actions {
if cleaning, ok := a.(cleaningAction); ok {
defer func() {
require.NoError(t, cleaning.Cleanup(ctx, client))
}()
}
}))
service.Start(t, ctx)
require.NoError(t, a.Do(ctx, client))
}
}()
// For each groupversion served by our resourcemanager, create an APIService
// object connected to our fake APIServer
for _, versionInfo := range basicTestGroup.Versions {
groupVersion := metav1.GroupVersion{
Group: basicTestGroup.Name,
Version: versionInfo.Version,
var diff string
err := WaitForV1GroupsWithCondition(ctx, client, func(result metav1.APIGroupList) bool {
diff = cmp.Diff(originalV1, result)
return reflect.DeepEqual(result, originalV1)
})
require.NoError(t, err, "v1 discovery must reset between tests: "+diff)
err = WaitForResultWithCondition(ctx, client, func(result apidiscoveryv2beta1.APIGroupDiscoveryList) bool {
diff = cmp.Diff(originalV2, result)
return reflect.DeepEqual(result, originalV2)
})
require.NoError(t, err, "v2 discovery must reset between tests: "+diff)
})
}
}
registerAPIService(ctx, client, groupVersion, service)
}
// Declarative tests targeting CRD integration
func TestCRD(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.AggregatedDiscoveryEndpoint, true)()
// Keep repeatedly fetching document from aggregator.
// Check to see if it contains our service within a reasonable amount of time
require.NoError(t, WaitForGroups(ctx, client, basicTestGroup))
// Create a CRD
crd, err := client.ApiextensionsV1().CustomResourceDefinitions().Create(ctx, &apiextensionsv1.CustomResourceDefinition{
ObjectMeta: metav1.ObjectMeta{
runTestCases(t, []testCase{
{
// Show that when a CRD is added it gets included on the discovery doc
// within a reasonable amount of time
Name: "CRDInclusion",
Actions: []testAction{
applyCRD(makeCRDSpec(stableGroup, "Foo", false, []string{"v1", "v1alpha1", "v1beta1", "v2"})),
waitForGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}),
waitForGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}),
},
},
{
// Show that a CRD added to the discovery doc can also be removed
Name: "CRDRemoval",
Actions: []testAction{
applyCRD(makeCRDSpec(stableGroup, "Foo", false, []string{"v1", "v1alpha1", "v1beta1", "v2"})),
waitForGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}),
waitForGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}),
deleteObject{
GroupVersionResource: metav1.GroupVersionResource(apiextensionsv1.SchemeGroupVersion.WithResource("customresourcedefinitions")),
Name: "foos.stable.example.com",
},
Spec: apiextensionsv1.CustomResourceDefinitionSpec{
Group: "stable.example.com",
Names: apiextensionsv1.CustomResourceDefinitionNames{
Singular: "foo",
Plural: "foos",
Kind: "Foo",
waitForAbsentGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}),
waitForAbsentGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}),
},
},
Scope: apiextensionsv1.ClusterScoped,
Versions: []apiextensionsv1.CustomResourceDefinitionVersion{
{
Name: "v1",
// Show that if CRD and APIService share a groupversion, and the
// APIService is deleted, and CRD updated, the APIService remains in
// discovery.
// This test simulates a resync of CRD controler to show that eventually
// APIService is recreated
Name: "CRDAPIServiceOverlap",
Actions: []testAction{
applyAPIService(
apiregistrationv1.APIServiceSpec{
Group: stableGroup,
Version: "v1",
InsecureSkipTLSVerify: true,
GroupPriorityMinimum: int32(1000),
VersionPriority: int32(15),
Service: &apiregistrationv1.ServiceReference{
Name: "unused",
Namespace: "default",
},
},
),
// Wait for GV to appear in both discovery documents
waitForGroupVersionsV1([]metav1.GroupVersion{stableV1}),
waitForGroupVersionsV2([]metav1.GroupVersion{stableV1}),
applyCRD(makeCRDSpec(stableGroup, "Bar", false, []string{"v1", "v2"})),
// only CRD has stable v2, this will show that CRD has been synced
waitForGroupVersionsV1([]metav1.GroupVersion{stableV2}),
waitForGroupVersionsV2([]metav1.GroupVersion{stableV2}),
// Delete APIService shared by the aggregated apiservice and
// CRD
deleteObject{
GroupVersionResource: metav1.GroupVersionResource(apiregistrationv1.SchemeGroupVersion.WithResource("apiservices")),
Name: "v1.stable.example.com",
},
// Update CRD to trigger a resync by adding a category and new groupversion
applyCRD(makeCRDSpec(stableGroup, "Bar", false, []string{"v1", "v2", "v1alpha1"}, "all")),
// Show that the groupversion is re-added back
waitForGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV2, stableV1alpha1}),
waitForGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV2, stableV1alpha1}),
},
},
})
}
func TestFreshness(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.AggregatedDiscoveryEndpoint, true)()
requireStaleGVs := func(gvs ...metav1.GroupVersion) inlineAction {
return inlineAction(func(ctx context.Context, client testClient) error {
document, err := FetchV2Discovery(ctx, client)
if err != nil {
return nil
}
// Track the stale gvs in array for nice diff output upon test failure
staleGVs := []metav1.GroupVersion{}
// Iterate through input so order does not matter
for _, targetGv := range gvs {
entry := FindGroupVersionV2(document, targetGv)
if entry == nil {
continue
}
switch entry.Freshness {
case apidiscoveryv2beta1.DiscoveryFreshnessCurrent:
// Skip
case apidiscoveryv2beta1.DiscoveryFreshnessStale:
staleGVs = append(staleGVs, targetGv)
default:
return fmt.Errorf("unrecognized freshness '%v' on gv '%v'", entry.Freshness, targetGv)
}
}
if !(len(staleGVs) == 0 && len(gvs) == 0) && !reflect.DeepEqual(staleGVs, gvs) {
diff := cmp.Diff(staleGVs, gvs)
return fmt.Errorf("expected sets of stale gvs to be equal:\n%v", diff)
}
return nil
})
}
runTestCases(t, []testCase{
{
Name: "BuiltinsFresh",
Actions: []testAction{
// Wait for discovery ready
waitForGroupVersionsV2{metav1.GroupVersion(apiregistrationv1.SchemeGroupVersion)},
// Require there are no stale groupversions and no unrecognized
// GVs
requireStaleGVs(),
},
},
{
// CRD freshness is always current
Name: "CRDFresh",
Actions: []testAction{
// Add a CRD and wait for it to appear in discovery
applyCRD(makeCRDSpec(stableGroup, "Foo", false, []string{"v1", "v1alpha1", "v1beta1", "v2"})),
waitForGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}),
waitForGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}),
// Test CRD is current by requiring there is nothing stale
requireStaleGVs(),
},
},
{
// Make an aggregated APIService that's unreachable and show
// that its groupversion is included in the discovery document as
// stale
Name: "AggregatedUnreachable",
Actions: []testAction{
applyAPIService{
Group: stableGroup,
Version: "v1",
GroupPriorityMinimum: 1000,
VersionPriority: 15,
Service: &apiregistrationv1.ServiceReference{
Name: "doesnt-exist",
Namespace: "default",
},
},
waitForGroupVersionsV2([]metav1.GroupVersion{stableV1}),
// Require there is one and only one stale GV and it is stableV1
requireStaleGVs(stableV1),
},
},
})
}
// Shows a group for which multiple APIServices specify a GroupPriorityMinimum,
// it is sorted the same in both versions of discovery
func TestGroupPriorty(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.AggregatedDiscoveryEndpoint, true)()
makeApiServiceSpec := func(gv metav1.GroupVersion, groupPriorityMin, versionPriority int) apiregistrationv1.APIServiceSpec {
return apiregistrationv1.APIServiceSpec{
Group: gv.Group,
Version: gv.Version,
InsecureSkipTLSVerify: true,
GroupPriorityMinimum: int32(groupPriorityMin),
VersionPriority: int32(versionPriority),
Service: &apiregistrationv1.ServiceReference{
Name: "unused",
Namespace: "default",
},
}
}
checkGVOrder := inlineAction(func(ctx context.Context, client testClient) (err error) {
// Fetch v1 document and v2 document, and ensure they have
// equal orderings of groupversions. and nothing missing or
// extra.
v1GroupsAndVersions, err := FetchV1DiscoveryGroups(ctx, client)
if err != nil {
return err
}
v2GroupsAndVersions, err := FetchV2Discovery(ctx, client)
if err != nil {
return err
}
v1Gvs := []metav1.GroupVersion{}
v2Gvs := []metav1.GroupVersion{}
for _, group := range v1GroupsAndVersions.Groups {
for _, version := range group.Versions {
v1Gvs = append(v1Gvs, metav1.GroupVersion{
Group: group.Name,
Version: version.Version,
})
}
}
for _, group := range v2GroupsAndVersions.Items {
for _, version := range group.Versions {
v2Gvs = append(v2Gvs, metav1.GroupVersion{
Group: group.Name,
Version: version.Version,
})
}
}
if !reflect.DeepEqual(v1Gvs, v2Gvs) {
return fmt.Errorf("expected equal orderings and lists of groupversions in both v1 and v2 discovery:\n%v", cmp.Diff(v1Gvs, v2Gvs))
}
return nil
})
runTestCases(t, []testCase{
{
// Show that the legacy and aggregated discovery docs have the same
// set of builtin groupversions
Name: "BuiltinsAndOrdering",
Actions: []testAction{
waitForGroupVersionsV1{metav1.GroupVersion(apiregistrationv1.SchemeGroupVersion)},
waitForGroupVersionsV2{metav1.GroupVersion(apiregistrationv1.SchemeGroupVersion)},
checkGVOrder,
},
},
{
// Show that a very high priority group is sorted first (below apiregistration v1)
// Also show the ordering is same for both v1 and v2 discovery apis
// Does not vary version priority
Name: "HighGroupPriority",
Actions: []testAction{
// A VERY high priority which should take precedence
// 20000 is highest possible priority
applyAPIService(makeApiServiceSpec(stableV1, 20000, 15)),
// A VERY low priority which should be ignored
applyAPIService(makeApiServiceSpec(stableV1alpha1, 1, 15)),
// A medium-high priority (that conflicts with k8s) which should be ignored
applyAPIService(makeApiServiceSpec(stableV1alpha2, 17300, 15)),
// Wait for all the added group-versions to appear in both discovery documents
waitForGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1alpha2}),
waitForGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1alpha2}),
// Check that both v1 and v2 endpoints have exactly the same
// sets of groupversions
checkGVOrder,
// Check that the first group-version is the one with the highest
// priority
inlineAction(func(ctx context.Context, client testClient) error {
v2GroupsAndVersions, err := FetchV2Discovery(ctx, client)
if err != nil {
return err
}
// First group should always be apiregistration.k8s.io
secondGV := metav1.GroupVersion{
Group: v2GroupsAndVersions.Items[1].Name,
Version: v2GroupsAndVersions.Items[1].Versions[0].Version,
}
if !reflect.DeepEqual(&stableV1, &secondGV) {
return fmt.Errorf("expected second group's first version to be %v, not %v", stableV1, secondGV)
}
return nil
}),
},
},
{
// Show that a very low group priority is ordered last
Name: "LowGroupPriority",
Actions: []testAction{
// A minimal priority
applyAPIService(makeApiServiceSpec(stableV1alpha1, 1, 15)),
// Wait for all the added group-versions to appear in v2 discovery
waitForGroupVersionsV2([]metav1.GroupVersion{stableV1alpha1}),
// Check that the last group-version is the one with the lowest
// priority
inlineAction(func(ctx context.Context, client testClient) error {
v2GroupsAndVersions, err := FetchV2Discovery(ctx, client)
if err != nil {
return err
}
lastGroup := v2GroupsAndVersions.Items[len(v2GroupsAndVersions.Items)-1]
lastGV := metav1.GroupVersion{
Group: lastGroup.Name,
Version: lastGroup.Versions[0].Version,
}
if !reflect.DeepEqual(&stableV1alpha1, &lastGV) {
return fmt.Errorf("expected last group to be %v, not %v", stableV1alpha1, lastGV)
}
return nil
}),
// Wait for all the added group-versions to appear in both discovery documents
waitForGroupVersionsV1([]metav1.GroupVersion{stableV1alpha1}),
// Check that both v1 and v2 endpoints have exactly the same
// sets of groupversions
checkGVOrder,
},
},
{
// Show that versions within a group are sorted by priority
Name: "VersionPriority",
Actions: []testAction{
applyAPIService(makeApiServiceSpec(stableV1, 1000, 2)),
applyAPIService(makeApiServiceSpec(stableV1alpha1, 1000, 1)),
applyAPIService(makeApiServiceSpec(stableV1alpha2, 1000, 3)),
// Wait for all the added group-versions to appear in both discovery documents
waitForGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1alpha2}),
waitForGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1alpha2}),
// Check that both v1 and v2 endpoints have exactly the same
// sets of groupversions
checkGVOrder,
inlineAction(func(ctx context.Context, client testClient) error {
// Find the entry for stable.example.com
// and show the versions are ordered how we expect
v2GroupsAndVersions, err := FetchV2Discovery(ctx, client)
if err != nil {
return err
}
// Should be ordered last for this test
group := v2GroupsAndVersions.Items[len(v2GroupsAndVersions.Items)-1]
if group.Name != stableGroup {
return fmt.Errorf("group is not where we expect: found %v, expected %v", group.Name, stableGroup)
}
versionOrder := []string{}
for _, version := range group.Versions {
versionOrder = append(versionOrder, version.Version)
}
expectedOrder := []string{
stableV1alpha2.Version,
stableV1.Version,
stableV1alpha1.Version,
}
if !reflect.DeepEqual(expectedOrder, versionOrder) {
return fmt.Errorf("version in wrong order: %v", cmp.Diff(expectedOrder, versionOrder))
}
return nil
}),
},
},
{
// Show that versions within a group are sorted by priority
// and that equal versions will be sorted by a kube-aware version
// comparator
Name: "VersionPriorityTiebreaker",
Actions: []testAction{
applyAPIService(makeApiServiceSpec(stableV1, 1000, 15)),
applyAPIService(makeApiServiceSpec(stableV1alpha1, 1000, 15)),
applyAPIService(makeApiServiceSpec(stableV1alpha2, 1000, 15)),
applyAPIService(makeApiServiceSpec(stableV1beta1, 1000, 15)),
applyAPIService(makeApiServiceSpec(stableV2, 1000, 15)),
// Wait for all the added group-versions to appear in both discovery documents
waitForGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1alpha2, stableV1beta1, stableV2}),
waitForGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1alpha2, stableV1beta1, stableV2}),
// Check that both v1 and v2 endpoints have exactly the same
// sets of groupversions
checkGVOrder,
inlineAction(func(ctx context.Context, client testClient) error {
// Find the entry for stable.example.com
// and show the versions are ordered how we expect
v2GroupsAndVersions, err := FetchV2Discovery(ctx, client)
if err != nil {
return err
}
// Should be ordered last for this test
group := v2GroupsAndVersions.Items[len(v2GroupsAndVersions.Items)-1]
if group.Name != stableGroup {
return fmt.Errorf("group is not where we expect: found %v, expected %v", group.Name, stableGroup)
}
versionOrder := []string{}
for _, version := range group.Versions {
versionOrder = append(versionOrder, version.Version)
}
expectedOrder := []string{
stableV2.Version,
stableV1.Version,
stableV1beta1.Version,
stableV1alpha2.Version,
stableV1alpha1.Version,
}
if !reflect.DeepEqual(expectedOrder, versionOrder) {
return fmt.Errorf("version in wrong order: %v", cmp.Diff(expectedOrder, versionOrder))
}
return nil
}),
},
},
})
}
func makeCRDSpec(group string, kind string, namespaced bool, versions []string, categories ...string) apiextensionsv1.CustomResourceDefinitionSpec {
scope := apiextensionsv1.NamespaceScoped
if !namespaced {
scope = apiextensionsv1.ClusterScoped
}
plural, singular := meta.UnsafeGuessKindToResource(schema.GroupVersionKind{Kind: kind})
res := apiextensionsv1.CustomResourceDefinitionSpec{
Group: group,
Scope: scope,
Names: apiextensionsv1.CustomResourceDefinitionNames{
Plural: plural.Resource,
Singular: singular.Resource,
Kind: kind,
Categories: categories,
},
}
for i, version := range versions {
res.Versions = append(res.Versions, apiextensionsv1.CustomResourceDefinitionVersion{
Name: version,
Served: true,
Storage: true,
Storage: i == 0,
Schema: &apiextensionsv1.CustomResourceValidation{
OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
Type: "object",
Properties: map[string]apiextensionsv1.JSONSchemaProps{
"stringMap": {
Description: "a map[string]string",
Type: "object",
AdditionalProperties: &apiextensionsv1.JSONSchemaPropsOrBool{
Schema: &apiextensionsv1.JSONSchemaProps{
"data": {
Type: "string",
},
},
},
},
},
},
},
},
},
}, metav1.CreateOptions{
FieldManager: "test-manager",
})
require.NoError(t, err)
// Create a CR for the CRD
// Keep trying until it succeeds (or should we try for discovery?)
require.NoError(t, wait.PollWithContext(ctx, 100*time.Millisecond, 1*time.Second, func(ctx context.Context) (done bool, err error) {
toCreate := &unstructured.Unstructured{}
toCreate.SetUnstructuredContent(map[string]any{
"apiVersion": "stable.example.com/v1",
"kind": "Foo",
"key": "value",
})
_, err = client.dynamicClientset.Resource(schema.GroupVersionResource{
Group: "stable.example.com",
Version: "v1",
Resource: "foos",
}).Create(ctx, toCreate, metav1.CreateOptions{
FieldManager: "test-manager",
})
return err != nil, nil
}))
// For each groupversion served by our resourcemanager, delete an APIService
// object connected to our fake APIServer
for _, versionInfo := range basicTestGroup.Versions {
groupVersion := metav1.GroupVersion{
Group: basicTestGroup.Name,
Version: versionInfo.Version,
}
unregisterAPIService(ctx, client, groupVersion)
}
// Wait for the apiservice to be deleted from discovery
require.NoError(t, WaitForGroupsAbsent(ctx, client, "stable.example.com"))
// Update the CRD with a minor change to show that reconciliation will
// eventually refresh the discovery group on resync
obj := &unstructured.Unstructured{}
obj.SetUnstructuredContent(map[string]interface{}{
"apiVersion": "apiextensions.k8s.io/v1",
"kind": "CustomResourceDefinition",
"metadata": map[string]any{
"name": crd.Name,
},
"spec": map[string]interface{}{
"names": map[string]any{
"categories": []string{"all"},
},
},
})
buf := bytes.NewBuffer(nil)
err = unstructured.UnstructuredJSONScheme.Encode(obj, buf)
require.NoError(t, err)
//Is there a better way to force crd resync?
_, err = client.ApiextensionsV1().CustomResourceDefinitions().Patch(
ctx,
crd.Name,
types.ApplyPatchType,
buf.Bytes(),
metav1.PatchOptions{
FieldManager: "test-manager",
},
)
require.NoError(t, err)
// Wait until the crd appears in discovery
expectedDiscovery := apidiscoveryv2beta1.APIGroupDiscovery{
ObjectMeta: metav1.ObjectMeta{
Name: basicTestGroup.Name,
},
Versions: []apidiscoveryv2beta1.APIVersionDiscovery{
{
Version: "v1",
Resources: []apidiscoveryv2beta1.APIResourceDiscovery{
{
Resource: "foos",
ResponseKind: &metav1.GroupVersionKind{
Group: basicTestGroup.Name,
Version: "v1",
Kind: "Foo",
},
Scope: apidiscoveryv2beta1.ScopeCluster,
SingularResource: crd.Spec.Names.Singular,
Verbs: []string{"delete", "deletecollection", "get", "list", "patch", "create", "update", "watch"},
Categories: []string{"all"},
},
},
//!TODO: set freshness of builtin/crds
Freshness: "",
},
},
}
require.NoError(t, WaitForGroups(ctx, client, expectedDiscovery))
return res
}

478
test/integration/apiserver/discovery/framework.go Normal file
View File

@ -0,0 +1,478 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package discovery
import (
"context"
"encoding/json"
"fmt"
"reflect"
"time"
apiextensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"
aggregator "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
apidiscoveryv2beta1 "k8s.io/api/apidiscovery/v2beta1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
)
const acceptV1JSON = "application/json"
const acceptV2JSON = "application/json;g=apidiscovery.k8s.io;v=v2beta1;as=APIGroupDiscoveryList"
type testClient interface {
kubernetes.Interface
aggregator.Interface
apiextensions.Interface
dynamic.Interface
}
// declarative framework for discovery integration tests
// each test has metadata and a list of actions which each must pass for the
// test to pass
type testCase struct {
Name string
Actions []testAction
}
// interface defining a function that does something with the integration test
// api server and returns an error. the test fails if the error is non nil
type testAction interface {
Do(ctx context.Context, client testClient) error
}
type cleaningAction interface {
testAction
Cleanup(ctx context.Context, client testClient) error
}
// apply an apiservice to the cluster
type applyAPIService apiregistrationv1.APIServiceSpec
type applyCRD apiextensionsv1.CustomResourceDefinitionSpec
type deleteObject struct {
metav1.GroupVersionResource
Namespace string
Name string
}
// Wait for groupversions to appear in v1 discovery
type waitForGroupVersionsV1 []metav1.GroupVersion
// Wait for groupversions to disappear from v2 discovery
type waitForAbsentGroupVersionsV1 []metav1.GroupVersion
// Wait for groupversions to appear in v2 discovery
type waitForGroupVersionsV2 []metav1.GroupVersion
// Wait for groupversions to disappear from v2 discovery
type waitForAbsentGroupVersionsV2 []metav1.GroupVersion
// Assert something about the current state of v2 discovery
type inlineAction func(ctx context.Context, client testClient) error
func (a applyAPIService) Do(ctx context.Context, client testClient) error {
// using dynamic client since the typed client does not support `Apply`
// operation?
obj := &apiregistrationv1.APIService{
ObjectMeta: metav1.ObjectMeta{
Name: a.Version + "." + a.Group,
},
Spec: apiregistrationv1.APIServiceSpec(a),
}
unstructuredContent, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj)
if err != nil {
return err
}
unstructedObject := &unstructured.Unstructured{}
unstructedObject.SetUnstructuredContent(unstructuredContent)
unstructedObject.SetGroupVersionKind(apiregistrationv1.SchemeGroupVersion.WithKind("APIService"))
_, err = client.
Resource(apiregistrationv1.SchemeGroupVersion.WithResource("apiservices")).
Apply(ctx, obj.Name, unstructedObject, metav1.ApplyOptions{
FieldManager: "test-manager",
})
return err
}
func (a applyAPIService) Cleanup(ctx context.Context, client testClient) error {
name := a.Version + "." + a.Group
err := client.ApiregistrationV1().APIServices().Delete(ctx, name, metav1.DeleteOptions{})
if !errors.IsNotFound(err) {
return err
}
err = wait.PollWithContext(
ctx,
250*time.Millisecond,
1*time.Second,
func(ctx context.Context) (done bool, err error) {
_, err = client.ApiregistrationV1().APIServices().Get(ctx, name, metav1.GetOptions{})
if err == nil {
return false, nil
}
if !errors.IsNotFound(err) {
return false, err
}
return true, nil
},
)
if err != nil {
return fmt.Errorf("error waiting for APIService %v to clean up: %w", name, err)
}
return nil
}
func (a applyCRD) Do(ctx context.Context, client testClient) error {
// using dynamic client since the typed client does not support `Apply`
// operation?
name := a.Names.Plural + "." + a.Group
obj := &apiextensionsv1.CustomResourceDefinition{
ObjectMeta: metav1.ObjectMeta{
Name: name,
},
Spec: apiextensionsv1.CustomResourceDefinitionSpec(a),
}
unstructuredContent, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj)
if err != nil {
return err
}
unstructedObject := &unstructured.Unstructured{}
unstructedObject.SetUnstructuredContent(unstructuredContent)
unstructedObject.SetGroupVersionKind(apiextensionsv1.SchemeGroupVersion.WithKind("CustomResourceDefinition"))
_, err = client.
Resource(apiextensionsv1.SchemeGroupVersion.WithResource("customresourcedefinitions")).
Apply(ctx, obj.Name, unstructedObject, metav1.ApplyOptions{
FieldManager: "test-manager",
})
return err
}
func (a applyCRD) Cleanup(ctx context.Context, client testClient) error {
name := a.Names.Plural + "." + a.Group
err := client.ApiextensionsV1().CustomResourceDefinitions().Delete(ctx, name, metav1.DeleteOptions{})
if !errors.IsNotFound(err) {
return err
}
err = wait.PollWithContext(
ctx,
250*time.Millisecond,
1*time.Second,
func(ctx context.Context) (done bool, err error) {
_, err = client.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, name, metav1.GetOptions{})
if err == nil {
return false, nil
}
if !errors.IsNotFound(err) {
return false, err
}
return true, nil
},
)
if err != nil {
return fmt.Errorf("error waiting for CRD %v to clean up: %w", name, err)
}
return nil
}
func (d deleteObject) Do(ctx context.Context, client testClient) error {
if d.Namespace == "" {
return client.Resource(schema.GroupVersionResource(d.GroupVersionResource)).
Delete(ctx, d.Name, metav1.DeleteOptions{})
} else {
return client.Resource(schema.GroupVersionResource(d.GroupVersionResource)).
Namespace(d.Namespace).
Delete(ctx, d.Name, metav1.DeleteOptions{})
}
}
func (w waitForGroupVersionsV2) Do(ctx context.Context, client testClient) error {
err := WaitForResultWithCondition(ctx, client, func(result apidiscoveryv2beta1.APIGroupDiscoveryList) bool {
for _, gv := range w {
if FindGroupVersionV2(result, gv) == nil {
return false
}
}
return true
})
if err != nil {
return fmt.Errorf("waiting for groupversions v2 (%v): %w", w, err)
}
return nil
}
func (w waitForAbsentGroupVersionsV2) Do(ctx context.Context, client testClient) error {
err := WaitForResultWithCondition(ctx, client, func(result apidiscoveryv2beta1.APIGroupDiscoveryList) bool {
for _, gv := range w {
if FindGroupVersionV2(result, gv) != nil {
return false
}
}
return true
})
if err != nil {
return fmt.Errorf("waiting for absent groupversions v2 (%v): %w", w, err)
}
return nil
}
func (w waitForGroupVersionsV1) Do(ctx context.Context, client testClient) error {
err := WaitForV1GroupsWithCondition(ctx, client, func(result metav1.APIGroupList) bool {
for _, gv := range w {
if !FindGroupVersionV1(result, gv) {
return false
}
}
return true
})
if err != nil {
return fmt.Errorf("waiting for groupversions v1 (%v): %w", w, err)
}
return nil
}
func (w waitForAbsentGroupVersionsV1) Do(ctx context.Context, client testClient) error {
err := WaitForV1GroupsWithCondition(ctx, client, func(result metav1.APIGroupList) bool {
for _, gv := range w {
if FindGroupVersionV1(result, gv) {
return false
}
}
return true
})
if err != nil {
return fmt.Errorf("waiting for absent groupversions v1 (%v): %w", w, err)
}
return nil
}
func (i inlineAction) Do(ctx context.Context, client testClient) error {
return i(ctx, client)
}
func FetchV2Discovery(ctx context.Context, client testClient) (apidiscoveryv2beta1.APIGroupDiscoveryList, error) {
result, err := client.
Discovery().
RESTClient().
Get().
AbsPath("/apis").
SetHeader("Accept", acceptV2JSON).
Do(ctx).
Raw()
if err != nil {
return apidiscoveryv2beta1.APIGroupDiscoveryList{}, fmt.Errorf("failed to fetch v2 discovery: %w", err)
}
groupList := apidiscoveryv2beta1.APIGroupDiscoveryList{}
err = json.Unmarshal(result, &groupList)
if err != nil {
return apidiscoveryv2beta1.APIGroupDiscoveryList{}, fmt.Errorf("failed to parse v2 discovery: %w", err)
}
return groupList, nil
}
func FetchV1DiscoveryGroups(ctx context.Context, client testClient) (metav1.APIGroupList, error) {
return FetchV1DiscoveryGroupsAtPath(ctx, client, "/apis")
}
func FetchV1DiscoveryLegacyGroups(ctx context.Context, client testClient) (metav1.APIGroupList, error) {
return FetchV1DiscoveryGroupsAtPath(ctx, client, "/api")
}
func FetchV1DiscoveryGroupsAtPath(ctx context.Context, client testClient, path string) (metav1.APIGroupList, error) {
result, err := client.
Discovery().
RESTClient().
Get().
AbsPath("/apis").
SetHeader("Accept", acceptV1JSON).
Do(ctx).
Raw()
if err != nil {
return metav1.APIGroupList{}, fmt.Errorf("failed to fetch v1 discovery at %v: %w", path, err)
}
groupList := metav1.APIGroupList{}
err = json.Unmarshal(result, &groupList)
if err != nil {
return metav1.APIGroupList{}, fmt.Errorf("failed to parse v1 discovery at %v: %w", path, err)
}
return groupList, nil
}
func FetchV1DiscoveryResource(ctx context.Context, client testClient, gv metav1.GroupVersion) (metav1.APIGroupList, error) {
result, err := client.
Discovery().
RESTClient().
Get().
AbsPath("/apis/"+gv.Group+"/"+gv.Version).
SetHeader("Accept", acceptV1JSON).
Do(ctx).
Raw()
if err != nil {
return metav1.APIGroupList{}, err
}
groupList := metav1.APIGroupList{}
err = json.Unmarshal(result, &groupList)
if err != nil {
return metav1.APIGroupList{}, err
}
return groupList, nil
}
func WaitForGroupsAbsent(ctx context.Context, client testClient, groups ...string) error {
return WaitForResultWithCondition(ctx, client, func(groupList apidiscoveryv2beta1.APIGroupDiscoveryList) bool {
for _, searchGroup := range groups {
for _, docGroup := range groupList.Items {
if docGroup.Name == searchGroup {
return false
}
}
}
return true
})
}
func WaitForGroups(ctx context.Context, client testClient, groups ...apidiscoveryv2beta1.APIGroupDiscovery) error {
return WaitForResultWithCondition(ctx, client, func(groupList apidiscoveryv2beta1.APIGroupDiscoveryList) bool {
for _, searchGroup := range groups {
for _, docGroup := range groupList.Items {
if reflect.DeepEqual(searchGroup, docGroup) {
return true
}
}
}
return false
})
}
func WaitForResultWithCondition(ctx context.Context, client testClient, condition func(result apidiscoveryv2beta1.APIGroupDiscoveryList) bool) error {
// Keep repeatedly fetching document from aggregator.
// Check to see if it contains our service within a reasonable amount of time
return wait.PollWithContext(
ctx,
250*time.Millisecond,
1*time.Second,
func(ctx context.Context) (done bool, err error) {
groupList, err := FetchV2Discovery(ctx, client)
if err != nil {
return false, err
}
if condition(groupList) {
return true, nil
}
return false, nil
})
}
func WaitForV1GroupsWithCondition(ctx context.Context, client testClient, condition func(result metav1.APIGroupList) bool) error {
// Keep repeatedly fetching document from aggregator.
// Check to see if it contains our service within a reasonable amount of time
return wait.PollWithContext(
ctx,
250*time.Millisecond,
1*time.Second,
func(ctx context.Context) (done bool, err error) {
groupList, err := FetchV1DiscoveryGroups(ctx, client)
if err != nil {
return false, err
}
if condition(groupList) {
return true, nil
}
return false, nil
})
}
func FindGroupVersionV1(discovery metav1.APIGroupList, gv metav1.GroupVersion) bool {
for _, documentGroup := range discovery.Groups {
if documentGroup.Name != gv.Group {
continue
}
for _, documentVersion := range documentGroup.Versions {
if documentVersion.Version == gv.Version {
return true
}
}
}
return false
}
func FindGroupVersionV2(discovery apidiscoveryv2beta1.APIGroupDiscoveryList, gv metav1.GroupVersion) *apidiscoveryv2beta1.APIVersionDiscovery {
for _, documentGroup := range discovery.Items {
if documentGroup.Name != gv.Group {
continue
}
for _, documentVersion := range documentGroup.Versions {
if documentVersion.Version == gv.Version {
return &documentVersion
}
}
}
return nil
}

23
test/integration/apiserver/discovery/service.go
View File

@ -19,15 +19,14 @@ package discovery
import (
"context"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"net/url"
"strconv"
"sync"
"testing"
"time"
"github.com/stretchr/testify/require"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
@ -38,7 +37,7 @@ import (
)
type FakeService interface {
Start(t *testing.T, ctx context.Context)
Run(ctx context.Context) error
Port() *int32
Name() string
}
@ -62,7 +61,7 @@ func NewFakeService(name string, client kubernetes.Interface, handler http.Handl
}
}
func (f *fakeService) run(ctx context.Context) error {
func (f *fakeService) Run(ctx context.Context) error {
aggregatedServer := httptest.NewUnstartedServer(f.handler)
aggregatedServer.StartTLS()
defer aggregatedServer.Close()
@ -111,26 +110,24 @@ func (f *fakeService) run(ctx context.Context) error {
// Uninstall service from the cluser
err = f.client.CoreV1().Services("default").Delete(ctx, service.Name, metav1.DeleteOptions{})
return err
}
func (f *fakeService) Start(t *testing.T, ctx context.Context) {
go func() {
err := f.run(ctx)
if errors.Is(err, context.Canceled) {
err = nil
}
require.NoError(t, err)
}()
return err
}
func (f *fakeService) WaitForReady(ctx context.Context) error {
err := wait.PollWithContext(ctx, 1*time.Second, 200*time.Millisecond, func(ctx context.Context) (done bool, err error) {
return f.Port() != nil, nil
})
if errors.Is(err, context.Canceled) {
err = nil
} else if err != nil {
err = fmt.Errorf("service should have come alive in a reasonable amount of time: %w", err)
}
require.NoError(t, err, "service should have come alive in a reasonable amount of time")
return err
}
func (f *fakeService) Port() *int32 {