From a02e94ae2b4e8965a60e8b4b83836d4ceb95aa2c Mon Sep 17 00:00:00 2001 From: TommyStarK Date: Tue, 18 Apr 2023 21:23:30 +0200 Subject: [PATCH] e2e: fix flaky test 'should contain OpenAPI V3 for Aggregated APIServer' Signed-off-by: TommyStarK --- test/e2e/apimachinery/aggregator.go | 108 +++++++++++++++------------- test/e2e/apimachinery/openapiv3.go | 7 +- 2 files changed, 61 insertions(+), 54 deletions(-) diff --git a/test/e2e/apimachinery/aggregator.go b/test/e2e/apimachinery/aggregator.go index 3058e0169df..6c36d81b6d0 100644 --- a/test/e2e/apimachinery/aggregator.go +++ b/test/e2e/apimachinery/aggregator.go @@ -60,6 +60,9 @@ const ( apiServiceRetryPeriod = 1 * time.Second apiServiceRetryTimeout = 2 * time.Minute + + defaultApiServiceGroupName = samplev1alpha1.GroupName + defaultApiServiceVersion = "v1alpha1" ) var _ = SIGDescribe("Aggregator", func() { @@ -80,24 +83,25 @@ var _ = SIGDescribe("Aggregator", func() { if err != nil { framework.Failf("could not create aggregator client: %v", err) } - ginkgo.DeferCleanup(cleanupSampleAPIServer, f.ClientSet, aggrclient, generateSampleAPIServerObjectNames(f.Namespace.Name)) + apiServiceName := defaultApiServiceVersion + "." + defaultApiServiceGroupName + ginkgo.DeferCleanup(cleanupSampleAPIServer, f.ClientSet, aggrclient, generateSampleAPIServerObjectNames(f.Namespace.Name), apiServiceName) }) /* - Release: v1.17, v1.21, v1.27 - Testname: aggregator-supports-the-sample-apiserver - Description: Ensure that the sample-apiserver code from 1.17 and compiled against 1.17 - will work on the current Aggregator/API-Server. + Release: v1.17, v1.21, v1.27 + Testname: aggregator-supports-the-sample-apiserver + Description: Ensure that the sample-apiserver code from 1.17 and compiled against 1.17 + will work on the current Aggregator/API-Server. */ framework.ConformanceIt("Should be able to support the 1.17 Sample API Server using the current Aggregator", func(ctx context.Context) { // Testing a 1.17 version of the sample-apiserver - TestSampleAPIServer(ctx, f, aggrclient, imageutils.GetE2EImage(imageutils.APIServer)) + TestSampleAPIServer(ctx, f, aggrclient, imageutils.GetE2EImage(imageutils.APIServer), defaultApiServiceGroupName, defaultApiServiceVersion) }) }) -func cleanupSampleAPIServer(ctx context.Context, client clientset.Interface, aggrclient *aggregatorclient.Clientset, n sampleAPIServerObjectNames) { +func cleanupSampleAPIServer(ctx context.Context, client clientset.Interface, aggrclient *aggregatorclient.Clientset, n sampleAPIServerObjectNames, apiServiceName string) { // delete the APIService first to avoid causing discovery errors - _ = aggrclient.ApiregistrationV1().APIServices().Delete(ctx, "v1alpha1.wardle.example.com", metav1.DeleteOptions{}) + _ = aggrclient.ApiregistrationV1().APIServices().Delete(ctx, apiServiceName, metav1.DeleteOptions{}) _ = client.AppsV1().Deployments(n.namespace).Delete(ctx, "sample-apiserver-deployment", metav1.DeleteOptions{}) _ = client.CoreV1().Secrets(n.namespace).Delete(ctx, "sample-apiserver-secret", metav1.DeleteOptions{}) @@ -124,11 +128,13 @@ func generateSampleAPIServerObjectNames(namespace string) sampleAPIServerObjectN clusterRoleBinding: "wardler:" + namespace + "sample-apiserver-reader-" + namespace, } } -func SetUpSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient *aggregatorclient.Clientset, image string, n sampleAPIServerObjectNames) { + +func SetUpSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient *aggregatorclient.Clientset, image string, n sampleAPIServerObjectNames, apiServiceGroupName, apiServiceVersion string) { ginkgo.By("Registering the sample API server.") client := f.ClientSet restClient := client.Discovery().RESTClient() certCtx := setupServerCert(n.namespace, "sample-api") + apiServiceName := apiServiceVersion + "." + apiServiceGroupName // kubectl create -f namespace.yaml // NOTE: aggregated apis should generally be set up in their own namespace. As the test framework is setting up a new namespace, we are just using that. @@ -341,21 +347,21 @@ func SetUpSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclien // kubectl create -f apiservice.yaml _, err = aggrclient.ApiregistrationV1().APIServices().Create(ctx, &apiregistrationv1.APIService{ - ObjectMeta: metav1.ObjectMeta{Name: "v1alpha1.wardle.example.com"}, + ObjectMeta: metav1.ObjectMeta{Name: apiServiceName}, Spec: apiregistrationv1.APIServiceSpec{ Service: &apiregistrationv1.ServiceReference{ Namespace: n.namespace, Name: "sample-api", Port: pointer.Int32(aggregatorServicePort), }, - Group: "wardle.example.com", - Version: "v1alpha1", + Group: apiServiceGroupName, + Version: apiServiceVersion, CABundle: certCtx.signingCert, GroupPriorityMinimum: 2000, VersionPriority: 200, }, }, metav1.CreateOptions{}) - framework.ExpectNoError(err, "creating apiservice %s", "v1alpha1.wardle.example.com") + framework.ExpectNoError(err, "creating apiservice %s", apiServiceName) var ( currentAPIService *apiregistrationv1.APIService @@ -364,10 +370,10 @@ func SetUpSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclien err = pollTimed(ctx, 100*time.Millisecond, 60*time.Second, func(ctx context.Context) (bool, error) { - currentAPIService, _ = aggrclient.ApiregistrationV1().APIServices().Get(ctx, "v1alpha1.wardle.example.com", metav1.GetOptions{}) + currentAPIService, _ = aggrclient.ApiregistrationV1().APIServices().Get(ctx, apiServiceName, metav1.GetOptions{}) currentPods, _ = client.CoreV1().Pods(n.namespace).List(ctx, metav1.ListOptions{}) - request := restClient.Get().AbsPath("/apis/wardle.example.com/v1alpha1/namespaces/default/flunders") + request := restClient.Get().AbsPath("/apis/" + apiServiceGroupName + "/" + apiServiceVersion + "/namespaces/default/flunders") request.SetHeader("Accept", "application/json") _, err := request.DoRaw(ctx) if err != nil { @@ -406,19 +412,20 @@ func SetUpSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclien // TestSampleAPIServer is a basic test if the sample-apiserver code from 1.10 and compiled against 1.10 // will work on the current Aggregator/API-Server. -func TestSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient *aggregatorclient.Clientset, image string) { +func TestSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient *aggregatorclient.Clientset, image, apiServiceGroupName, apiServiceVersion string) { n := generateSampleAPIServerObjectNames(f.Namespace.Name) - SetUpSampleAPIServer(ctx, f, aggrclient, image, n) + SetUpSampleAPIServer(ctx, f, aggrclient, image, n, apiServiceGroupName, apiServiceVersion) client := f.ClientSet restClient := client.Discovery().RESTClient() flunderName := generateFlunderName("rest-flunder") + apiServiceName := apiServiceVersion + "." + apiServiceGroupName // kubectl create -f flunders-1.yaml -v 9 // curl -k -v -XPOST https://localhost/apis/wardle.example.com/v1alpha1/namespaces/default/flunders // Request Body: {"apiVersion":"wardle.example.com/v1alpha1","kind":"Flunder","metadata":{"labels":{"sample-label":"true"},"name":"test-flunder","namespace":"default"}} - flunder := `{"apiVersion":"wardle.example.com/v1alpha1","kind":"Flunder","metadata":{"labels":{"sample-label":"true"},"name":"` + flunderName + `","namespace":"default"}}` - result := restClient.Post().AbsPath("/apis/wardle.example.com/v1alpha1/namespaces/default/flunders").Body([]byte(flunder)).SetHeader("Accept", "application/json").Do(ctx) + flunder := `{"apiVersion":"` + apiServiceGroupName + `/` + apiServiceVersion + `","kind":"Flunder","metadata":{"labels":{"sample-label":"true"},"name":"` + flunderName + `","namespace":"default"}}` + result := restClient.Post().AbsPath("/apis/"+apiServiceGroupName+"/"+apiServiceVersion+"/namespaces/default/flunders").Body([]byte(flunder)).SetHeader("Accept", "application/json").Do(ctx) framework.ExpectNoError(result.Error(), "creating a new flunders resource") var statusCode int result.StatusCode(&statusCode) @@ -429,7 +436,7 @@ func TestSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient if err := result.Into(u); err != nil { framework.ExpectNoError(err, "reading created response") } - framework.ExpectEqual(u.GetAPIVersion(), "wardle.example.com/v1alpha1") + framework.ExpectEqual(u.GetAPIVersion(), apiServiceGroupName+"/"+apiServiceVersion) framework.ExpectEqual(u.GetKind(), "Flunder") framework.ExpectEqual(u.GetName(), flunderName) @@ -438,26 +445,26 @@ func TestSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient // kubectl get flunders -v 9 // curl -k -v -XGET https://localhost/apis/wardle.example.com/v1alpha1/namespaces/default/flunders - contents, err := restClient.Get().AbsPath("/apis/wardle.example.com/v1alpha1/namespaces/default/flunders").SetHeader("Accept", "application/json").DoRaw(ctx) + contents, err := restClient.Get().AbsPath("/apis/"+apiServiceGroupName+"/"+apiServiceVersion+"/namespaces/default/flunders").SetHeader("Accept", "application/json").DoRaw(ctx) framework.ExpectNoError(err, "attempting to get a newly created flunders resource") var flundersList samplev1alpha1.FlunderList err = json.Unmarshal(contents, &flundersList) - validateErrorWithDebugInfo(ctx, f, err, pods, "Error in unmarshalling %T response from server %s", contents, "/apis/wardle.example.com/v1alpha1") + validateErrorWithDebugInfo(ctx, f, err, pods, "Error in unmarshalling %T response from server %s", contents, "/apis/"+apiServiceGroupName+"/"+apiServiceVersion) if len(flundersList.Items) != 1 { framework.Failf("failed to get back the correct flunders list %v", flundersList) } // kubectl delete flunder test-flunder -v 9 // curl -k -v -XDELETE https://35.193.112.40/apis/wardle.example.com/v1alpha1/namespaces/default/flunders/test-flunder - _, err = restClient.Delete().AbsPath("/apis/wardle.example.com/v1alpha1/namespaces/default/flunders/" + flunderName).DoRaw(ctx) + _, err = restClient.Delete().AbsPath("/apis/" + apiServiceGroupName + "/" + apiServiceVersion + "/namespaces/default/flunders/" + flunderName).DoRaw(ctx) validateErrorWithDebugInfo(ctx, f, err, pods, "attempting to delete a newly created flunders(%v) resource", flundersList.Items) // kubectl get flunders -v 9 // curl -k -v -XGET https://localhost/apis/wardle.example.com/v1alpha1/namespaces/default/flunders - contents, err = restClient.Get().AbsPath("/apis/wardle.example.com/v1alpha1/namespaces/default/flunders").SetHeader("Accept", "application/json").DoRaw(ctx) + contents, err = restClient.Get().AbsPath("/apis/"+apiServiceGroupName+"/"+apiServiceVersion+"/namespaces/default/flunders").SetHeader("Accept", "application/json").DoRaw(ctx) framework.ExpectNoError(err, "confirming delete of a newly created flunders resource") err = json.Unmarshal(contents, &flundersList) - validateErrorWithDebugInfo(ctx, f, err, pods, "Error in unmarshalling %T response from server %s", contents, "/apis/wardle.example.com/v1alpha1") + validateErrorWithDebugInfo(ctx, f, err, pods, "Error in unmarshalling %T response from server %s", contents, "/apis/"+apiServiceGroupName+"/"+apiServiceVersion) if len(flundersList.Items) != 0 { framework.Failf("failed to get back the correct deleted flunders list %v", flundersList) } @@ -468,7 +475,7 @@ func TestSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient resources, discoveryErr := client.Discovery().ServerPreferredNamespacedResources() groupVersionResources, err := discovery.GroupVersionResources(resources) framework.ExpectNoError(err, "getting group version resources for dynamic client") - gvr := schema.GroupVersionResource{Group: "wardle.example.com", Version: "v1alpha1", Resource: "flunders"} + gvr := schema.GroupVersionResource{Group: apiServiceGroupName, Version: apiServiceVersion, Resource: "flunders"} _, ok := groupVersionResources[gvr] if !ok { framework.Failf("could not find group version resource for dynamic client and wardle/flunders (discovery error: %v, discovery results: %#v)", discoveryErr, groupVersionResources) @@ -480,7 +487,7 @@ func TestSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient testFlunder := samplev1alpha1.Flunder{ TypeMeta: metav1.TypeMeta{ Kind: "Flunder", - APIVersion: "wardle.example.com/v1alpha1", + APIVersion: apiServiceGroupName + "/" + apiServiceVersion, }, ObjectMeta: metav1.ObjectMeta{Name: flunderName}, Spec: samplev1alpha1.FlunderSpec{}, @@ -500,24 +507,24 @@ func TestSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient framework.Failf("failed to get back the correct flunders list %v from the dynamic client", unstructuredList) } - ginkgo.By("Read Status for v1alpha1.wardle.example.com") + ginkgo.By("Read Status for " + apiServiceName) statusContent, err := restClient.Get(). - AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/v1alpha1.wardle.example.com/status"). + AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/"+apiServiceName+"/status"). SetHeader("Accept", "application/json").DoRaw(ctx) - framework.ExpectNoError(err, "No response for .../apiservices/v1alpha1.wardle.example.com/status. Error: %v", err) + framework.ExpectNoError(err, "No response for .../apiservices/"+apiServiceName+"/status. Error: %v", err) var jr *apiregistrationv1.APIService err = json.Unmarshal([]byte(statusContent), &jr) framework.ExpectNoError(err, "Failed to process statusContent: %v | err: %v ", string(statusContent), err) framework.ExpectEqual(jr.Status.Conditions[0].Message, "all checks passed", "The Message returned was %v", jr.Status.Conditions[0].Message) - ginkgo.By("kubectl patch apiservice v1alpha1.wardle.example.com -p '{\"spec\":{\"versionPriority\": 400}}'") + ginkgo.By("kubectl patch apiservice " + apiServiceName + " -p '{\"spec\":{\"versionPriority\": 400}}'") patchContent, err := restClient.Patch(types.MergePatchType). - AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/v1alpha1.wardle.example.com"). + AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/"+apiServiceName). SetHeader("Accept", "application/json"). Body([]byte(`{"spec":{"versionPriority": 400}}`)).DoRaw(ctx) - framework.ExpectNoError(err, "Patch failed for .../apiservices/v1alpha1.wardle.example.com. Error: %v", err) + framework.ExpectNoError(err, "Patch failed for .../apiservices/"+apiServiceName+". Error: %v", err) err = json.Unmarshal([]byte(patchContent), &jr) framework.ExpectNoError(err, "Failed to process patchContent: %v | err: %v ", string(patchContent), err) framework.ExpectEqual(jr.Spec.VersionPriority, int32(400), "The VersionPriority returned was %d", jr.Spec.VersionPriority) @@ -535,20 +542,19 @@ func TestSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient locatedWardle := false for _, item := range list.Items { - if item.Name == "v1alpha1.wardle.example.com" { - framework.Logf("Found v1alpha1.wardle.example.com in APIServiceList") + if item.Name == apiServiceName { + framework.Logf("Found " + apiServiceName + " in APIServiceList") locatedWardle = true break } } if !locatedWardle { - framework.Failf("Unable to find v1alpha1.wardle.example.com in APIServiceList") + framework.Failf("Unable to find " + apiServiceName + " in APIServiceList") } // As the APIService doesn't have any labels currently set we need to // set one so that we can select it later when we call deleteCollection ginkgo.By("Adding a label to the APIService") - apiServiceName := "v1alpha1.wardle.example.com" apiServiceClient := aggrclient.ApiregistrationV1().APIServices() apiServiceLabel := map[string]string{"e2e-apiservice": "patched"} apiServiceLabelSelector := labels.SelectorFromSet(apiServiceLabel).String() @@ -571,9 +577,9 @@ func TestSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient err = retry.RetryOnConflict(retry.DefaultRetry, func() error { var statusToUpdate *apiregistrationv1.APIService statusContent, err = restClient.Get(). - AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/v1alpha1.wardle.example.com/status"). + AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/"+apiServiceName+"/status"). SetHeader("Accept", "application/json").DoRaw(ctx) - framework.ExpectNoError(err, "No response for .../apiservices/v1alpha1.wardle.example.com/status. Error: %v", err) + framework.ExpectNoError(err, "No response for .../apiservices/"+apiServiceName+"/status. Error: %v", err) err = json.Unmarshal([]byte(statusContent), &statusToUpdate) framework.ExpectNoError(err, "Failed to process statusContent: %v | err: %v ", string(statusContent), err) @@ -591,11 +597,11 @@ func TestSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient framework.ExpectNoError(err, "Failed to update status. %v", err) framework.Logf("updatedStatus.Conditions: %#v", updatedStatus.Status.Conditions) - ginkgo.By("Confirm that v1alpha1.wardle.example.com /status was updated") + ginkgo.By("Confirm that " + apiServiceName + " /status was updated") statusContent, err = restClient.Get(). - AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/v1alpha1.wardle.example.com/status"). + AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/"+apiServiceName+"/status"). SetHeader("Accept", "application/json").DoRaw(ctx) - framework.ExpectNoError(err, "No response for .../apiservices/v1alpha1.wardle.example.com/status. Error: %v", err) + framework.ExpectNoError(err, "No response for .../apiservices/"+apiServiceName+"/status. Error: %v", err) err = json.Unmarshal([]byte(statusContent), &wardle) framework.ExpectNoError(err, "Failed to process statusContent: %v | err: %v ", string(statusContent), err) @@ -657,11 +663,11 @@ func TestSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient framework.Failf("failed to get back the correct flunders list %v from the dynamic client", unstructuredList) } - ginkgo.By("Read v1alpha1.wardle.example.com /status before patching it") + ginkgo.By("Read " + apiServiceName + " /status before patching it") statusContent, err = restClient.Get(). - AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/v1alpha1.wardle.example.com/status"). + AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/"+apiServiceName+"/status"). SetHeader("Accept", "application/json").DoRaw(ctx) - framework.ExpectNoError(err, "No response for .../apiservices/v1alpha1.wardle.example.com/status. Error: %v", err) + framework.ExpectNoError(err, "No response for .../apiservices/"+apiServiceName+"/status. Error: %v", err) wardle.Reset() err = json.Unmarshal([]byte(statusContent), &wardle) @@ -682,17 +688,17 @@ func TestSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient framework.ExpectNoError(err, "Failed to marshal JSON. %v", err) _, err = restClient.Patch(types.MergePatchType). - AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/v1alpha1.wardle.example.com/status"). + AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/"+apiServiceName+"/status"). SetHeader("Accept", "application/json"). Body([]byte(payload)). DoRaw(ctx) - framework.ExpectNoError(err, "Patch failed for .../apiservices/v1alpha1.wardle.example.com/status. Error: %v", err) + framework.ExpectNoError(err, "Patch failed for .../apiservices/"+apiServiceName+"/status. Error: %v", err) - ginkgo.By("Confirm that v1alpha1.wardle.example.com /status was patched") + ginkgo.By("Confirm that " + apiServiceName + " /status was patched") statusContent, err = restClient.Get(). - AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/v1alpha1.wardle.example.com/status"). + AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/"+apiServiceName+"/status"). SetHeader("Accept", "application/json").DoRaw(ctx) - framework.ExpectNoError(err, "No response for .../apiservices/v1alpha1.wardle.example.com/status. Error: %v", err) + framework.ExpectNoError(err, "No response for .../apiservices/"+apiServiceName+"/status. Error: %v", err) wardle.Reset() err = json.Unmarshal([]byte(statusContent), &wardle) @@ -723,7 +729,7 @@ func TestSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient framework.ExpectNoError(err, "failed to count the required APIServices") framework.Logf("APIService %s has been deleted.", apiServiceName) - cleanupSampleAPIServer(ctx, client, aggrclient, n) + cleanupSampleAPIServer(ctx, client, aggrclient, n, apiServiceName) } // pollTimed will call Poll but time how long Poll actually took. diff --git a/test/e2e/apimachinery/openapiv3.go b/test/e2e/apimachinery/openapiv3.go index 0c087957628..4fbadd1023f 100644 --- a/test/e2e/apimachinery/openapiv3.go +++ b/test/e2e/apimachinery/openapiv3.go @@ -36,6 +36,7 @@ import ( "k8s.io/kube-openapi/pkg/spec3" imageutils "k8s.io/kubernetes/test/utils/image" admissionapi "k8s.io/pod-security-admission/api" + samplev1beta1 "k8s.io/sample-apiserver/pkg/apis/wardle/v1beta1" "k8s.io/kubernetes/test/e2e/framework" @@ -138,11 +139,11 @@ var _ = SIGDescribe("OpenAPIV3", func() { aggrclient, err := aggregatorclient.NewForConfig(config) framework.ExpectNoError(err) names := generateSampleAPIServerObjectNames(f.Namespace.Name) - SetUpSampleAPIServer(ctx, f, aggrclient, imageutils.GetE2EImage(imageutils.APIServer), names) - defer cleanupSampleAPIServer(ctx, f.ClientSet, aggrclient, names) + SetUpSampleAPIServer(ctx, f, aggrclient, imageutils.GetE2EImage(imageutils.APIServer), names, samplev1beta1.GroupName, "v1beta1") + defer cleanupSampleAPIServer(ctx, f.ClientSet, aggrclient, names, "v1beta1.wardle.example.com") c := openapi3.NewRoot(f.ClientSet.Discovery().OpenAPIV3()) - gv := schema.GroupVersion{Group: "wardle.example.com", Version: "v1alpha1"} + gv := schema.GroupVersion{Group: samplev1beta1.GroupName, Version: "v1beta1"} var openAPISpec *spec3.OpenAPI // Poll for the OpenAPI to be updated with the new aggregated apiserver. wait.Poll(time.Second*1, wait.ForeverTestTimeout, func() (bool, error) {