Refactor aggregated apiserver e2e, add openapi e2e

This commit is contained in:
Jefftree 2023-03-01 22:29:21 +00:00
parent ddb0d06744
commit 658ea4b591
3 changed files with 235 additions and 48 deletions

View File

@ -80,7 +80,7 @@ var _ = SIGDescribe("Aggregator", func() {
if err != nil {
framework.Failf("could not create aggregator client: %v", err)
}
ginkgo.DeferCleanup(cleanTest, f.ClientSet, aggrclient, f.Namespace.Name)
ginkgo.DeferCleanup(cleanupSampleAPIServer, f.ClientSet, aggrclient, generateSampleAPIServerObjectNames(f.Namespace.Name))
})
/*
@ -95,29 +95,40 @@ var _ = SIGDescribe("Aggregator", func() {
})
})
func cleanTest(ctx context.Context, client clientset.Interface, aggrclient *aggregatorclient.Clientset, namespace string) {
func cleanupSampleAPIServer(ctx context.Context, client clientset.Interface, aggrclient *aggregatorclient.Clientset, n sampleAPIServerObjectNames) {
// delete the APIService first to avoid causing discovery errors
_ = aggrclient.ApiregistrationV1().APIServices().Delete(ctx, "v1alpha1.wardle.example.com", metav1.DeleteOptions{})
_ = client.AppsV1().Deployments(namespace).Delete(ctx, "sample-apiserver-deployment", metav1.DeleteOptions{})
_ = client.CoreV1().Secrets(namespace).Delete(ctx, "sample-apiserver-secret", metav1.DeleteOptions{})
_ = client.CoreV1().Services(namespace).Delete(ctx, "sample-api", metav1.DeleteOptions{})
_ = client.CoreV1().ServiceAccounts(namespace).Delete(ctx, "sample-apiserver", metav1.DeleteOptions{})
_ = client.RbacV1().RoleBindings("kube-system").Delete(ctx, "wardler-auth-reader", metav1.DeleteOptions{})
_ = client.RbacV1().ClusterRoleBindings().Delete(ctx, "wardler:"+namespace+":auth-delegator", metav1.DeleteOptions{})
_ = client.RbacV1().ClusterRoles().Delete(ctx, "sample-apiserver-reader", metav1.DeleteOptions{})
_ = client.RbacV1().ClusterRoleBindings().Delete(ctx, "wardler:"+namespace+":sample-apiserver-reader", metav1.DeleteOptions{})
_ = client.AppsV1().Deployments(n.namespace).Delete(ctx, "sample-apiserver-deployment", metav1.DeleteOptions{})
_ = client.CoreV1().Secrets(n.namespace).Delete(ctx, "sample-apiserver-secret", metav1.DeleteOptions{})
_ = client.CoreV1().Services(n.namespace).Delete(ctx, "sample-api", metav1.DeleteOptions{})
_ = client.CoreV1().ServiceAccounts(n.namespace).Delete(ctx, "sample-apiserver", metav1.DeleteOptions{})
_ = client.RbacV1().RoleBindings("kube-system").Delete(ctx, n.roleBinding, metav1.DeleteOptions{})
_ = client.RbacV1().ClusterRoleBindings().Delete(ctx, "wardler:"+n.namespace+":auth-delegator", metav1.DeleteOptions{})
_ = client.RbacV1().ClusterRoles().Delete(ctx, n.clusterRole, metav1.DeleteOptions{})
_ = client.RbacV1().ClusterRoleBindings().Delete(ctx, n.clusterRoleBinding, metav1.DeleteOptions{})
}
// TestSampleAPIServer is a basic test if the sample-apiserver code from 1.10 and compiled against 1.10
// will work on the current Aggregator/API-Server.
func TestSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient *aggregatorclient.Clientset, image string) {
type sampleAPIServerObjectNames struct {
namespace string
roleBinding string
clusterRole string
clusterRoleBinding string
}
func generateSampleAPIServerObjectNames(namespace string) sampleAPIServerObjectNames {
return sampleAPIServerObjectNames{
namespace: namespace,
roleBinding: "wardler-auth-reader-" + namespace,
clusterRole: "sample-apiserver-reader-" + namespace,
clusterRoleBinding: "wardler:" + namespace + "sample-apiserver-reader-" + namespace,
}
}
func SetUpSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient *aggregatorclient.Clientset, image string, n sampleAPIServerObjectNames) {
ginkgo.By("Registering the sample API server.")
client := f.ClientSet
restClient := client.Discovery().RESTClient()
namespace := f.Namespace.Name
certCtx := setupServerCert(namespace, "sample-api")
certCtx := setupServerCert(n.namespace, "sample-api")
// kubectl create -f namespace.yaml
// NOTE: aggregated apis should generally be set up in their own namespace. As the test framework is setting up a new namespace, we are just using that.
@ -134,45 +145,45 @@ func TestSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient
"tls.key": certCtx.key,
},
}
_, err := client.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
framework.ExpectNoError(err, "creating secret %s in namespace %s", secretName, namespace)
_, err := client.CoreV1().Secrets(n.namespace).Create(ctx, secret, metav1.CreateOptions{})
framework.ExpectNoError(err, "creating secret %s in.namespace %s", secretName, n.namespace)
if e2eauth.IsRBACEnabled(ctx, client.RbacV1()) {
// kubectl create -f clusterrole.yaml
_, err = client.RbacV1().ClusterRoles().Create(ctx, &rbacv1.ClusterRole{
ObjectMeta: metav1.ObjectMeta{Name: "sample-apiserver-reader"},
ObjectMeta: metav1.ObjectMeta{Name: n.clusterRole},
Rules: []rbacv1.PolicyRule{
rbacv1helpers.NewRule("get", "list", "watch").Groups("").Resources("namespaces").RuleOrDie(),
rbacv1helpers.NewRule("get", "list", "watch").Groups("admissionregistration.k8s.io").Resources("*").RuleOrDie(),
},
}, metav1.CreateOptions{})
framework.ExpectNoError(err, "creating cluster role %s", "sample-apiserver-reader")
framework.ExpectNoError(err, "creating cluster role %s", n.clusterRole)
_, err = client.RbacV1().ClusterRoleBindings().Create(ctx, &rbacv1.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: "wardler:" + namespace + ":sample-apiserver-reader",
Name: n.clusterRoleBinding,
},
RoleRef: rbacv1.RoleRef{
APIGroup: "rbac.authorization.k8s.io",
Kind: "ClusterRole",
Name: "sample-apiserver-reader",
Name: n.clusterRole,
},
Subjects: []rbacv1.Subject{
{
APIGroup: "",
Kind: "ServiceAccount",
Name: "default",
Namespace: namespace,
Namespace: n.namespace,
},
},
}, metav1.CreateOptions{})
framework.ExpectNoError(err, "creating cluster role binding %s", "wardler:"+namespace+":sample-apiserver-reader")
framework.ExpectNoError(err, "creating cluster role binding %s", n.clusterRoleBinding)
// kubectl create -f authDelegator.yaml
_, err = client.RbacV1().ClusterRoleBindings().Create(ctx, &rbacv1.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: "wardler:" + namespace + ":auth-delegator",
Name: "wardler:" + n.namespace + ":auth-delegator",
},
RoleRef: rbacv1.RoleRef{
APIGroup: "rbac.authorization.k8s.io",
@ -184,11 +195,11 @@ func TestSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient
APIGroup: "",
Kind: "ServiceAccount",
Name: "default",
Namespace: namespace,
Namespace: n.namespace,
},
},
}, metav1.CreateOptions{})
framework.ExpectNoError(err, "creating cluster role binding %s", "wardler:"+namespace+":auth-delegator")
framework.ExpectNoError(err, "creating cluster role binding %s", "wardler:"+n.namespace+":auth-delegator")
}
// kubectl create -f deploy.yaml
@ -260,20 +271,20 @@ func TestSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient
d.Spec.Template.Spec.Containers = containers
d.Spec.Template.Spec.Volumes = volumes
deployment, err := client.AppsV1().Deployments(namespace).Create(ctx, d, metav1.CreateOptions{})
framework.ExpectNoError(err, "creating deployment %s in namespace %s", deploymentName, namespace)
deployment, err := client.AppsV1().Deployments(n.namespace).Create(ctx, d, metav1.CreateOptions{})
framework.ExpectNoError(err, "creating deployment %s in namespace %s", deploymentName, n.namespace)
err = e2edeployment.WaitForDeploymentRevisionAndImage(client, namespace, deploymentName, "1", image)
framework.ExpectNoError(err, "waiting for the deployment of image %s in %s in %s to complete", image, deploymentName, namespace)
err = e2edeployment.WaitForDeploymentRevisionAndImage(client, n.namespace, deploymentName, "1", image)
framework.ExpectNoError(err, "waiting for the deployment of image %s in %s in %s to complete", image, deploymentName, n.namespace)
err = e2edeployment.WaitForDeploymentRevisionAndImage(client, namespace, deploymentName, "1", etcdImage)
framework.ExpectNoError(err, "waiting for the deployment of image %s in %s in %s to complete", etcdImage, deploymentName, namespace)
err = e2edeployment.WaitForDeploymentRevisionAndImage(client, n.namespace, deploymentName, "1", etcdImage)
framework.ExpectNoError(err, "waiting for the deployment of image %s in %s in %s to complete", etcdImage, deploymentName, n.namespace)
// kubectl create -f service.yaml
serviceLabels := map[string]string{"apiserver": "true"}
service := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Namespace: namespace,
Namespace: n.namespace,
Name: "sample-api",
Labels: map[string]string{"test": "aggregator"},
},
@ -288,19 +299,19 @@ func TestSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient
},
},
}
_, err = client.CoreV1().Services(namespace).Create(ctx, service, metav1.CreateOptions{})
framework.ExpectNoError(err, "creating service %s in namespace %s", "sample-api", namespace)
_, err = client.CoreV1().Services(n.namespace).Create(ctx, service, metav1.CreateOptions{})
framework.ExpectNoError(err, "creating service %s in namespace %s", "sample-api", n.namespace)
// kubectl create -f serviceAccount.yaml
sa := &v1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "sample-apiserver"}}
_, err = client.CoreV1().ServiceAccounts(namespace).Create(ctx, sa, metav1.CreateOptions{})
framework.ExpectNoError(err, "creating service account %s in namespace %s", "sample-apiserver", namespace)
_, err = client.CoreV1().ServiceAccounts(n.namespace).Create(ctx, sa, metav1.CreateOptions{})
framework.ExpectNoError(err, "creating service account %s in namespace %s", "sample-apiserver", n.namespace)
if e2eauth.IsRBACEnabled(ctx, client.RbacV1()) {
// kubectl create -f auth-reader.yaml
_, err = client.RbacV1().RoleBindings("kube-system").Create(ctx, &rbacv1.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: "wardler-auth-reader",
Name: n.roleBinding,
Annotations: map[string]string{
rbacv1.AutoUpdateAnnotationKey: "true",
},
@ -314,11 +325,11 @@ func TestSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient
{
Kind: "ServiceAccount",
Name: "default",
Namespace: namespace,
Namespace: n.namespace,
},
},
}, metav1.CreateOptions{})
framework.ExpectNoError(err, "creating role binding %s in namespace %s", "wardler-auth-reader", "kube-system")
framework.ExpectNoError(err, "creating role binding %s in namespace %s", n.roleBinding, "kube-system")
}
// Wait for the extension apiserver to be up and healthy
@ -326,14 +337,14 @@ func TestSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient
// NOTE: aggregated apis should generally be set up in their own namespace (<aggregated-api-namespace>). As the test framework
// is setting up a new namespace, we are just using that.
err = e2edeployment.WaitForDeploymentComplete(client, deployment)
framework.ExpectNoError(err, "deploying extension apiserver in namespace %s", namespace)
framework.ExpectNoError(err, "deploying extension apiserver in namespace %s", n.namespace)
// kubectl create -f apiservice.yaml
_, err = aggrclient.ApiregistrationV1().APIServices().Create(ctx, &apiregistrationv1.APIService{
ObjectMeta: metav1.ObjectMeta{Name: "v1alpha1.wardle.example.com"},
Spec: apiregistrationv1.APIServiceSpec{
Service: &apiregistrationv1.ServiceReference{
Namespace: namespace,
Namespace: n.namespace,
Name: "sample-api",
Port: pointer.Int32(aggregatorServicePort),
},
@ -354,7 +365,7 @@ func TestSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient
err = pollTimed(ctx, 100*time.Millisecond, 60*time.Second, func(ctx context.Context) (bool, error) {
currentAPIService, _ = aggrclient.ApiregistrationV1().APIServices().Get(ctx, "v1alpha1.wardle.example.com", metav1.GetOptions{})
currentPods, _ = client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
currentPods, _ = client.CoreV1().Pods(n.namespace).List(ctx, metav1.ListOptions{})
request := restClient.Get().AbsPath("/apis/wardle.example.com/v1alpha1/namespaces/default/flunders")
request.SetHeader("Accept", "application/json")
@ -384,13 +395,22 @@ func TestSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient
if currentPods != nil {
for _, pod := range currentPods.Items {
for _, container := range pod.Spec.Containers {
logs, err := e2epod.GetPodLogs(ctx, client, namespace, pod.Name, container.Name)
logs, err := e2epod.GetPodLogs(ctx, client, n.namespace, pod.Name, container.Name)
framework.Logf("logs of %s/%s (error: %v): %s", pod.Name, container.Name, err, logs)
}
}
}
}
framework.ExpectNoError(err, "gave up waiting for apiservice wardle to come up successfully")
}
// TestSampleAPIServer is a basic test if the sample-apiserver code from 1.10 and compiled against 1.10
// will work on the current Aggregator/API-Server.
func TestSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient *aggregatorclient.Clientset, image string) {
n := generateSampleAPIServerObjectNames(f.Namespace.Name)
SetUpSampleAPIServer(ctx, f, aggrclient, image, n)
client := f.ClientSet
restClient := client.Discovery().RESTClient()
flunderName := generateFlunderName("rest-flunder")
@ -413,7 +433,7 @@ func TestSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient
framework.ExpectEqual(u.GetKind(), "Flunder")
framework.ExpectEqual(u.GetName(), flunderName)
pods, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
pods, err := client.CoreV1().Pods(n.namespace).List(ctx, metav1.ListOptions{})
framework.ExpectNoError(err, "getting pods for flunders service")
// kubectl get flunders -v 9
@ -453,7 +473,7 @@ func TestSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient
if !ok {
framework.Failf("could not find group version resource for dynamic client and wardle/flunders (discovery error: %v, discovery results: %#v)", discoveryErr, groupVersionResources)
}
dynamicClient := f.DynamicClient.Resource(gvr).Namespace(namespace)
dynamicClient := f.DynamicClient.Resource(gvr).Namespace(n.namespace)
// kubectl create -f flunders-1.yaml
// Request Body: {"apiVersion":"wardle.example.com/v1alpha1","kind":"Flunder","metadata":{"labels":{"sample-label":"true"},"name":"test-flunder","namespace":"default"}}
@ -703,7 +723,7 @@ func TestSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient
framework.ExpectNoError(err, "failed to count the required APIServices")
framework.Logf("APIService %s has been deleted.", apiServiceName)
cleanTest(ctx, client, aggrclient, namespace)
cleanupSampleAPIServer(ctx, client, aggrclient, n)
}
// pollTimed will call Poll but time how long Poll actually took.

View File

@ -0,0 +1,166 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package apimachinery
import (
"context"
"encoding/json"
"reflect"
"strings"
"time"
"github.com/google/go-cmp/cmp"
"github.com/onsi/ginkgo/v2"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apiextensionclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
"k8s.io/apiextensions-apiserver/test/integration/fixtures"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/openapi3"
aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
"k8s.io/kube-openapi/pkg/spec3"
imageutils "k8s.io/kubernetes/test/utils/image"
admissionapi "k8s.io/pod-security-admission/api"
"k8s.io/kubernetes/test/e2e/framework"
// ensure libs have a chance to initialize
_ "github.com/stretchr/testify/assert"
)
var _ = SIGDescribe("OpenAPIV3", func() {
f := framework.NewDefaultFramework("openapiv3")
f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline
/*
Release : v1.27
Testname: OpenAPI V3 RoundTrip
Description: Fetch the OpenAPI v3 of all built-in group versions. The OpenAPI specs MUST roundtrip successfully.
*/
ginkgo.It("should round trip OpenAPI V3 for all built-in group versions", func(ctx context.Context) {
c := openapi3.NewRoot(f.ClientSet.Discovery().OpenAPIV3())
gvs, err := c.GroupVersions()
framework.ExpectNoError(err)
// List of built in types that do not contain the k8s.io suffix
builtinGVs := map[string]bool{
"apps": true,
"autoscaling": true,
"batch": true,
"policy": true,
}
for _, gv := range gvs {
// Prevent race conditions with looking up gvs of CRDs and
// other aggregated apiservers added by other tests
if !strings.HasSuffix(gv.Group, "k8s.io") && !builtinGVs[gv.Group] {
continue
}
spec1, err := c.GVSpec(gv)
framework.ExpectNoError(err)
specMarshalled, err := json.Marshal(spec1)
framework.ExpectNoError(err)
var spec2 spec3.OpenAPI
json.Unmarshal(specMarshalled, &spec2)
if !reflect.DeepEqual(*spec1, spec2) {
diff := cmp.Diff(*spec1, spec2)
framework.Failf("%s", diff)
}
}
})
/*
Release : v1.27
Testname: OpenAPI V3 CustomResourceDefinition
Description: Create a CustomResourceDefinition. The OpenAPI V3 document of the CustomResourceDefinition MUST be created. The OpenAPI V3 MUST be round trippable.
*/
ginkgo.It("should publish OpenAPI V3 for CustomResourceDefinition", func(ctx context.Context) {
config, err := framework.LoadConfig()
framework.ExpectNoError(err)
apiExtensionClient, err := apiextensionclientset.NewForConfig(config)
framework.ExpectNoError(err)
dynamicClient, err := dynamic.NewForConfig(config)
framework.ExpectNoError(err)
crd := fixtures.NewRandomNameV1CustomResourceDefinition(apiextensionsv1.NamespaceScoped)
gv := schema.GroupVersion{Group: crd.Spec.Group, Version: crd.Spec.Versions[0].Name}
_, err = fixtures.CreateNewV1CustomResourceDefinition(crd, apiExtensionClient, dynamicClient)
defer func() {
err = fixtures.DeleteV1CustomResourceDefinition(crd, apiExtensionClient)
framework.ExpectNoError(err, "deleting CustomResourceDefinition")
}()
framework.ExpectNoError(err)
c := openapi3.NewRoot(f.ClientSet.Discovery().OpenAPIV3())
var openAPISpec *spec3.OpenAPI
// Poll for the OpenAPI to be updated with the new CRD
wait.Poll(time.Second*1, wait.ForeverTestTimeout, func() (bool, error) {
openAPISpec, err = c.GVSpec(gv)
if err == nil {
return true, nil
}
return false, nil
})
specMarshalled, err := json.Marshal(openAPISpec)
framework.ExpectNoError(err)
var spec2 spec3.OpenAPI
json.Unmarshal(specMarshalled, &spec2)
if !reflect.DeepEqual(*openAPISpec, spec2) {
diff := cmp.Diff(*openAPISpec, spec2)
framework.Failf("%s", diff)
}
})
/*
Release : v1.27
Testname: OpenAPI V3 Aggregated APIServer
Description: Create an Aggregated APIServer. The OpenAPI V3 for the aggregated apiserver MUST be aggregated by the aggregator and published. The specification MUST be round trippable.
*/
ginkgo.It("should contain OpenAPI V3 for Aggregated APIServer", func(ctx context.Context) {
config, err := framework.LoadConfig()
framework.ExpectNoError(err)
aggrclient, err := aggregatorclient.NewForConfig(config)
framework.ExpectNoError(err)
names := generateSampleAPIServerObjectNames(f.Namespace.Name)
SetUpSampleAPIServer(ctx, f, aggrclient, imageutils.GetE2EImage(imageutils.APIServer), names)
defer cleanupSampleAPIServer(ctx, f.ClientSet, aggrclient, names)
c := openapi3.NewRoot(f.ClientSet.Discovery().OpenAPIV3())
gv := schema.GroupVersion{Group: "wardle.example.com", Version: "v1alpha1"}
var openAPISpec *spec3.OpenAPI
// Poll for the OpenAPI to be updated with the new aggregated apiserver.
wait.Poll(time.Second*1, wait.ForeverTestTimeout, func() (bool, error) {
openAPISpec, err = c.GVSpec(gv)
if err == nil {
return true, nil
}
return false, nil
})
specMarshalled, err := json.Marshal(openAPISpec)
framework.ExpectNoError(err)
var spec2 spec3.OpenAPI
json.Unmarshal(specMarshalled, &spec2)
if !reflect.DeepEqual(*openAPISpec, spec2) {
diff := cmp.Diff(*openAPISpec, spec2)
framework.Failf("%s", diff)
}
})
})

1
vendor/modules.txt vendored
View File

@ -1891,6 +1891,7 @@ k8s.io/client-go/metadata/metadatainformer
k8s.io/client-go/metadata/metadatalister
k8s.io/client-go/openapi
k8s.io/client-go/openapi/cached
k8s.io/client-go/openapi3
k8s.io/client-go/pkg/apis/clientauthentication
k8s.io/client-go/pkg/apis/clientauthentication/install
k8s.io/client-go/pkg/apis/clientauthentication/v1