From 4625bf7dadb1ecc1d87c0f5d0c2ea22a7d698e41 Mon Sep 17 00:00:00 2001 From: Haowei Cai Date: Thu, 15 Nov 2018 14:18:27 -0800 Subject: [PATCH] add basic e2e test and integration for CRD openapi --- .../custom_resource_definition.go | 59 +++++++++++++ test/integration/master/BUILD | 1 + test/integration/master/crd_test.go | 88 +++++++++++++++++++ 3 files changed, 148 insertions(+) diff --git a/test/e2e/apimachinery/custom_resource_definition.go b/test/e2e/apimachinery/custom_resource_definition.go index fb0469487e6..b282bc184cf 100644 --- a/test/e2e/apimachinery/custom_resource_definition.go +++ b/test/e2e/apimachinery/custom_resource_definition.go @@ -17,16 +17,22 @@ limitations under the License. package apimachinery import ( + "fmt" + "strings" + "time" + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/apiextensions-apiserver/test/integration/fixtures" utilversion "k8s.io/apimachinery/pkg/util/version" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/kubernetes/test/e2e/framework" . "github.com/onsi/ginkgo" ) var crdVersion = utilversion.MustParseSemantic("v1.7.0") +var crdOpenAPIVersion = utilversion.MustParseSemantic("v1.13.0") var _ = SIGDescribe("CustomResourceDefinition resources", func() { @@ -67,5 +73,58 @@ var _ = SIGDescribe("CustomResourceDefinition resources", func() { } }() }) + + It("has OpenAPI spec served with CRD Validation chema", func() { + framework.SkipUnlessServerVersionGTE(crdOpenAPIVersion, f.ClientSet.Discovery()) + + config, err := framework.LoadConfig() + if err != nil { + framework.Failf("failed to load config: %v", err) + } + + apiExtensionClient, err := clientset.NewForConfig(config) + if err != nil { + framework.Failf("failed to initialize apiExtensionClient: %v", err) + } + + randomDefinition := fixtures.NewRandomNameCustomResourceDefinition(v1beta1.ClusterScoped) + + //create CRD and waits for the resource to be recognized and available. + randomDefinition, err = fixtures.CreateNewCustomResourceDefinition(randomDefinition, apiExtensionClient, f.DynamicClient) + if err != nil { + framework.Failf("failed to create CustomResourceDefinition: %v", err) + } + + // TODO(roycaihw): think about tweaking feature gates in e2e test (is it possible/easy + // to do so?) and have CRD use top-level/per-version schema + // Also need to test NamespaceScoped CRDs + + // We use a wait.Poll block here because the kube-aggregator openapi + // controller takes time to rotate the queue and resync apiextensions-apiserver's spec + if err := wait.Poll(5*time.Second, 120*time.Second, func() (bool, error) { + data, err := f.ClientSet.CoreV1().RESTClient().Get(). + AbsPath("/swagger.json"). + DoRaw() + + if err != nil { + return false, err + } + // TODO(roycaihw): verify more Paths and List Definitions, also for multiple versions + baseDefinition := fmt.Sprintf("%s.%s.%s", randomDefinition.Spec.Group, randomDefinition.Spec.Version, randomDefinition.Spec.Names.Kind) + basePath := fmt.Sprintf("/apis/%s/%s/%s", randomDefinition.Spec.Group, randomDefinition.Spec.Version, randomDefinition.Spec.Names.Plural) + return strings.Contains(string(data), basePath) && + strings.Contains(string(data), baseDefinition), nil + }); err != nil { + framework.Failf("failed to wait for apiserver to serve openapi spec for registered CRD: %v", err) + } + + defer func() { + err = fixtures.DeleteCustomResourceDefinition(randomDefinition, apiExtensionClient) + if err != nil { + framework.Failf("failed to delete CustomResourceDefinition: %v", err) + } + }() + }) + }) }) diff --git a/test/integration/master/BUILD b/test/integration/master/BUILD index 87c9afda8fe..ff8f048bb6a 100644 --- a/test/integration/master/BUILD +++ b/test/integration/master/BUILD @@ -66,6 +66,7 @@ go_test( "//test/integration/framework:go_default_library", "//test/utils:go_default_library", "//vendor/github.com/evanphx/json-patch:go_default_library", + "//vendor/github.com/go-openapi/spec:go_default_library", "//vendor/sigs.k8s.io/yaml:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:android": [ diff --git a/test/integration/master/crd_test.go b/test/integration/master/crd_test.go index ac28b037c30..56e4a037ed2 100644 --- a/test/integration/master/crd_test.go +++ b/test/integration/master/crd_test.go @@ -18,9 +18,12 @@ package master import ( "encoding/json" + "fmt" "testing" "time" + "github.com/go-openapi/spec" + admissionregistrationv1alpha1 "k8s.io/api/admissionregistration/v1alpha1" networkingv1 "k8s.io/api/networking/v1" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" @@ -272,6 +275,91 @@ func TestCRD(t *testing.T) { } } +func TestCRDOpenAPI(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.Initializers, true)() + result := kubeapiservertesting.StartTestServerOrDie(t, nil, nil, framework.SharedEtcd()) + defer result.TearDownFn() + kubeclient, err := kubernetes.NewForConfig(result.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + apiextensionsclient, err := apiextensionsclientset.NewForConfig(result.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + t.Logf("Trying to create a custom resource without conflict") + crd := &apiextensionsv1beta1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foos.cr.bar.com", + }, + Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ + Group: "cr.bar.com", + Version: "v1", + Scope: apiextensionsv1beta1.NamespaceScoped, + Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ + Plural: "foos", + Kind: "Foo", + }, + Validation: &apiextensionsv1beta1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{ + Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ + "foo": {Type: "string"}, + }, + }, + }, + }, + } + etcd.CreateTestCRDs(t, apiextensionsclient, false, crd) + waitForSpec := func(expectedType string) { + t.Logf(`Waiting for {properties: {"foo": {"type":"%s"}}} to show up in schema`, expectedType) + lastMsg := "" + if err := wait.PollImmediate(500*time.Millisecond, 120*time.Second, func() (bool, error) { + lastMsg = "" + bs, err := kubeclient.RESTClient().Get().AbsPath("openapi", "v2").DoRaw() + if err != nil { + return false, err + } + spec := spec.Swagger{} + if err := json.Unmarshal(bs, &spec); err != nil { + return false, err + } + if spec.SwaggerProps.Paths == nil { + lastMsg = "spec.SwaggerProps.Paths is nil" + return false, nil + } + d, ok := spec.SwaggerProps.Definitions["cr.bar.com.v1.Foo"] + if !ok { + lastMsg = `spec.SwaggerProps.Definitions["cr.bar.com.v1.Foo"] not found` + return false, nil + } + p, ok := d.Properties["foo"] + if !ok { + lastMsg = `spec.SwaggerProps.Definitions["cr.bar.com.v1.Foo"].Properties["foo"] not found` + return false, nil + } + if !p.Type.Contains(expectedType) { + lastMsg = fmt.Sprintf(`spec.SwaggerProps.Definitions["cr.bar.com.v1.Foo"].Properties["foo"].Type should be %q, but got: %q`, expectedType, p.Type) + return false, nil + } + return true, nil + }); err != nil { + t.Fatalf("Failed to see %s OpenAPI spec in discovery: %v, last message: %s", crd.Name, err, lastMsg) + } + } + waitForSpec("string") + crd, err = apiextensionsclient.ApiextensionsV1beta1().CustomResourceDefinitions().Get(crd.Name, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + prop := crd.Spec.Validation.OpenAPIV3Schema.Properties["foo"] + prop.Type = "boolean" + crd.Spec.Validation.OpenAPIV3Schema.Properties["foo"] = prop + if _, err = apiextensionsclient.ApiextensionsV1beta1().CustomResourceDefinitions().Update(crd); err != nil { + t.Fatal(err) + } + waitForSpec("boolean") +} + type Foo struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`