From f39158a31068da63d7e790816f55a8da0ad20648 Mon Sep 17 00:00:00 2001 From: Monis Khan Date: Sun, 4 Nov 2018 16:44:43 -0500 Subject: [PATCH] Add CRDs to etcd storage path test This change updates the etcd storage path test to exercise custom resource storage by creating custom resource definitions before running the test. Duplicated custom resource definition test logic was consolidated. Signed-off-by: Monis Khan --- test/integration/auth/BUILD | 2 +- test/integration/auth/node_test.go | 46 ++-------- test/integration/etcd/BUILD | 2 + test/integration/etcd/data.go | 91 ++++++++++++++++++- .../etcd/etcd_storage_path_test.go | 4 + test/integration/etcd/server.go | 84 +++++++++++++++++ test/integration/master/BUILD | 1 + test/integration/master/crd_test.go | 63 +------------ 8 files changed, 195 insertions(+), 98 deletions(-) diff --git a/test/integration/auth/BUILD b/test/integration/auth/BUILD index 6d4d735bdab..6a640ac36a5 100644 --- a/test/integration/auth/BUILD +++ b/test/integration/auth/BUILD @@ -49,7 +49,6 @@ go_test( "//staging/src/k8s.io/api/authentication/v1beta1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/storage/v1beta1:go_default_library", - "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", @@ -84,6 +83,7 @@ go_test( "//staging/src/k8s.io/csi-api/pkg/crd:go_default_library", "//test/e2e/lifecycle/bootstrap:go_default_library", "//test/integration:go_default_library", + "//test/integration/etcd:go_default_library", "//test/integration/framework:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/gopkg.in/square/go-jose.v2/jwt:go_default_library", diff --git a/test/integration/auth/node_test.go b/test/integration/auth/node_test.go index db871d6d37b..e3eeb9cf00f 100644 --- a/test/integration/auth/node_test.go +++ b/test/integration/auth/node_test.go @@ -18,11 +18,12 @@ package auth import ( "fmt" + "io/ioutil" + "strings" "testing" "time" storagev1beta1 "k8s.io/api/storage/v1beta1" - apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -32,21 +33,18 @@ import ( utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" externalclientset "k8s.io/client-go/kubernetes" csiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1" + csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned" + csicrd "k8s.io/csi-api/pkg/crd" + kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" "k8s.io/kubernetes/pkg/apis/coordination" + "k8s.io/kubernetes/pkg/apis/core" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/policy" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/pkg/features" + "k8s.io/kubernetes/test/integration/etcd" "k8s.io/kubernetes/test/integration/framework" "k8s.io/utils/pointer" - - "io/ioutil" - apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" - csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned" - csicrd "k8s.io/csi-api/pkg/crd" - kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" - "k8s.io/kubernetes/pkg/apis/core" - "strings" ) func TestNodeAuthorizer(t *testing.T) { @@ -158,13 +156,7 @@ func TestNodeAuthorizer(t *testing.T) { t.Fatal(err) } - crd, err := superuserCRDClient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(csicrd.CSINodeInfoCRD()) - if err != nil { - t.Fatal(err) - } - if err := waitForEstablishedCRD(superuserCRDClient, crd.Name); err != nil { - t.Fatalf("Failed to establish CSINodeInfo CRD: %v", err) - } + etcd.CreateTestCRDs(t, superuserCRDClient, false, csicrd.CSINodeInfoCRD()) getSecret := func(client clientset.Interface) func() error { return func() error { @@ -672,25 +664,3 @@ func expectAllowed(t *testing.T, f func() error) { t.Errorf("Expected no error, got %v", err) } } - -func waitForEstablishedCRD(client apiextensionsclientset.Interface, name string) error { - return wait.PollImmediate(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { - crd, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Get(name, metav1.GetOptions{}) - if err != nil { - return false, err - } - for _, cond := range crd.Status.Conditions { - switch cond.Type { - case apiextensionsv1beta1.Established: - if cond.Status == apiextensionsv1beta1.ConditionTrue { - return true, err - } - case apiextensionsv1beta1.NamesAccepted: - if cond.Status == apiextensionsv1beta1.ConditionFalse { - fmt.Printf("Name conflict: %v\n", cond.Reason) - } - } - } - return false, nil - }) -} diff --git a/test/integration/etcd/BUILD b/test/integration/etcd/BUILD index d17427684fa..d86e89cc42d 100644 --- a/test/integration/etcd/BUILD +++ b/test/integration/etcd/BUILD @@ -53,6 +53,8 @@ go_library( "//cmd/kube-apiserver/app:go_default_library", "//cmd/kube-apiserver/app/options:go_default_library", "//pkg/master:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", diff --git a/test/integration/etcd/data.go b/test/integration/etcd/data.go index ab495b9dae1..d5cf04d7a6e 100644 --- a/test/integration/etcd/data.go +++ b/test/integration/etcd/data.go @@ -16,7 +16,11 @@ limitations under the License. package etcd -import "k8s.io/apimachinery/pkg/runtime/schema" +import ( + apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" +) // GetEtcdStorageData returns etcd data for all persisted objects. // It is exported so that it can be reused across multiple tests. @@ -427,6 +431,23 @@ func GetEtcdStorageData() map[schema.GroupVersionResource]StorageData { Stub: `{"metadata": {"name": "openshiftwebconsoleconfigs.webconsole.operator.openshift.io"},"spec": {"scope": "Cluster","group": "webconsole.operator.openshift.io","version": "v1alpha1","names": {"kind": "OpenShiftWebConsoleConfig","plural": "openshiftwebconsoleconfigs","singular": "openshiftwebconsoleconfig"}}}`, ExpectedEtcdPath: "/registry/apiextensions.k8s.io/customresourcedefinitions/openshiftwebconsoleconfigs.webconsole.operator.openshift.io", }, + gvr("cr.bar.com", "v1", "foos"): { + Stub: `{"kind": "Foo", "apiVersion": "cr.bar.com/v1", "metadata": {"name": "cr1foo"}, "color": "blue"}`, // requires TypeMeta due to CRD scheme's UnstructuredObjectTyper + ExpectedEtcdPath: "/registry/cr.bar.com/foos/etcdstoragepathtestnamespace/cr1foo", + }, + gvr("custom.fancy.com", "v2", "pants"): { + Stub: `{"kind": "Pant", "apiVersion": "custom.fancy.com/v2", "metadata": {"name": "cr2pant"}, "isFancy": true}`, // requires TypeMeta due to CRD scheme's UnstructuredObjectTyper + ExpectedEtcdPath: "/registry/custom.fancy.com/pants/cr2pant", + }, + gvr("awesome.bears.com", "v1", "pandas"): { + Stub: `{"kind": "Panda", "apiVersion": "awesome.bears.com/v1", "metadata": {"name": "cr3panda"}, "weight": 100}`, // requires TypeMeta due to CRD scheme's UnstructuredObjectTyper + ExpectedEtcdPath: "/registry/awesome.bears.com/pandas/cr3panda", + }, + gvr("awesome.bears.com", "v3", "pandas"): { + Stub: `{"kind": "Panda", "apiVersion": "awesome.bears.com/v3", "metadata": {"name": "cr4panda"}, "weight": 300}`, // requires TypeMeta due to CRD scheme's UnstructuredObjectTyper + ExpectedEtcdPath: "/registry/awesome.bears.com/pandas/cr4panda", + ExpectedGVK: gvkP("awesome.bears.com", "v1", "Panda"), + }, // -- } } @@ -446,6 +467,74 @@ type Prerequisite struct { Stub string } +// GetCustomResourceDefinitionData returns the resource definitions that back the custom resources +// included in GetEtcdStorageData. They should be created using CreateTestCRDs before running any tests. +func GetCustomResourceDefinitionData() []*apiextensionsv1beta1.CustomResourceDefinition { + return []*apiextensionsv1beta1.CustomResourceDefinition{ + // namespaced with legacy version field + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foos.cr.bar.com", + }, + Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ + Group: "cr.bar.com", + Version: "v1", + Scope: apiextensionsv1beta1.NamespaceScoped, + Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ + Plural: "foos", + Kind: "Foo", + }, + }, + }, + // cluster scoped with legacy version field + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pants.custom.fancy.com", + }, + Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ + Group: "custom.fancy.com", + Version: "v2", + Scope: apiextensionsv1beta1.ClusterScoped, + Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ + Plural: "pants", + Kind: "Pant", + }, + }, + }, + // cluster scoped with versions field + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pandas.awesome.bears.com", + }, + Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ + Group: "awesome.bears.com", + Versions: []apiextensionsv1beta1.CustomResourceDefinitionVersion{ + { + Name: "v1", + Served: true, + Storage: true, + }, + { + Name: "v2", + Served: false, + Storage: false, + }, + { + Name: "v3", + Served: true, + Storage: false, + }, + }, + Scope: apiextensionsv1beta1.ClusterScoped, + Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ + Plural: "pandas", + Kind: "Panda", + }, + }, + }, + } +} + func gvr(g, v, r string) schema.GroupVersionResource { return schema.GroupVersionResource{Group: g, Version: v, Resource: r} } diff --git a/test/integration/etcd/etcd_storage_path_test.go b/test/integration/etcd/etcd_storage_path_test.go index 1ec75a315f7..0335a5030c2 100644 --- a/test/integration/etcd/etcd_storage_path_test.go +++ b/test/integration/etcd/etcd_storage_path_test.go @@ -99,6 +99,10 @@ func TestEtcdStoragePath(t *testing.T) { if input, err = jsonToMetaObject([]byte(testData.Stub)); err != nil || input.isEmpty() { t.Fatalf("invalid test data for %s: %v", gvResource, err) } + // unset type meta fields - we only set these in the CRD test data and it makes + // any CRD test with an expectedGVK override fail the DeepDerivative test + input.Kind = "" + input.APIVersion = "" } all := &[]cleanupData{} diff --git a/test/integration/etcd/server.go b/test/integration/etcd/server.go index d47465d4ea0..d86534ba292 100644 --- a/test/integration/etcd/server.go +++ b/test/integration/etcd/server.go @@ -30,6 +30,8 @@ import ( "github.com/coreos/etcd/clientv3" "github.com/coreos/etcd/clientv3/concurrency" + apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -153,6 +155,9 @@ func StartRealMasterOrDie(t *testing.T) *Master { t.Fatal(err) } + // create CRDs so we can make sure that custom resources do not get lost + CreateTestCRDs(t, apiextensionsclientset.NewForConfigOrDie(kubeClientConfig), false, GetCustomResourceDefinitionData()...) + // force cached discovery reset discoveryClient := cacheddiscovery.NewMemCacheClient(kubeClient.Discovery()) restMapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient) @@ -169,6 +174,9 @@ func StartRealMasterOrDie(t *testing.T) *Master { } close(stopCh) lock.Unlock() + if err := session.Close(); err != nil { + t.Log(err) + } } return &Master{ @@ -281,3 +289,79 @@ func JSONToUnstructured(stub, namespace string, mapping *meta.RESTMapping, dynam return dynamicClient.Resource(mapping.Resource).Namespace(namespace), &unstructured.Unstructured{Object: typeMetaAdder}, nil } + +// CreateTestCRDs creates the given CRDs, any failure causes the test to Fatal. +// If skipCrdExistsInDiscovery is true, the CRDs are only checked for the Established condition via their Status. +// If skipCrdExistsInDiscovery is false, the CRDs are checked via discovery, see CrdExistsInDiscovery. +func CreateTestCRDs(t *testing.T, client apiextensionsclientset.Interface, skipCrdExistsInDiscovery bool, crds ...*apiextensionsv1beta1.CustomResourceDefinition) { + for _, crd := range crds { + createTestCRD(t, client, skipCrdExistsInDiscovery, crd) + } +} + +func createTestCRD(t *testing.T, client apiextensionsclientset.Interface, skipCrdExistsInDiscovery bool, crd *apiextensionsv1beta1.CustomResourceDefinition) { + if _, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd); err != nil { + t.Fatalf("Failed to create %s CRD; %v", crd.Name, err) + } + if skipCrdExistsInDiscovery { + if err := waitForEstablishedCRD(client, crd.Name); err != nil { + t.Fatalf("Failed to establish %s CRD; %v", crd.Name, err) + } + return + } + if err := wait.PollImmediate(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { + return CrdExistsInDiscovery(client, crd), nil + }); err != nil { + t.Fatalf("Failed to see %s in discovery: %v", crd.Name, err) + } +} + +func waitForEstablishedCRD(client apiextensionsclientset.Interface, name string) error { + return wait.PollImmediate(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { + crd, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Get(name, metav1.GetOptions{}) + if err != nil { + return false, err + } + for _, cond := range crd.Status.Conditions { + switch cond.Type { + case apiextensionsv1beta1.Established: + if cond.Status == apiextensionsv1beta1.ConditionTrue { + return true, nil + } + } + } + return false, nil + }) +} + +// CrdExistsInDiscovery checks to see if the given CRD exists in discovery at all served versions. +func CrdExistsInDiscovery(client apiextensionsclientset.Interface, crd *apiextensionsv1beta1.CustomResourceDefinition) bool { + var versions []string + if len(crd.Spec.Version) != 0 { + versions = append(versions, crd.Spec.Version) + } + for _, v := range crd.Spec.Versions { + if v.Served { + versions = append(versions, v.Name) + } + } + for _, v := range versions { + if !crdVersionExistsInDiscovery(client, crd, v) { + return false + } + } + return true +} + +func crdVersionExistsInDiscovery(client apiextensionsclientset.Interface, crd *apiextensionsv1beta1.CustomResourceDefinition, version string) bool { + resourceList, err := client.Discovery().ServerResourcesForGroupVersion(crd.Spec.Group + "/" + version) + if err != nil { + return false + } + for _, resource := range resourceList.APIResources { + if resource.Name == crd.Spec.Names.Plural { + return true + } + } + return false +} diff --git a/test/integration/master/BUILD b/test/integration/master/BUILD index fa012cd25ad..4575d2667c4 100644 --- a/test/integration/master/BUILD +++ b/test/integration/master/BUILD @@ -62,6 +62,7 @@ go_test( "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration:go_default_library", "//test/integration:go_default_library", + "//test/integration/etcd:go_default_library", "//test/integration/framework:go_default_library", "//test/utils:go_default_library", "//vendor/github.com/evanphx/json-patch:go_default_library", diff --git a/test/integration/master/crd_test.go b/test/integration/master/crd_test.go index 61c11c6a6d4..ac28b037c30 100644 --- a/test/integration/master/crd_test.go +++ b/test/integration/master/crd_test.go @@ -18,7 +18,6 @@ package master import ( "encoding/json" - "fmt" "testing" "time" @@ -37,6 +36,7 @@ import ( "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" + "k8s.io/kubernetes/test/integration/etcd" "k8s.io/kubernetes/test/integration/framework" ) @@ -81,12 +81,8 @@ func TestCRDShadowGroup(t *testing.T) { }, }, } - if _, err = apiextensionsclient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd); err != nil { - t.Fatalf("Failed to create networking group CRD: %v", err) - } - if err := waitForEstablishedCRD(apiextensionsclient, crd.Name); err != nil { - t.Fatalf("Failed to establish networking group CRD: %v", err) - } + etcd.CreateTestCRDs(t, apiextensionsclient, true, crd) + // wait to give aggregator time to update time.Sleep(2 * time.Second) @@ -97,11 +93,7 @@ func TestCRDShadowGroup(t *testing.T) { } t.Logf("Checking that crd resource does not show up in networking group") - found, err := crdExistsInDiscovery(apiextensionsclient, crd) - if err != nil { - t.Fatalf("unexpected discovery error: %v", err) - } - if found { + if etcd.CrdExistsInDiscovery(apiextensionsclient, crd) { t.Errorf("CRD resource shows up in discovery, but shouldn't.") } } @@ -137,17 +129,7 @@ func TestCRD(t *testing.T) { }, }, } - if _, err = apiextensionsclient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd); err != nil { - t.Fatalf("Failed to create foos.cr.bar.com CRD; %v", err) - } - if err := waitForEstablishedCRD(apiextensionsclient, crd.Name); err != nil { - t.Fatalf("Failed to establish foos.cr.bar.com CRD: %v", err) - } - if err := wait.PollImmediate(500*time.Millisecond, 30*time.Second, func() (bool, error) { - return crdExistsInDiscovery(apiextensionsclient, crd) - }); err != nil { - t.Fatalf("Failed to see foos.cr.bar.com in discovery: %v", err) - } + etcd.CreateTestCRDs(t, apiextensionsclient, false, crd) t.Logf("Trying to access foos.cr.bar.com with dynamic client") dynamicClient, err := dynamic.NewForConfig(result.ClientConfig) @@ -306,38 +288,3 @@ func unstructuredFoo(foo *Foo) (*unstructured.Unstructured, error) { } return ret, nil } - -func waitForEstablishedCRD(client apiextensionsclientset.Interface, name string) error { - return wait.PollImmediate(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { - crd, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Get(name, metav1.GetOptions{}) - if err != nil { - return false, err - } - for _, cond := range crd.Status.Conditions { - switch cond.Type { - case apiextensionsv1beta1.Established: - if cond.Status == apiextensionsv1beta1.ConditionTrue { - return true, err - } - case apiextensionsv1beta1.NamesAccepted: - if cond.Status == apiextensionsv1beta1.ConditionFalse { - fmt.Printf("Name conflict: %v\n", cond.Reason) - } - } - } - return false, nil - }) -} - -func crdExistsInDiscovery(client apiextensionsclientset.Interface, crd *apiextensionsv1beta1.CustomResourceDefinition) (bool, error) { - resourceList, err := client.Discovery().ServerResourcesForGroupVersion(crd.Spec.Group + "/" + crd.Spec.Version) - if err != nil { - return false, nil - } - for _, resource := range resourceList.APIResources { - if resource.Name == crd.Spec.Names.Plural { - return true, nil - } - } - return false, nil -}