diff --git a/test/integration/auth/BUILD b/test/integration/auth/BUILD index 6d4d735bdab..6a640ac36a5 100644 --- a/test/integration/auth/BUILD +++ b/test/integration/auth/BUILD @@ -49,7 +49,6 @@ go_test( "//staging/src/k8s.io/api/authentication/v1beta1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/storage/v1beta1:go_default_library", - "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", @@ -84,6 +83,7 @@ go_test( "//staging/src/k8s.io/csi-api/pkg/crd:go_default_library", "//test/e2e/lifecycle/bootstrap:go_default_library", "//test/integration:go_default_library", + "//test/integration/etcd:go_default_library", "//test/integration/framework:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/gopkg.in/square/go-jose.v2/jwt:go_default_library", diff --git a/test/integration/auth/node_test.go b/test/integration/auth/node_test.go index db871d6d37b..e3eeb9cf00f 100644 --- a/test/integration/auth/node_test.go +++ b/test/integration/auth/node_test.go @@ -18,11 +18,12 @@ package auth import ( "fmt" + "io/ioutil" + "strings" "testing" "time" storagev1beta1 "k8s.io/api/storage/v1beta1" - apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -32,21 +33,18 @@ import ( utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" externalclientset "k8s.io/client-go/kubernetes" csiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1" + csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned" + csicrd "k8s.io/csi-api/pkg/crd" + kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" "k8s.io/kubernetes/pkg/apis/coordination" + "k8s.io/kubernetes/pkg/apis/core" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/policy" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/pkg/features" + "k8s.io/kubernetes/test/integration/etcd" "k8s.io/kubernetes/test/integration/framework" "k8s.io/utils/pointer" - - "io/ioutil" - apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" - csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned" - csicrd "k8s.io/csi-api/pkg/crd" - kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" - "k8s.io/kubernetes/pkg/apis/core" - "strings" ) func TestNodeAuthorizer(t *testing.T) { @@ -158,13 +156,7 @@ func TestNodeAuthorizer(t *testing.T) { t.Fatal(err) } - crd, err := superuserCRDClient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(csicrd.CSINodeInfoCRD()) - if err != nil { - t.Fatal(err) - } - if err := waitForEstablishedCRD(superuserCRDClient, crd.Name); err != nil { - t.Fatalf("Failed to establish CSINodeInfo CRD: %v", err) - } + etcd.CreateTestCRDs(t, superuserCRDClient, false, csicrd.CSINodeInfoCRD()) getSecret := func(client clientset.Interface) func() error { return func() error { @@ -672,25 +664,3 @@ func expectAllowed(t *testing.T, f func() error) { t.Errorf("Expected no error, got %v", err) } } - -func waitForEstablishedCRD(client apiextensionsclientset.Interface, name string) error { - return wait.PollImmediate(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { - crd, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Get(name, metav1.GetOptions{}) - if err != nil { - return false, err - } - for _, cond := range crd.Status.Conditions { - switch cond.Type { - case apiextensionsv1beta1.Established: - if cond.Status == apiextensionsv1beta1.ConditionTrue { - return true, err - } - case apiextensionsv1beta1.NamesAccepted: - if cond.Status == apiextensionsv1beta1.ConditionFalse { - fmt.Printf("Name conflict: %v\n", cond.Reason) - } - } - } - return false, nil - }) -} diff --git a/test/integration/etcd/BUILD b/test/integration/etcd/BUILD index d17427684fa..d86e89cc42d 100644 --- a/test/integration/etcd/BUILD +++ b/test/integration/etcd/BUILD @@ -53,6 +53,8 @@ go_library( "//cmd/kube-apiserver/app:go_default_library", "//cmd/kube-apiserver/app/options:go_default_library", "//pkg/master:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", diff --git a/test/integration/etcd/data.go b/test/integration/etcd/data.go index ab495b9dae1..d5cf04d7a6e 100644 --- a/test/integration/etcd/data.go +++ b/test/integration/etcd/data.go @@ -16,7 +16,11 @@ limitations under the License. package etcd -import "k8s.io/apimachinery/pkg/runtime/schema" +import ( + apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" +) // GetEtcdStorageData returns etcd data for all persisted objects. // It is exported so that it can be reused across multiple tests. @@ -427,6 +431,23 @@ func GetEtcdStorageData() map[schema.GroupVersionResource]StorageData { Stub: `{"metadata": {"name": "openshiftwebconsoleconfigs.webconsole.operator.openshift.io"},"spec": {"scope": "Cluster","group": "webconsole.operator.openshift.io","version": "v1alpha1","names": {"kind": "OpenShiftWebConsoleConfig","plural": "openshiftwebconsoleconfigs","singular": "openshiftwebconsoleconfig"}}}`, ExpectedEtcdPath: "/registry/apiextensions.k8s.io/customresourcedefinitions/openshiftwebconsoleconfigs.webconsole.operator.openshift.io", }, + gvr("cr.bar.com", "v1", "foos"): { + Stub: `{"kind": "Foo", "apiVersion": "cr.bar.com/v1", "metadata": {"name": "cr1foo"}, "color": "blue"}`, // requires TypeMeta due to CRD scheme's UnstructuredObjectTyper + ExpectedEtcdPath: "/registry/cr.bar.com/foos/etcdstoragepathtestnamespace/cr1foo", + }, + gvr("custom.fancy.com", "v2", "pants"): { + Stub: `{"kind": "Pant", "apiVersion": "custom.fancy.com/v2", "metadata": {"name": "cr2pant"}, "isFancy": true}`, // requires TypeMeta due to CRD scheme's UnstructuredObjectTyper + ExpectedEtcdPath: "/registry/custom.fancy.com/pants/cr2pant", + }, + gvr("awesome.bears.com", "v1", "pandas"): { + Stub: `{"kind": "Panda", "apiVersion": "awesome.bears.com/v1", "metadata": {"name": "cr3panda"}, "weight": 100}`, // requires TypeMeta due to CRD scheme's UnstructuredObjectTyper + ExpectedEtcdPath: "/registry/awesome.bears.com/pandas/cr3panda", + }, + gvr("awesome.bears.com", "v3", "pandas"): { + Stub: `{"kind": "Panda", "apiVersion": "awesome.bears.com/v3", "metadata": {"name": "cr4panda"}, "weight": 300}`, // requires TypeMeta due to CRD scheme's UnstructuredObjectTyper + ExpectedEtcdPath: "/registry/awesome.bears.com/pandas/cr4panda", + ExpectedGVK: gvkP("awesome.bears.com", "v1", "Panda"), + }, // -- } } @@ -446,6 +467,74 @@ type Prerequisite struct { Stub string } +// GetCustomResourceDefinitionData returns the resource definitions that back the custom resources +// included in GetEtcdStorageData. They should be created using CreateTestCRDs before running any tests. +func GetCustomResourceDefinitionData() []*apiextensionsv1beta1.CustomResourceDefinition { + return []*apiextensionsv1beta1.CustomResourceDefinition{ + // namespaced with legacy version field + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foos.cr.bar.com", + }, + Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ + Group: "cr.bar.com", + Version: "v1", + Scope: apiextensionsv1beta1.NamespaceScoped, + Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ + Plural: "foos", + Kind: "Foo", + }, + }, + }, + // cluster scoped with legacy version field + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pants.custom.fancy.com", + }, + Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ + Group: "custom.fancy.com", + Version: "v2", + Scope: apiextensionsv1beta1.ClusterScoped, + Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ + Plural: "pants", + Kind: "Pant", + }, + }, + }, + // cluster scoped with versions field + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pandas.awesome.bears.com", + }, + Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ + Group: "awesome.bears.com", + Versions: []apiextensionsv1beta1.CustomResourceDefinitionVersion{ + { + Name: "v1", + Served: true, + Storage: true, + }, + { + Name: "v2", + Served: false, + Storage: false, + }, + { + Name: "v3", + Served: true, + Storage: false, + }, + }, + Scope: apiextensionsv1beta1.ClusterScoped, + Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ + Plural: "pandas", + Kind: "Panda", + }, + }, + }, + } +} + func gvr(g, v, r string) schema.GroupVersionResource { return schema.GroupVersionResource{Group: g, Version: v, Resource: r} } diff --git a/test/integration/etcd/etcd_storage_path_test.go b/test/integration/etcd/etcd_storage_path_test.go index 1ec75a315f7..0335a5030c2 100644 --- a/test/integration/etcd/etcd_storage_path_test.go +++ b/test/integration/etcd/etcd_storage_path_test.go @@ -99,6 +99,10 @@ func TestEtcdStoragePath(t *testing.T) { if input, err = jsonToMetaObject([]byte(testData.Stub)); err != nil || input.isEmpty() { t.Fatalf("invalid test data for %s: %v", gvResource, err) } + // unset type meta fields - we only set these in the CRD test data and it makes + // any CRD test with an expectedGVK override fail the DeepDerivative test + input.Kind = "" + input.APIVersion = "" } all := &[]cleanupData{} diff --git a/test/integration/etcd/server.go b/test/integration/etcd/server.go index d47465d4ea0..d86534ba292 100644 --- a/test/integration/etcd/server.go +++ b/test/integration/etcd/server.go @@ -30,6 +30,8 @@ import ( "github.com/coreos/etcd/clientv3" "github.com/coreos/etcd/clientv3/concurrency" + apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -153,6 +155,9 @@ func StartRealMasterOrDie(t *testing.T) *Master { t.Fatal(err) } + // create CRDs so we can make sure that custom resources do not get lost + CreateTestCRDs(t, apiextensionsclientset.NewForConfigOrDie(kubeClientConfig), false, GetCustomResourceDefinitionData()...) + // force cached discovery reset discoveryClient := cacheddiscovery.NewMemCacheClient(kubeClient.Discovery()) restMapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient) @@ -169,6 +174,9 @@ func StartRealMasterOrDie(t *testing.T) *Master { } close(stopCh) lock.Unlock() + if err := session.Close(); err != nil { + t.Log(err) + } } return &Master{ @@ -281,3 +289,79 @@ func JSONToUnstructured(stub, namespace string, mapping *meta.RESTMapping, dynam return dynamicClient.Resource(mapping.Resource).Namespace(namespace), &unstructured.Unstructured{Object: typeMetaAdder}, nil } + +// CreateTestCRDs creates the given CRDs, any failure causes the test to Fatal. +// If skipCrdExistsInDiscovery is true, the CRDs are only checked for the Established condition via their Status. +// If skipCrdExistsInDiscovery is false, the CRDs are checked via discovery, see CrdExistsInDiscovery. +func CreateTestCRDs(t *testing.T, client apiextensionsclientset.Interface, skipCrdExistsInDiscovery bool, crds ...*apiextensionsv1beta1.CustomResourceDefinition) { + for _, crd := range crds { + createTestCRD(t, client, skipCrdExistsInDiscovery, crd) + } +} + +func createTestCRD(t *testing.T, client apiextensionsclientset.Interface, skipCrdExistsInDiscovery bool, crd *apiextensionsv1beta1.CustomResourceDefinition) { + if _, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd); err != nil { + t.Fatalf("Failed to create %s CRD; %v", crd.Name, err) + } + if skipCrdExistsInDiscovery { + if err := waitForEstablishedCRD(client, crd.Name); err != nil { + t.Fatalf("Failed to establish %s CRD; %v", crd.Name, err) + } + return + } + if err := wait.PollImmediate(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { + return CrdExistsInDiscovery(client, crd), nil + }); err != nil { + t.Fatalf("Failed to see %s in discovery: %v", crd.Name, err) + } +} + +func waitForEstablishedCRD(client apiextensionsclientset.Interface, name string) error { + return wait.PollImmediate(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { + crd, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Get(name, metav1.GetOptions{}) + if err != nil { + return false, err + } + for _, cond := range crd.Status.Conditions { + switch cond.Type { + case apiextensionsv1beta1.Established: + if cond.Status == apiextensionsv1beta1.ConditionTrue { + return true, nil + } + } + } + return false, nil + }) +} + +// CrdExistsInDiscovery checks to see if the given CRD exists in discovery at all served versions. +func CrdExistsInDiscovery(client apiextensionsclientset.Interface, crd *apiextensionsv1beta1.CustomResourceDefinition) bool { + var versions []string + if len(crd.Spec.Version) != 0 { + versions = append(versions, crd.Spec.Version) + } + for _, v := range crd.Spec.Versions { + if v.Served { + versions = append(versions, v.Name) + } + } + for _, v := range versions { + if !crdVersionExistsInDiscovery(client, crd, v) { + return false + } + } + return true +} + +func crdVersionExistsInDiscovery(client apiextensionsclientset.Interface, crd *apiextensionsv1beta1.CustomResourceDefinition, version string) bool { + resourceList, err := client.Discovery().ServerResourcesForGroupVersion(crd.Spec.Group + "/" + version) + if err != nil { + return false + } + for _, resource := range resourceList.APIResources { + if resource.Name == crd.Spec.Names.Plural { + return true + } + } + return false +} diff --git a/test/integration/master/BUILD b/test/integration/master/BUILD index fa012cd25ad..4575d2667c4 100644 --- a/test/integration/master/BUILD +++ b/test/integration/master/BUILD @@ -62,6 +62,7 @@ go_test( "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration:go_default_library", "//test/integration:go_default_library", + "//test/integration/etcd:go_default_library", "//test/integration/framework:go_default_library", "//test/utils:go_default_library", "//vendor/github.com/evanphx/json-patch:go_default_library", diff --git a/test/integration/master/crd_test.go b/test/integration/master/crd_test.go index 61c11c6a6d4..ac28b037c30 100644 --- a/test/integration/master/crd_test.go +++ b/test/integration/master/crd_test.go @@ -18,7 +18,6 @@ package master import ( "encoding/json" - "fmt" "testing" "time" @@ -37,6 +36,7 @@ import ( "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" + "k8s.io/kubernetes/test/integration/etcd" "k8s.io/kubernetes/test/integration/framework" ) @@ -81,12 +81,8 @@ func TestCRDShadowGroup(t *testing.T) { }, }, } - if _, err = apiextensionsclient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd); err != nil { - t.Fatalf("Failed to create networking group CRD: %v", err) - } - if err := waitForEstablishedCRD(apiextensionsclient, crd.Name); err != nil { - t.Fatalf("Failed to establish networking group CRD: %v", err) - } + etcd.CreateTestCRDs(t, apiextensionsclient, true, crd) + // wait to give aggregator time to update time.Sleep(2 * time.Second) @@ -97,11 +93,7 @@ func TestCRDShadowGroup(t *testing.T) { } t.Logf("Checking that crd resource does not show up in networking group") - found, err := crdExistsInDiscovery(apiextensionsclient, crd) - if err != nil { - t.Fatalf("unexpected discovery error: %v", err) - } - if found { + if etcd.CrdExistsInDiscovery(apiextensionsclient, crd) { t.Errorf("CRD resource shows up in discovery, but shouldn't.") } } @@ -137,17 +129,7 @@ func TestCRD(t *testing.T) { }, }, } - if _, err = apiextensionsclient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd); err != nil { - t.Fatalf("Failed to create foos.cr.bar.com CRD; %v", err) - } - if err := waitForEstablishedCRD(apiextensionsclient, crd.Name); err != nil { - t.Fatalf("Failed to establish foos.cr.bar.com CRD: %v", err) - } - if err := wait.PollImmediate(500*time.Millisecond, 30*time.Second, func() (bool, error) { - return crdExistsInDiscovery(apiextensionsclient, crd) - }); err != nil { - t.Fatalf("Failed to see foos.cr.bar.com in discovery: %v", err) - } + etcd.CreateTestCRDs(t, apiextensionsclient, false, crd) t.Logf("Trying to access foos.cr.bar.com with dynamic client") dynamicClient, err := dynamic.NewForConfig(result.ClientConfig) @@ -306,38 +288,3 @@ func unstructuredFoo(foo *Foo) (*unstructured.Unstructured, error) { } return ret, nil } - -func waitForEstablishedCRD(client apiextensionsclientset.Interface, name string) error { - return wait.PollImmediate(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { - crd, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Get(name, metav1.GetOptions{}) - if err != nil { - return false, err - } - for _, cond := range crd.Status.Conditions { - switch cond.Type { - case apiextensionsv1beta1.Established: - if cond.Status == apiextensionsv1beta1.ConditionTrue { - return true, err - } - case apiextensionsv1beta1.NamesAccepted: - if cond.Status == apiextensionsv1beta1.ConditionFalse { - fmt.Printf("Name conflict: %v\n", cond.Reason) - } - } - } - return false, nil - }) -} - -func crdExistsInDiscovery(client apiextensionsclientset.Interface, crd *apiextensionsv1beta1.CustomResourceDefinition) (bool, error) { - resourceList, err := client.Discovery().ServerResourcesForGroupVersion(crd.Spec.Group + "/" + crd.Spec.Version) - if err != nil { - return false, nil - } - for _, resource := range resourceList.APIResources { - if resource.Name == crd.Spec.Names.Plural { - return true, nil - } - } - return false, nil -}