From bb143d3e33458aa1b21bbded4ced98120ed42eb3 Mon Sep 17 00:00:00 2001 From: deads2k Date: Mon, 1 May 2017 15:49:03 -0400 Subject: [PATCH] add integration tests --- hack/.linted_packages | 1 + hack/make-rules/test-integration.sh | 3 + hack/make-rules/test.sh | 6 +- .../pkg/apiserver/apiserver.go | 21 +- .../pkg/apiserver/customresource_discovery.go | 30 +-- .../customresource_discovery_controller.go | 52 ++--- .../pkg/apiserver/customresource_handler.go | 38 ++-- .../test/integration/BUILD | 5 +- .../certificates/apiserver.crt | 19 ++ .../certificates/apiserver.key | 27 +++ .../test/integration/basic_test.go | 189 ++++++++++++++++++ .../test/integration/testserver/resources.go | 91 +++++++++ .../test/integration/testserver/start.go | 178 +++++++++++++++++ 13 files changed, 595 insertions(+), 65 deletions(-) create mode 100644 staging/src/k8s.io/kube-apiextensions-server/test/integration/apiserver.local.config/certificates/apiserver.crt create mode 100644 staging/src/k8s.io/kube-apiextensions-server/test/integration/apiserver.local.config/certificates/apiserver.key create mode 100644 staging/src/k8s.io/kube-apiextensions-server/test/integration/basic_test.go create mode 100644 staging/src/k8s.io/kube-apiextensions-server/test/integration/testserver/resources.go create mode 100644 staging/src/k8s.io/kube-apiextensions-server/test/integration/testserver/start.go diff --git a/hack/.linted_packages b/hack/.linted_packages index ef6547e7ddd..a5a6b85717d 100644 --- a/hack/.linted_packages +++ b/hack/.linted_packages @@ -408,6 +408,7 @@ staging/src/k8s.io/kube-apiextensions-server/pkg/client/informers/internalversio staging/src/k8s.io/kube-apiextensions-server/pkg/client/informers/internalversion/apiextensions/internalversion staging/src/k8s.io/kube-apiextensions-server/pkg/client/listers/apiextensions/internalversion staging/src/k8s.io/kube-apiextensions-server/pkg/client/listers/apiextensions/v1alpha1 +staging/src/k8s.io/kube-apiextensions-server/test/integration staging/src/k8s.io/metrics/pkg/apis/custom_metrics/install staging/src/k8s.io/metrics/pkg/apis/metrics/install staging/src/k8s.io/sample-apiserver diff --git a/hack/make-rules/test-integration.sh b/hack/make-rules/test-integration.sh index 61033469e2c..865328b6cb7 100755 --- a/hack/make-rules/test-integration.sh +++ b/hack/make-rules/test-integration.sh @@ -44,6 +44,9 @@ kube::test::find_integration_test_dirs() { find test/integration/ -name '*_test.go' -print0 \ | xargs -0n1 dirname | sed "s|^|${KUBE_GO_PACKAGE}/|" \ | LC_ALL=C sort -u + find vendor/k8s.io/kube-apiextensions-server/test/integration/ -name '*_test.go' -print0 \ + | xargs -0n1 dirname | sed "s|^|${KUBE_GO_PACKAGE}/|" \ + | LC_ALL=C sort -u ) } diff --git a/hack/make-rules/test.sh b/hack/make-rules/test.sh index 8cce276e3af..748f4fa08e6 100755 --- a/hack/make-rules/test.sh +++ b/hack/make-rules/test.sh @@ -72,7 +72,11 @@ kube::test::find_dirs() { find ./staging/src/k8s.io/kube-aggregator -name '*_test.go' \ -name '*_test.go' -print0 | xargs -0n1 dirname | sed 's|^\./staging/src/|./vendor/|' | LC_ALL=C sort -u - find ./staging/src/k8s.io/kube-apiextensions-server -name '*_test.go' \ + find ./staging/src/k8s.io/kube-apiextensions-server -not \( \ + \( \ + -o -path './test/integration/*' \ + \) -prune \ + \) -name '*_test.go' \ -name '*_test.go' -print0 | xargs -0n1 dirname | sed 's|^\./staging/src/|./vendor/|' | LC_ALL=C sort -u find ./staging/src/k8s.io/sample-apiserver -name '*_test.go' \ diff --git a/staging/src/k8s.io/kube-apiextensions-server/pkg/apiserver/apiserver.go b/staging/src/k8s.io/kube-apiextensions-server/pkg/apiserver/apiserver.go index 157a77dfb34..abb05d970d6 100644 --- a/staging/src/k8s.io/kube-apiextensions-server/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/kube-apiextensions-server/pkg/apiserver/apiserver.go @@ -17,6 +17,7 @@ limitations under the License. package apiserver import ( + "net/http" "time" "k8s.io/apimachinery/pkg/apimachinery/announced" @@ -126,13 +127,18 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget, } customResourceInformers := internalinformers.NewSharedInformerFactory(customResourceClient, 5*time.Minute) - versionDiscoveryHandler := &customResourceVersionDiscoveryHandler{ - discovery: map[schema.GroupVersion]*discovery.APIVersionHandler{}, - delegate: delegationTarget.UnprotectedHandler(), + delegateHandler := delegationTarget.UnprotectedHandler() + if delegateHandler == nil { + delegateHandler = http.NotFoundHandler() } - groupDiscoveryHandler := &customResourceGroupDiscoveryHandler{ + + versionDiscoveryHandler := &versionDiscoveryHandler{ + discovery: map[schema.GroupVersion]*discovery.APIVersionHandler{}, + delegate: delegateHandler, + } + groupDiscoveryHandler := &groupDiscoveryHandler{ discovery: map[string]*discovery.APIGroupHandler{}, - delegate: delegationTarget.UnprotectedHandler(), + delegate: delegateHandler, } customResourceHandler := NewCustomResourceHandler( versionDiscoveryHandler, @@ -143,9 +149,10 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget, c.CustomResourceRESTOptionsGetter, c.GenericConfig.AdmissionControl, ) - s.GenericAPIServer.FallThroughHandler.Handle("/apis/", customResourceHandler) + s.GenericAPIServer.FallThroughHandler.Handle("/apis", customResourceHandler) + s.GenericAPIServer.FallThroughHandler.HandlePrefix("/apis/", customResourceHandler) - customResourceController := NewCustomResourceDiscoveryController(customResourceInformers.Apiextensions().InternalVersion().CustomResources(), versionDiscoveryHandler, groupDiscoveryHandler) + customResourceController := NewDiscoveryController(customResourceInformers.Apiextensions().InternalVersion().CustomResources(), versionDiscoveryHandler, groupDiscoveryHandler) s.GenericAPIServer.AddPostStartHook("start-apiextensions-informers", func(context genericapiserver.PostStartHookContext) error { customResourceInformers.Start(stopCh) diff --git a/staging/src/k8s.io/kube-apiextensions-server/pkg/apiserver/customresource_discovery.go b/staging/src/k8s.io/kube-apiextensions-server/pkg/apiserver/customresource_discovery.go index 892e9692369..f40c33791b6 100644 --- a/staging/src/k8s.io/kube-apiextensions-server/pkg/apiserver/customresource_discovery.go +++ b/staging/src/k8s.io/kube-apiextensions-server/pkg/apiserver/customresource_discovery.go @@ -25,7 +25,7 @@ import ( "k8s.io/apiserver/pkg/endpoints/discovery" ) -type customResourceVersionDiscoveryHandler struct { +type versionDiscoveryHandler struct { // TODO, writing is infrequent, optimize this discoveryLock sync.RWMutex discovery map[schema.GroupVersion]*discovery.APIVersionHandler @@ -33,10 +33,10 @@ type customResourceVersionDiscoveryHandler struct { delegate http.Handler } -func (r *customResourceVersionDiscoveryHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { +func (r *versionDiscoveryHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { pathParts := splitPath(req.URL.Path) // only match /apis// - if len(pathParts) != 3 { + if len(pathParts) != 3 || pathParts[0] != "apis" { r.delegate.ServeHTTP(w, req) return } @@ -49,29 +49,29 @@ func (r *customResourceVersionDiscoveryHandler) ServeHTTP(w http.ResponseWriter, discovery.ServeHTTP(w, req) } -func (r *customResourceVersionDiscoveryHandler) getDiscovery(version schema.GroupVersion) (*discovery.APIVersionHandler, bool) { +func (r *versionDiscoveryHandler) getDiscovery(gv schema.GroupVersion) (*discovery.APIVersionHandler, bool) { r.discoveryLock.RLock() defer r.discoveryLock.RUnlock() - ret, ok := r.discovery[version] + ret, ok := r.discovery[gv] return ret, ok } -func (r *customResourceVersionDiscoveryHandler) setDiscovery(version schema.GroupVersion, discovery *discovery.APIVersionHandler) { +func (r *versionDiscoveryHandler) setDiscovery(gv schema.GroupVersion, discovery *discovery.APIVersionHandler) { r.discoveryLock.Lock() defer r.discoveryLock.Unlock() - r.discovery[version] = discovery + r.discovery[gv] = discovery } -func (r *customResourceVersionDiscoveryHandler) unsetDiscovery(version schema.GroupVersion) { +func (r *versionDiscoveryHandler) unsetDiscovery(gv schema.GroupVersion) { r.discoveryLock.Lock() defer r.discoveryLock.Unlock() - delete(r.discovery, version) + delete(r.discovery, gv) } -type customResourceGroupDiscoveryHandler struct { +type groupDiscoveryHandler struct { // TODO, writing is infrequent, optimize this discoveryLock sync.RWMutex discovery map[string]*discovery.APIGroupHandler @@ -79,10 +79,10 @@ type customResourceGroupDiscoveryHandler struct { delegate http.Handler } -func (r *customResourceGroupDiscoveryHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { +func (r *groupDiscoveryHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { pathParts := splitPath(req.URL.Path) // only match /apis/ - if len(pathParts) != 2 { + if len(pathParts) != 2 || pathParts[0] != "apis" { r.delegate.ServeHTTP(w, req) return } @@ -95,7 +95,7 @@ func (r *customResourceGroupDiscoveryHandler) ServeHTTP(w http.ResponseWriter, r discovery.ServeHTTP(w, req) } -func (r *customResourceGroupDiscoveryHandler) getDiscovery(group string) (*discovery.APIGroupHandler, bool) { +func (r *groupDiscoveryHandler) getDiscovery(group string) (*discovery.APIGroupHandler, bool) { r.discoveryLock.RLock() defer r.discoveryLock.RUnlock() @@ -103,14 +103,14 @@ func (r *customResourceGroupDiscoveryHandler) getDiscovery(group string) (*disco return ret, ok } -func (r *customResourceGroupDiscoveryHandler) setDiscovery(group string, discovery *discovery.APIGroupHandler) { +func (r *groupDiscoveryHandler) setDiscovery(group string, discovery *discovery.APIGroupHandler) { r.discoveryLock.Lock() defer r.discoveryLock.Unlock() r.discovery[group] = discovery } -func (r *customResourceGroupDiscoveryHandler) unsetDiscovery(group string) { +func (r *groupDiscoveryHandler) unsetDiscovery(group string) { r.discoveryLock.Lock() defer r.discoveryLock.Unlock() diff --git a/staging/src/k8s.io/kube-apiextensions-server/pkg/apiserver/customresource_discovery_controller.go b/staging/src/k8s.io/kube-apiextensions-server/pkg/apiserver/customresource_discovery_controller.go index acdc065178e..c3d87c58474 100644 --- a/staging/src/k8s.io/kube-apiextensions-server/pkg/apiserver/customresource_discovery_controller.go +++ b/staging/src/k8s.io/kube-apiextensions-server/pkg/apiserver/customresource_discovery_controller.go @@ -36,9 +36,9 @@ import ( listers "k8s.io/kube-apiextensions-server/pkg/client/listers/apiextensions/internalversion" ) -type CustomResourceDiscoveryController struct { - versionHandler *customResourceVersionDiscoveryHandler - groupHandler *customResourceGroupDiscoveryHandler +type DiscoveryController struct { + versionHandler *versionDiscoveryHandler + groupHandler *groupDiscoveryHandler customResourceLister listers.CustomResourceLister customResourcesSynced cache.InformerSynced @@ -49,14 +49,14 @@ type CustomResourceDiscoveryController struct { queue workqueue.RateLimitingInterface } -func NewCustomResourceDiscoveryController(customResourceInformer informers.CustomResourceInformer, versionHandler *customResourceVersionDiscoveryHandler, groupHandler *customResourceGroupDiscoveryHandler) *CustomResourceDiscoveryController { - c := &CustomResourceDiscoveryController{ +func NewDiscoveryController(customResourceInformer informers.CustomResourceInformer, versionHandler *versionDiscoveryHandler, groupHandler *groupDiscoveryHandler) *DiscoveryController { + c := &DiscoveryController{ versionHandler: versionHandler, groupHandler: groupHandler, customResourceLister: customResourceInformer.Lister(), customResourcesSynced: customResourceInformer.Informer().HasSynced, - queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "CustomResourceDiscoveryController"), + queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "DiscoveryController"), } customResourceInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ @@ -70,10 +70,7 @@ func NewCustomResourceDiscoveryController(customResourceInformer informers.Custo return c } -func (c *CustomResourceDiscoveryController) sync(version schema.GroupVersion) error { - - foundVersion := false - foundGroup := false +func (c *DiscoveryController) sync(version schema.GroupVersion) error { apiVersionsForDiscovery := []metav1.GroupVersionForDiscovery{} apiResourcesForDiscovery := []metav1.APIResource{} @@ -82,7 +79,11 @@ func (c *CustomResourceDiscoveryController) sync(version schema.GroupVersion) er if err != nil { return err } + foundVersion := false + foundGroup := false for _, customResource := range customResources { + // TODO add status checking + if customResource.Spec.Group != version.Group { continue } @@ -114,8 +115,9 @@ func (c *CustomResourceDiscoveryController) sync(version schema.GroupVersion) er } apiGroup := metav1.APIGroup{ - Name: version.Group, - Versions: apiVersionsForDiscovery, + Name: version.Group, + Versions: apiVersionsForDiscovery, + // the preferred versions for a group is arbitrary since there cannot be duplicate resources PreferredVersion: apiVersionsForDiscovery[0], } c.groupHandler.setDiscovery(version.Group, discovery.NewAPIGroupHandler(Codecs, apiGroup)) @@ -131,12 +133,12 @@ func (c *CustomResourceDiscoveryController) sync(version schema.GroupVersion) er return nil } -func (c *CustomResourceDiscoveryController) Run(stopCh <-chan struct{}) { +func (c *DiscoveryController) Run(stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer c.queue.ShutDown() - defer glog.Infof("Shutting down CustomResourceDiscoveryController") + defer glog.Infof("Shutting down DiscoveryController") - glog.Infof("Starting CustomResourceDiscoveryController") + glog.Infof("Starting DiscoveryController") if !cache.WaitForCacheSync(stopCh, c.customResourcesSynced) { utilruntime.HandleError(fmt.Errorf("timed out waiting for caches to sync")) @@ -149,13 +151,13 @@ func (c *CustomResourceDiscoveryController) Run(stopCh <-chan struct{}) { <-stopCh } -func (c *CustomResourceDiscoveryController) runWorker() { +func (c *DiscoveryController) runWorker() { for c.processNextWorkItem() { } } // processNextWorkItem deals with one key off the queue. It returns false when it's time to quit. -func (c *CustomResourceDiscoveryController) processNextWorkItem() bool { +func (c *DiscoveryController) processNextWorkItem() bool { key, quit := c.queue.Get() if quit { return false @@ -168,29 +170,29 @@ func (c *CustomResourceDiscoveryController) processNextWorkItem() bool { return true } - utilruntime.HandleError(fmt.Errorf("%v failed with : %v", key, err)) + utilruntime.HandleError(fmt.Errorf("%v failed with: %v", key, err)) c.queue.AddRateLimited(key) return true } -func (c *CustomResourceDiscoveryController) enqueue(obj *apiextensions.CustomResource) { +func (c *DiscoveryController) enqueue(obj *apiextensions.CustomResource) { c.queue.Add(schema.GroupVersion{Group: obj.Spec.Group, Version: obj.Spec.Version}) } -func (c *CustomResourceDiscoveryController) addCustomResource(obj interface{}) { +func (c *DiscoveryController) addCustomResource(obj interface{}) { castObj := obj.(*apiextensions.CustomResource) - glog.V(4).Infof("Adding %s", castObj.Name) + glog.V(4).Infof("Adding customresource %s", castObj.Name) c.enqueue(castObj) } -func (c *CustomResourceDiscoveryController) updateCustomResource(obj, _ interface{}) { +func (c *DiscoveryController) updateCustomResource(obj, _ interface{}) { castObj := obj.(*apiextensions.CustomResource) - glog.V(4).Infof("Updating %s", castObj.Name) + glog.V(4).Infof("Updating customresource %s", castObj.Name) c.enqueue(castObj) } -func (c *CustomResourceDiscoveryController) deleteCustomResource(obj interface{}) { +func (c *DiscoveryController) deleteCustomResource(obj interface{}) { castObj, ok := obj.(*apiextensions.CustomResource) if !ok { tombstone, ok := obj.(cache.DeletedFinalStateUnknown) @@ -204,6 +206,6 @@ func (c *CustomResourceDiscoveryController) deleteCustomResource(obj interface{} return } } - glog.V(4).Infof("Deleting %q", castObj.Name) + glog.V(4).Infof("Deleting customresource %q", castObj.Name) c.enqueue(castObj) } diff --git a/staging/src/k8s.io/kube-apiextensions-server/pkg/apiserver/customresource_handler.go b/staging/src/k8s.io/kube-apiextensions-server/pkg/apiserver/customresource_handler.go index f979ee74087..9cec84421ea 100644 --- a/staging/src/k8s.io/kube-apiextensions-server/pkg/apiserver/customresource_handler.go +++ b/staging/src/k8s.io/kube-apiextensions-server/pkg/apiserver/customresource_handler.go @@ -47,14 +47,14 @@ import ( "k8s.io/kube-apiextensions-server/pkg/registry/customresourcestorage" ) -// apisHandler serves the `/apis` endpoint. +// customResourceHandler serves the `/apis` endpoint. // This is registered as a filter so that it never collides with any explictly registered endpoints type customResourceHandler struct { - versionDiscoveryHandler *customResourceVersionDiscoveryHandler - groupDiscoveryHandler *customResourceGroupDiscoveryHandler + versionDiscoveryHandler *versionDiscoveryHandler + groupDiscoveryHandler *groupDiscoveryHandler - storageMutationLock sync.Mutex - // customStorage contains a map[types.UID]*customResourceInfo + customStorageLock sync.Mutex + // customStorage contains a customResourceStorageMap customStorage atomic.Value requestContextMapper apirequest.RequestContextMapper @@ -72,9 +72,12 @@ type customResourceInfo struct { requestScope handlers.RequestScope } +// customResourceStorageMap goes from customresource to its storage +type customResourceStorageMap map[types.UID]*customResourceInfo + func NewCustomResourceHandler( - versionDiscoveryHandler *customResourceVersionDiscoveryHandler, - groupDiscoveryHandler *customResourceGroupDiscoveryHandler, + versionDiscoveryHandler *versionDiscoveryHandler, + groupDiscoveryHandler *groupDiscoveryHandler, requestContextMapper apirequest.RequestContextMapper, customResourceLister listers.CustomResourceLister, delegate http.Handler, @@ -91,24 +94,27 @@ func NewCustomResourceHandler( admission: admission, } - ret.customStorage.Store(map[types.UID]*customResourceInfo{}) + ret.customStorage.Store(customResourceStorageMap{}) return ret } func (r *customResourceHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { ctx, ok := r.requestContextMapper.Get(req) if !ok { - http.Error(w, "missing context", http.StatusInternalServerError) + // programmer error + panic("missing context") return } requestInfo, ok := apirequest.RequestInfoFrom(ctx) if !ok { - http.Error(w, "missing requestInfo", http.StatusInternalServerError) + // programmer error + panic("missing requestInfo") return } if !requestInfo.IsResourceRequest { pathParts := splitPath(requestInfo.Path) // only match /apis// + // only registered under /apis if len(pathParts) == 3 { r.versionDiscoveryHandler.ServeHTTP(w, req) return @@ -192,7 +198,7 @@ func (r *customResourceHandler) removeDeadStorage() { // these don't have to be live. A snapshot is fine // if we wrongly delete, that's ok. The rest storage will be recreated on the next request // if we wrongly miss one, that's ok. We'll get it next time - storageMap := r.customStorage.Load().(map[types.UID]*customResourceInfo) + storageMap := r.customStorage.Load().(customResourceStorageMap) allCustomResources, err := r.customResourceLister.List(labels.Everything()) if err != nil { utilruntime.HandleError(err) @@ -212,21 +218,21 @@ func (r *customResourceHandler) removeDeadStorage() { } } - r.storageMutationLock.Lock() - defer r.storageMutationLock.Unlock() + r.customStorageLock.Lock() + defer r.customStorageLock.Unlock() r.customStorage.Store(storageMap) } func (r *customResourceHandler) getServingInfoFor(customResource *apiextensions.CustomResource) *customResourceInfo { - storageMap := r.customStorage.Load().(map[types.UID]*customResourceInfo) + storageMap := r.customStorage.Load().(customResourceStorageMap) ret, ok := storageMap[customResource.UID] if ok { return ret } - r.storageMutationLock.Lock() - defer r.storageMutationLock.Unlock() + r.customStorageLock.Lock() + defer r.customStorageLock.Unlock() ret, ok = storageMap[customResource.UID] if ok { diff --git a/staging/src/k8s.io/kube-apiextensions-server/test/integration/BUILD b/staging/src/k8s.io/kube-apiextensions-server/test/integration/BUILD index d0915bc9031..558da716006 100644 --- a/staging/src/k8s.io/kube-apiextensions-server/test/integration/BUILD +++ b/staging/src/k8s.io/kube-apiextensions-server/test/integration/BUILD @@ -10,7 +10,10 @@ load( go_test( name = "go_default_test", srcs = ["basic_test.go"], - tags = ["automanaged"], + tags = [ + "automanaged", + "integration", + ], deps = [ "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/staging/src/k8s.io/kube-apiextensions-server/test/integration/apiserver.local.config/certificates/apiserver.crt b/staging/src/k8s.io/kube-apiextensions-server/test/integration/apiserver.local.config/certificates/apiserver.crt new file mode 100644 index 00000000000..6bd8196e890 --- /dev/null +++ b/staging/src/k8s.io/kube-apiextensions-server/test/integration/apiserver.local.config/certificates/apiserver.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDEzCCAfugAwIBAgIBATANBgkqhkiG9w0BAQsFADAfMR0wGwYDVQQDDBRsb2Nh +bGhvc3RAMTQ5MzY2NDQ4OTAeFw0xNzA1MDExODQ4MDlaFw0xODA1MDExODQ4MDla +MB8xHTAbBgNVBAMMFGxvY2FsaG9zdEAxNDkzNjY0NDg5MIIBIjANBgkqhkiG9w0B +AQEFAAOCAQ8AMIIBCgKCAQEAy6tMZcDbG1J4vbX+YdiPswxqO+hX1r+i9+DFb1N/ +xyodBbprn1Mmd2k1lv3AkiZKm38v7dgzQ9/teA8Jm/1tyjjZSV/CxsZZWNuukPGU +ykEtn4mvkb5tOI1159ieTBiL4mKx5VNq8DkIpy9CT22Ud9dHkJaxJHcIF601hXHg +GIRla/6CRlkY/GFUItl1oij4sgzXRTS2pdv8lsmt2s7dXj737l10QCz9YDVuGSfu +rYoHGwY5ofYYFWzscD7Ds4O0tPdu4mSPIu753K7nB3ilfBi+tUWcSXpw9wE4+hIF +a1In8jnM+lw5/j/UoghrCtQ54BGWzpivPPXKv2dlNIOPiwIDAQABo1owWDAOBgNV +HQ8BAf8EBAMCAqQwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDwYDVR0TAQH/BAUwAwEB +/zAgBgNVHREEGTAXgglsb2NhbGhvc3SHBH8AAAGHBH8AAAEwDQYJKoZIhvcNAQEL +BQADggEBAJbxTi/0Joxx/oja4QDksbWroip0qVJKh1ic7ryai52aSBTcHMF9pWiL +047lL3sL0sN0YavXPUiow4PMTQm14W01ciwuZj5DCCaXnmnGtBy0fy8ifUdQoD/J +9pvLQMWAsx+GP2XzY+KxYFQairKS7BehEF/d24TgNHPskgc2p2XgK3Z7Ipp7hQrj +yZiTNromeULT12d5Zuwf+IeDp3aopGyhxCTOoc+RCz4MKLfKov40xjlaA4jVWazd +ccHWnagwM5lDlXnmCqZRVvyOWaUulJCEzRFfRTHFxKgj6DSPNt00wHXNmQUvjhN/ +YXFAkfKQQEs3qQRXoHAXKquplnLgjyA= +-----END CERTIFICATE----- diff --git a/staging/src/k8s.io/kube-apiextensions-server/test/integration/apiserver.local.config/certificates/apiserver.key b/staging/src/k8s.io/kube-apiextensions-server/test/integration/apiserver.local.config/certificates/apiserver.key new file mode 100644 index 00000000000..152fc1b6f42 --- /dev/null +++ b/staging/src/k8s.io/kube-apiextensions-server/test/integration/apiserver.local.config/certificates/apiserver.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEAy6tMZcDbG1J4vbX+YdiPswxqO+hX1r+i9+DFb1N/xyodBbpr +n1Mmd2k1lv3AkiZKm38v7dgzQ9/teA8Jm/1tyjjZSV/CxsZZWNuukPGUykEtn4mv +kb5tOI1159ieTBiL4mKx5VNq8DkIpy9CT22Ud9dHkJaxJHcIF601hXHgGIRla/6C +RlkY/GFUItl1oij4sgzXRTS2pdv8lsmt2s7dXj737l10QCz9YDVuGSfurYoHGwY5 +ofYYFWzscD7Ds4O0tPdu4mSPIu753K7nB3ilfBi+tUWcSXpw9wE4+hIFa1In8jnM ++lw5/j/UoghrCtQ54BGWzpivPPXKv2dlNIOPiwIDAQABAoIBABy4qWtoCP4PYUuP +kLIHsiwTwh90on58Q+Uk43LRmaFihPk70tWDCleolJAYdMGneLn4869c383glEJs +DHTdBlCQN8QrJvKVIiBvymxSRSNIkcB/0CyDaC+jc08gsyIUDBX+yQuH+fqqcFfz +SCyfTWKhD0yKk6yKxK9iE7wf1PRf5uLtJD6x1vV0NBmsHH++feODjNVsHDRvnwy6 +3KXkgSvfCTQ7qnQPZ/MSsRxWRdMBCnhaQq9qRnJ8bv8XotrCsEG5laMybriyJYXX +wvr9Dt04ciUD/g3qwIPy1ygMAKE9ya8hivSRURptZxz9SKCenWWihsfIzk4uyOi+ +sVDkJVECgYEA2981ZhbkruG5JhOsWVXTxDlXOrjI0pUqfej0WEp8lwdBMOrTXFs1 +cB8kdSocVC5GnMTbg6bqJrfglbNDOA7sUA8E8APLFFUAAvPwVfnrdDWk+0jK7atu +2sixQGeIB97Y6ojWfSbjyA/0p3Z0zCTfP5SR6xt4hVWhInB11m5IpO0CgYEA7SKH +bfWPZ0xfkFdM2X2cWQjt4zYHIUQqAKLPs2OjPTBC8PioNiZlvZq9bNseYR+eyIg6 +P9Kwe4KV/hzOSScf7JYpsN+YQ1+4Y+E63BkhAHXyz7y/vD+DA/Z83oKbelQtJqwq +W+Zo1OGJrfZwEKK5JWN9HF+KI9Z1iMyZoyw8L1cCgYEAtaLDfj7TVBVs2qPN8U8R +zjyAbyZP4IcRv0o+8OE345w+oqabTOScVK+lcpUDKhfAhamqniu5q5qjkYextBG/ +7rM5pP29OmKty8KxfJUlia73SA9udMD2pw68PzRIEBhsofPBHUqPSarEtcMJ4ctk +EiYuFUdwXNXMc6Lr9eTNZlECgYEAsfFJIvAzjdY3l76KwmGJox4aNHdkXkgiJJwH +s5s+8Tl34g8VWpzxl5e4MSkz4LmzktL2stHM8MGLAEZpXWdog0YjPsBqJ5R6byih +3GtW4lufutbuIbqe+6hJB0eGmAL2ZqCmoJODcstTXyEf8rvIpw/C4DmpFT9mryKo +31LgTr0CgYEAuxusmnR2vzZP/RjpjzmZcvIHf4xORG+SXlg3BXsSEd6+g3Rqiy5t +Q0UkHHwYnYurBmJ2HL1LG9mZwU89D00F/4mJpJuWfwqtqvodIRZ7bimyGGbvKZ1t +BGLmUssF5MYn75v7E5opxcc51aieW8nUQbop/PPMvWsYLrL/mcJNBpA= +-----END RSA PRIVATE KEY----- diff --git a/staging/src/k8s.io/kube-apiextensions-server/test/integration/basic_test.go b/staging/src/k8s.io/kube-apiextensions-server/test/integration/basic_test.go new file mode 100644 index 00000000000..e622447f8ad --- /dev/null +++ b/staging/src/k8s.io/kube-apiextensions-server/test/integration/basic_test.go @@ -0,0 +1,189 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "reflect" + "testing" + "time" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/watch" + apiextensionsv1alpha1 "k8s.io/kube-apiextensions-server/pkg/apis/apiextensions/v1alpha1" + "k8s.io/kube-apiextensions-server/test/integration/testserver" +) + +func TestServerUp(t *testing.T) { + stopCh, _, _, err := testserver.StartDefaultServer() + if err != nil { + t.Fatal(err) + } + defer close(stopCh) +} + +func TestSimpleCRUD(t *testing.T) { + stopCh, apiExtensionClient, clientPool, err := testserver.StartDefaultServer() + if err != nil { + t.Fatal(err) + } + defer close(stopCh) + + noxuDefinition := testserver.NewNoxuCustomResourceDefinition() + noxuVersionClient, err := testserver.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, clientPool) + if err != nil { + t.Fatal(err) + } + + ns := "not-the-default" + noxuNamespacedResourceClient := noxuVersionClient.Resource(&metav1.APIResource{ + Name: noxuDefinition.Spec.Names.Plural, + Namespaced: noxuDefinition.Spec.Scope == apiextensionsv1alpha1.NamespaceScoped, + }, ns) + initialList, err := noxuNamespacedResourceClient.List(metav1.ListOptions{}) + if err != nil { + t.Fatal(err) + } + if e, a := 0, len(initialList.(*unstructured.UnstructuredList).Items); e != a { + t.Errorf("expected %v, got %v", e, a) + } + initialListTypeMeta, err := meta.TypeAccessor(initialList) + if err != nil { + t.Fatal(err) + } + if e, a := noxuDefinition.Spec.Group+"/"+noxuDefinition.Spec.Version, initialListTypeMeta.GetAPIVersion(); e != a { + t.Errorf("expected %v, got %v", e, a) + } + if e, a := noxuDefinition.Spec.Names.ListKind, initialListTypeMeta.GetKind(); e != a { + t.Errorf("expected %v, got %v", e, a) + } + + initialListListMeta, err := meta.ListAccessor(initialList) + if err != nil { + t.Fatal(err) + } + noxuNamespacedWatch, err := noxuNamespacedResourceClient.Watch(metav1.ListOptions{ResourceVersion: initialListListMeta.GetResourceVersion()}) + if err != nil { + t.Fatal(err) + } + defer noxuNamespacedWatch.Stop() + + noxuInstanceToCreate := testserver.NewNoxuInstance(ns, "foo") + createdNoxuInstance, err := noxuNamespacedResourceClient.Create(noxuInstanceToCreate) + if err != nil { + t.Logf("%#v", createdNoxuInstance) + t.Fatal(err) + } + createdObjectMeta, err := meta.Accessor(createdNoxuInstance) + if err != nil { + t.Fatal(err) + } + // it should have a UUID + if len(createdObjectMeta.GetUID()) == 0 { + t.Errorf("missing uuid: %#v", createdNoxuInstance) + } + createdTypeMeta, err := meta.TypeAccessor(createdNoxuInstance) + if err != nil { + t.Fatal(err) + } + if e, a := noxuDefinition.Spec.Group+"/"+noxuDefinition.Spec.Version, createdTypeMeta.GetAPIVersion(); e != a { + t.Errorf("expected %v, got %v", e, a) + } + if e, a := noxuDefinition.Spec.Names.Kind, createdTypeMeta.GetKind(); e != a { + t.Errorf("expected %v, got %v", e, a) + } + + select { + case watchEvent := <-noxuNamespacedWatch.ResultChan(): + if e, a := watch.Added, watchEvent.Type; e != a { + t.Errorf("expected %v, got %v", e, a) + break + } + createdObjectMeta, err := meta.Accessor(watchEvent.Object) + if err != nil { + t.Fatal(err) + } + // it should have a UUID + if len(createdObjectMeta.GetUID()) == 0 { + t.Errorf("missing uuid: %#v", watchEvent.Object) + } + createdTypeMeta, err := meta.TypeAccessor(watchEvent.Object) + if err != nil { + t.Fatal(err) + } + if e, a := noxuDefinition.Spec.Group+"/"+noxuDefinition.Spec.Version, createdTypeMeta.GetAPIVersion(); e != a { + t.Errorf("expected %v, got %v", e, a) + } + if e, a := noxuDefinition.Spec.Names.Kind, createdTypeMeta.GetKind(); e != a { + t.Errorf("expected %v, got %v", e, a) + } + + case <-time.After(5 * time.Second): + t.Errorf("missing watch event") + } + + gottenNoxuInstance, err := noxuNamespacedResourceClient.Get("foo") + if err != nil { + t.Fatal(err) + } + if e, a := createdNoxuInstance, gottenNoxuInstance; !reflect.DeepEqual(e, a) { + t.Errorf("expected %v, got %v", e, a) + } + + listWithItem, err := noxuNamespacedResourceClient.List(metav1.ListOptions{}) + if err != nil { + t.Fatal(err) + } + if e, a := 1, len(listWithItem.(*unstructured.UnstructuredList).Items); e != a { + t.Errorf("expected %v, got %v", e, a) + } + if e, a := *createdNoxuInstance, listWithItem.(*unstructured.UnstructuredList).Items[0]; !reflect.DeepEqual(e, a) { + t.Errorf("expected %v, got %v", e, a) + } + + if err := noxuNamespacedResourceClient.Delete("foo", nil); err != nil { + t.Fatal(err) + } + + listWithoutItem, err := noxuNamespacedResourceClient.List(metav1.ListOptions{}) + if err != nil { + t.Fatal(err) + } + if e, a := 0, len(listWithoutItem.(*unstructured.UnstructuredList).Items); e != a { + t.Errorf("expected %v, got %v", e, a) + } + + select { + case watchEvent := <-noxuNamespacedWatch.ResultChan(): + if e, a := watch.Deleted, watchEvent.Type; e != a { + t.Errorf("expected %v, got %v", e, a) + break + } + deletedObjectMeta, err := meta.Accessor(watchEvent.Object) + if err != nil { + t.Fatal(err) + } + // it should have a UUID + if e, a := createdObjectMeta.GetUID(), deletedObjectMeta.GetUID(); e != a { + t.Errorf("expected %v, got %v", e, a) + } + + case <-time.After(5 * time.Second): + t.Errorf("missing watch event") + } +} diff --git a/staging/src/k8s.io/kube-apiextensions-server/test/integration/testserver/resources.go b/staging/src/k8s.io/kube-apiextensions-server/test/integration/testserver/resources.go new file mode 100644 index 00000000000..70a33526d38 --- /dev/null +++ b/staging/src/k8s.io/kube-apiextensions-server/test/integration/testserver/resources.go @@ -0,0 +1,91 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testserver + +import ( + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/dynamic" + apiextensionsv1alpha1 "k8s.io/kube-apiextensions-server/pkg/apis/apiextensions/v1alpha1" + "k8s.io/kube-apiextensions-server/pkg/client/clientset/clientset" +) + +func NewNoxuCustomResourceDefinition() *apiextensionsv1alpha1.CustomResource { + return &apiextensionsv1alpha1.CustomResource{ + ObjectMeta: metav1.ObjectMeta{Name: "noxus.mygroup.example.com"}, + Spec: apiextensionsv1alpha1.CustomResourceSpec{ + Group: "mygroup.example.com", + Version: "v1alpha1", + Names: apiextensionsv1alpha1.CustomResourceNames{ + Plural: "noxus", + Singular: "nonenglishnoxu", + Kind: "WishIHadChosenNoxu", + ListKind: "NoxuItemList", + }, + }, + } +} + +func NewNoxuInstance(namespace, name string) *unstructured.Unstructured { + return &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "mygroup.example.com/v1alpha1", + "kind": "WishIHadChosenNoxu", + "metadata": map[string]interface{}{ + "namespace": namespace, + "name": name, + }, + "content": map[string]interface{}{ + "key": "value", + }, + }, + } +} + +func CreateNewCustomResourceDefinition(customResource *apiextensionsv1alpha1.CustomResource, apiExtensionsClient clientset.Interface, clientPool dynamic.ClientPool) (*dynamic.Client, error) { + _, err := apiExtensionsClient.Apiextensions().CustomResources().Create(customResource) + if err != nil { + return nil, err + } + + // wait until the resource appears in discovery + err = wait.PollImmediate(30*time.Millisecond, 30*time.Second, func() (bool, error) { + resourceList, err := apiExtensionsClient.Discovery().ServerResourcesForGroupVersion(customResource.Spec.Group + "/" + customResource.Spec.Version) + if err != nil { + return false, nil + } + for _, resource := range resourceList.APIResources { + if resource.Name == customResource.Spec.Names.Plural { + return true, nil + } + } + return false, nil + }) + if err != nil { + return nil, err + } + + dynamicClient, err := clientPool.ClientForGroupVersionResource(schema.GroupVersionResource{Group: customResource.Spec.Group, Version: customResource.Spec.Version, Resource: customResource.Spec.Names.Plural}) + if err != nil { + return nil, err + } + return dynamicClient, nil +} diff --git a/staging/src/k8s.io/kube-apiextensions-server/test/integration/testserver/start.go b/staging/src/k8s.io/kube-apiextensions-server/test/integration/testserver/start.go new file mode 100644 index 00000000000..f3927378e92 --- /dev/null +++ b/staging/src/k8s.io/kube-apiextensions-server/test/integration/testserver/start.go @@ -0,0 +1,178 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testserver + +import ( + "fmt" + "net" + "os" + "strconv" + "time" + + "github.com/pborman/uuid" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/authorization/authorizerfactory" + genericapiserver "k8s.io/apiserver/pkg/server" + "k8s.io/client-go/dynamic" + extensionsapiserver "k8s.io/kube-apiextensions-server/pkg/apiserver" + "k8s.io/kube-apiextensions-server/pkg/client/clientset/clientset" + "k8s.io/kube-apiextensions-server/pkg/cmd/server" +) + +func DefaultServerConfig() (*extensionsapiserver.Config, error) { + port, err := FindFreeLocalPort() + if err != nil { + return nil, err + } + + options := server.NewCustomResourcesServerOptions(os.Stdout, os.Stderr) + options.RecommendedOptions.Audit.Path = "-" + options.RecommendedOptions.SecureServing.BindPort = port + options.RecommendedOptions.Authentication.SkipInClusterLookup = true + options.RecommendedOptions.SecureServing.BindAddress = net.ParseIP("127.0.0.1") + etcdURL, ok := os.LookupEnv("KUBE_INTEGRATION_ETCD_URL") + if !ok { + etcdURL = "http://127.0.0.1:2379" + } + options.RecommendedOptions.Etcd.StorageConfig.ServerList = []string{etcdURL} + options.RecommendedOptions.Etcd.StorageConfig.Prefix = uuid.New() + + // TODO stop copying this + // because there isn't currently a way to disable authentication or authorization from options + // explode options.Config here + genericConfig := genericapiserver.NewConfig(extensionsapiserver.Codecs) + genericConfig.Authenticator = nil + genericConfig.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer() + + if err := options.RecommendedOptions.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{net.ParseIP("127.0.0.1")}); err != nil { + return nil, fmt.Errorf("error creating self-signed certificates: %v", err) + } + if err := options.RecommendedOptions.Etcd.ApplyTo(genericConfig); err != nil { + return nil, err + } + if err := options.RecommendedOptions.SecureServing.ApplyTo(genericConfig); err != nil { + return nil, err + } + if err := options.RecommendedOptions.Audit.ApplyTo(genericConfig); err != nil { + return nil, err + } + if err := options.RecommendedOptions.Features.ApplyTo(genericConfig); err != nil { + return nil, err + } + + customResourceRESTOptionsGetter := extensionsapiserver.CustomResourceRESTOptionsGetter{ + StorageConfig: options.RecommendedOptions.Etcd.StorageConfig, + StoragePrefix: options.RecommendedOptions.Etcd.StorageConfig.Prefix, + EnableWatchCache: options.RecommendedOptions.Etcd.EnableWatchCache, + EnableGarbageCollection: options.RecommendedOptions.Etcd.EnableGarbageCollection, + DeleteCollectionWorkers: options.RecommendedOptions.Etcd.DeleteCollectionWorkers, + } + customResourceRESTOptionsGetter.StorageConfig.Codec = unstructured.UnstructuredJSONScheme + customResourceRESTOptionsGetter.StorageConfig.Copier = extensionsapiserver.UnstructuredCopier{} + + config := &extensionsapiserver.Config{ + GenericConfig: genericConfig, + CustomResourceRESTOptionsGetter: customResourceRESTOptionsGetter, + } + + return config, nil +} + +func StartServer(config *extensionsapiserver.Config) (chan struct{}, clientset.Interface, dynamic.ClientPool, error) { + stopCh := make(chan struct{}) + server, err := config.Complete().New(genericapiserver.EmptyDelegate, stopCh) + if err != nil { + close(stopCh) + return nil, nil, nil, err + } + go func() { + err := server.GenericAPIServer.PrepareRun().Run(stopCh) + if err != nil { + close(stopCh) + panic(err) + } + }() + + // wait until the server is healthy + err = wait.PollImmediate(30*time.Millisecond, 30*time.Second, func() (bool, error) { + healthClient, err := clientset.NewForConfig(server.GenericAPIServer.LoopbackClientConfig) + if err != nil { + return false, nil + } + healthResult := healthClient.Discovery().RESTClient().Get().AbsPath("/healthz").Do() + if healthResult.Error() != nil { + return false, nil + } + rawHealth, err := healthResult.Raw() + if err != nil { + return false, nil + } + if string(rawHealth) != "ok" { + return false, nil + } + + return true, nil + }) + if err != nil { + close(stopCh) + return nil, nil, nil, err + } + + apiExtensionsClient, err := clientset.NewForConfig(server.GenericAPIServer.LoopbackClientConfig) + if err != nil { + close(stopCh) + return nil, nil, nil, err + } + + bytes, _ := apiExtensionsClient.Discovery().RESTClient().Get().AbsPath("/apis/apiextensions.k8s.io/v1alpha1").DoRaw() + fmt.Print(string(bytes)) + + return stopCh, apiExtensionsClient, dynamic.NewDynamicClientPool(server.GenericAPIServer.LoopbackClientConfig), nil +} + +func StartDefaultServer() (chan struct{}, clientset.Interface, dynamic.ClientPool, error) { + config, err := DefaultServerConfig() + if err != nil { + return nil, nil, nil, err + } + + return StartServer(config) +} + +// FindFreeLocalPort returns the number of an available port number on +// the loopback interface. Useful for determining the port to launch +// a server on. Error handling required - there is a non-zero chance +// that the returned port number will be bound by another process +// after this function returns. +func FindFreeLocalPort() (int, error) { + l, err := net.Listen("tcp", ":0") + if err != nil { + return 0, err + } + defer l.Close() + _, portStr, err := net.SplitHostPort(l.Addr().String()) + if err != nil { + return 0, err + } + port, err := strconv.Atoi(portStr) + if err != nil { + return 0, err + } + return port, nil +}