add integration tests

hack/.linted_packages

@@ -408,6 +408,7 @@ staging/src/k8s.io/kube-apiextensions-server/pkg/client/informers/internalversio
 staging/src/k8s.io/kube-apiextensions-server/pkg/client/informers/internalversion/apiextensions/internalversion
 staging/src/k8s.io/kube-apiextensions-server/pkg/client/listers/apiextensions/internalversion
 staging/src/k8s.io/kube-apiextensions-server/pkg/client/listers/apiextensions/v1alpha1
+staging/src/k8s.io/kube-apiextensions-server/test/integration
 staging/src/k8s.io/metrics/pkg/apis/custom_metrics/install
 staging/src/k8s.io/metrics/pkg/apis/metrics/install
 staging/src/k8s.io/sample-apiserver

hack/make-rules/test-integration.sh

@@ -44,6 +44,9 @@ kube::test::find_integration_test_dirs() {
     find test/integration/ -name '*_test.go' -print0 \
       | xargs -0n1 dirname | sed "s|^|${KUBE_GO_PACKAGE}/|" \
       | LC_ALL=C sort -u
+    find vendor/k8s.io/kube-apiextensions-server/test/integration/ -name '*_test.go' -print0 \
+      | xargs -0n1 dirname | sed "s|^|${KUBE_GO_PACKAGE}/|" \
+      | LC_ALL=C sort -u
   )
 }
 

hack/make-rules/test.sh

@@ -72,7 +72,11 @@ kube::test::find_dirs() {
     find ./staging/src/k8s.io/kube-aggregator -name '*_test.go' \
       -name '*_test.go' -print0 | xargs -0n1 dirname | sed 's|^\./staging/src/|./vendor/|' | LC_ALL=C sort -u
 
-    find ./staging/src/k8s.io/kube-apiextensions-server -name '*_test.go' \
+    find ./staging/src/k8s.io/kube-apiextensions-server -not \( \
+        \( \
+          -o -path './test/integration/*' \
+        \) -prune \
+      \) -name '*_test.go' \
       -name '*_test.go' -print0 | xargs -0n1 dirname | sed 's|^\./staging/src/|./vendor/|' | LC_ALL=C sort -u
 
     find ./staging/src/k8s.io/sample-apiserver -name '*_test.go' \

staging/src/k8s.io/kube-apiextensions-server/pkg/apiserver/apiserver.go

@@ -17,6 +17,7 @@ limitations under the License.
 package apiserver
 
 import (
+	"net/http"
 	"time"
 
 	"k8s.io/apimachinery/pkg/apimachinery/announced"
@@ -126,13 +127,18 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget,
 	}
 	customResourceInformers := internalinformers.NewSharedInformerFactory(customResourceClient, 5*time.Minute)
 
-	versionDiscoveryHandler := &customResourceVersionDiscoveryHandler{
+	delegateHandler := delegationTarget.UnprotectedHandler()
-		discovery: map[schema.GroupVersion]*discovery.APIVersionHandler{},
+	if delegateHandler == nil {
-		delegate:  delegationTarget.UnprotectedHandler(),
+		delegateHandler = http.NotFoundHandler()
 	}
-	groupDiscoveryHandler := &customResourceGroupDiscoveryHandler{
+
+	versionDiscoveryHandler := &versionDiscoveryHandler{
+		discovery: map[schema.GroupVersion]*discovery.APIVersionHandler{},
+		delegate:  delegateHandler,
+	}
+	groupDiscoveryHandler := &groupDiscoveryHandler{
 		discovery: map[string]*discovery.APIGroupHandler{},
-		delegate:  delegationTarget.UnprotectedHandler(),
+		delegate:  delegateHandler,
 	}
 	customResourceHandler := NewCustomResourceHandler(
 		versionDiscoveryHandler,
@@ -143,9 +149,10 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget,
 		c.CustomResourceRESTOptionsGetter,
 		c.GenericConfig.AdmissionControl,
 	)
-	s.GenericAPIServer.FallThroughHandler.Handle("/apis/", customResourceHandler)
+	s.GenericAPIServer.FallThroughHandler.Handle("/apis", customResourceHandler)
+	s.GenericAPIServer.FallThroughHandler.HandlePrefix("/apis/", customResourceHandler)
 
-	customResourceController := NewCustomResourceDiscoveryController(customResourceInformers.Apiextensions().InternalVersion().CustomResources(), versionDiscoveryHandler, groupDiscoveryHandler)
+	customResourceController := NewDiscoveryController(customResourceInformers.Apiextensions().InternalVersion().CustomResources(), versionDiscoveryHandler, groupDiscoveryHandler)
 
 	s.GenericAPIServer.AddPostStartHook("start-apiextensions-informers", func(context genericapiserver.PostStartHookContext) error {
 		customResourceInformers.Start(stopCh)

staging/src/k8s.io/kube-apiextensions-server/pkg/apiserver/customresource_discovery.go

@@ -25,7 +25,7 @@ import (
 	"k8s.io/apiserver/pkg/endpoints/discovery"
 )
 
-type customResourceVersionDiscoveryHandler struct {
+type versionDiscoveryHandler struct {
 	// TODO, writing is infrequent, optimize this
 	discoveryLock sync.RWMutex
 	discovery     map[schema.GroupVersion]*discovery.APIVersionHandler
@@ -33,10 +33,10 @@ type customResourceVersionDiscoveryHandler struct {
 	delegate http.Handler
 }
 
-func (r *customResourceVersionDiscoveryHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+func (r *versionDiscoveryHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	pathParts := splitPath(req.URL.Path)
 	// only match /apis/<group>/<version>
-	if len(pathParts) != 3 {
+	if len(pathParts) != 3 || pathParts[0] != "apis" {
 		r.delegate.ServeHTTP(w, req)
 		return
 	}
@@ -49,29 +49,29 @@ func (r *customResourceVersionDiscoveryHandler) ServeHTTP(w http.ResponseWriter,
 	discovery.ServeHTTP(w, req)
 }
 
-func (r *customResourceVersionDiscoveryHandler) getDiscovery(version schema.GroupVersion) (*discovery.APIVersionHandler, bool) {
+func (r *versionDiscoveryHandler) getDiscovery(gv schema.GroupVersion) (*discovery.APIVersionHandler, bool) {
 	r.discoveryLock.RLock()
 	defer r.discoveryLock.RUnlock()
 
-	ret, ok := r.discovery[version]
+	ret, ok := r.discovery[gv]
 	return ret, ok
 }
 
-func (r *customResourceVersionDiscoveryHandler) setDiscovery(version schema.GroupVersion, discovery *discovery.APIVersionHandler) {
+func (r *versionDiscoveryHandler) setDiscovery(gv schema.GroupVersion, discovery *discovery.APIVersionHandler) {
 	r.discoveryLock.Lock()
 	defer r.discoveryLock.Unlock()
 
-	r.discovery[version] = discovery
+	r.discovery[gv] = discovery
 }
 
-func (r *customResourceVersionDiscoveryHandler) unsetDiscovery(version schema.GroupVersion) {
+func (r *versionDiscoveryHandler) unsetDiscovery(gv schema.GroupVersion) {
 	r.discoveryLock.Lock()
 	defer r.discoveryLock.Unlock()
 
-	delete(r.discovery, version)
+	delete(r.discovery, gv)
 }
 
-type customResourceGroupDiscoveryHandler struct {
+type groupDiscoveryHandler struct {
 	// TODO, writing is infrequent, optimize this
 	discoveryLock sync.RWMutex
 	discovery     map[string]*discovery.APIGroupHandler
@@ -79,10 +79,10 @@ type customResourceGroupDiscoveryHandler struct {
 	delegate http.Handler
 }
 
-func (r *customResourceGroupDiscoveryHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+func (r *groupDiscoveryHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	pathParts := splitPath(req.URL.Path)
 	// only match /apis/<group>
-	if len(pathParts) != 2 {
+	if len(pathParts) != 2 || pathParts[0] != "apis" {
 		r.delegate.ServeHTTP(w, req)
 		return
 	}
@@ -95,7 +95,7 @@ func (r *customResourceGroupDiscoveryHandler) ServeHTTP(w http.ResponseWriter, r
 	discovery.ServeHTTP(w, req)
 }
 
-func (r *customResourceGroupDiscoveryHandler) getDiscovery(group string) (*discovery.APIGroupHandler, bool) {
+func (r *groupDiscoveryHandler) getDiscovery(group string) (*discovery.APIGroupHandler, bool) {
 	r.discoveryLock.RLock()
 	defer r.discoveryLock.RUnlock()
 
@@ -103,14 +103,14 @@ func (r *customResourceGroupDiscoveryHandler) getDiscovery(group string) (*disco
 	return ret, ok
 }
 
-func (r *customResourceGroupDiscoveryHandler) setDiscovery(group string, discovery *discovery.APIGroupHandler) {
+func (r *groupDiscoveryHandler) setDiscovery(group string, discovery *discovery.APIGroupHandler) {
 	r.discoveryLock.Lock()
 	defer r.discoveryLock.Unlock()
 
 	r.discovery[group] = discovery
 }
 
-func (r *customResourceGroupDiscoveryHandler) unsetDiscovery(group string) {
+func (r *groupDiscoveryHandler) unsetDiscovery(group string) {
 	r.discoveryLock.Lock()
 	defer r.discoveryLock.Unlock()
 

staging/src/k8s.io/kube-apiextensions-server/pkg/apiserver/customresource_discovery_controller.go

@@ -36,9 +36,9 @@ import (
 	listers "k8s.io/kube-apiextensions-server/pkg/client/listers/apiextensions/internalversion"
 )
 
-type CustomResourceDiscoveryController struct {
+type DiscoveryController struct {
-	versionHandler *customResourceVersionDiscoveryHandler
+	versionHandler *versionDiscoveryHandler
-	groupHandler   *customResourceGroupDiscoveryHandler
+	groupHandler   *groupDiscoveryHandler
 
 	customResourceLister  listers.CustomResourceLister
 	customResourcesSynced cache.InformerSynced
@@ -49,14 +49,14 @@ type CustomResourceDiscoveryController struct {
 	queue workqueue.RateLimitingInterface
 }
 
-func NewCustomResourceDiscoveryController(customResourceInformer informers.CustomResourceInformer, versionHandler *customResourceVersionDiscoveryHandler, groupHandler *customResourceGroupDiscoveryHandler) *CustomResourceDiscoveryController {
+func NewDiscoveryController(customResourceInformer informers.CustomResourceInformer, versionHandler *versionDiscoveryHandler, groupHandler *groupDiscoveryHandler) *DiscoveryController {
-	c := &CustomResourceDiscoveryController{
+	c := &DiscoveryController{
 		versionHandler:        versionHandler,
 		groupHandler:          groupHandler,
 		customResourceLister:  customResourceInformer.Lister(),
 		customResourcesSynced: customResourceInformer.Informer().HasSynced,
 
-		queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "CustomResourceDiscoveryController"),
+		queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "DiscoveryController"),
 	}
 
 	customResourceInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
@@ -70,10 +70,7 @@ func NewCustomResourceDiscoveryController(customResourceInformer informers.Custo
 	return c
 }
 
-func (c *CustomResourceDiscoveryController) sync(version schema.GroupVersion) error {
+func (c *DiscoveryController) sync(version schema.GroupVersion) error {
-
-	foundVersion := false
-	foundGroup := false
 
 	apiVersionsForDiscovery := []metav1.GroupVersionForDiscovery{}
 	apiResourcesForDiscovery := []metav1.APIResource{}
@@ -82,7 +79,11 @@ func (c *CustomResourceDiscoveryController) sync(version schema.GroupVersion) er
 	if err != nil {
 		return err
 	}
+	foundVersion := false
+	foundGroup := false
 	for _, customResource := range customResources {
+		// TODO add status checking
+
 		if customResource.Spec.Group != version.Group {
 			continue
 		}
@@ -114,8 +115,9 @@ func (c *CustomResourceDiscoveryController) sync(version schema.GroupVersion) er
 	}
 
 	apiGroup := metav1.APIGroup{
-		Name:             version.Group,
+		Name:     version.Group,
-		Versions:         apiVersionsForDiscovery,
+		Versions: apiVersionsForDiscovery,
+		// the preferred versions for a group is arbitrary since there cannot be duplicate resources
 		PreferredVersion: apiVersionsForDiscovery[0],
 	}
 	c.groupHandler.setDiscovery(version.Group, discovery.NewAPIGroupHandler(Codecs, apiGroup))
@@ -131,12 +133,12 @@ func (c *CustomResourceDiscoveryController) sync(version schema.GroupVersion) er
 	return nil
 }
 
-func (c *CustomResourceDiscoveryController) Run(stopCh <-chan struct{}) {
+func (c *DiscoveryController) Run(stopCh <-chan struct{}) {
 	defer utilruntime.HandleCrash()
 	defer c.queue.ShutDown()
-	defer glog.Infof("Shutting down CustomResourceDiscoveryController")
+	defer glog.Infof("Shutting down DiscoveryController")
 
-	glog.Infof("Starting CustomResourceDiscoveryController")
+	glog.Infof("Starting DiscoveryController")
 
 	if !cache.WaitForCacheSync(stopCh, c.customResourcesSynced) {
 		utilruntime.HandleError(fmt.Errorf("timed out waiting for caches to sync"))
@@ -149,13 +151,13 @@ func (c *CustomResourceDiscoveryController) Run(stopCh <-chan struct{}) {
 	<-stopCh
 }
 
-func (c *CustomResourceDiscoveryController) runWorker() {
+func (c *DiscoveryController) runWorker() {
 	for c.processNextWorkItem() {
 	}
 }
 
 // processNextWorkItem deals with one key off the queue.  It returns false when it's time to quit.
-func (c *CustomResourceDiscoveryController) processNextWorkItem() bool {
+func (c *DiscoveryController) processNextWorkItem() bool {
 	key, quit := c.queue.Get()
 	if quit {
 		return false
@@ -168,29 +170,29 @@ func (c *CustomResourceDiscoveryController) processNextWorkItem() bool {
 		return true
 	}
 
-	utilruntime.HandleError(fmt.Errorf("%v failed with : %v", key, err))
+	utilruntime.HandleError(fmt.Errorf("%v failed with: %v", key, err))
 	c.queue.AddRateLimited(key)
 
 	return true
 }
 
-func (c *CustomResourceDiscoveryController) enqueue(obj *apiextensions.CustomResource) {
+func (c *DiscoveryController) enqueue(obj *apiextensions.CustomResource) {
 	c.queue.Add(schema.GroupVersion{Group: obj.Spec.Group, Version: obj.Spec.Version})
 }
 
-func (c *CustomResourceDiscoveryController) addCustomResource(obj interface{}) {
+func (c *DiscoveryController) addCustomResource(obj interface{}) {
 	castObj := obj.(*apiextensions.CustomResource)
-	glog.V(4).Infof("Adding %s", castObj.Name)
+	glog.V(4).Infof("Adding customresource %s", castObj.Name)
 	c.enqueue(castObj)
 }
 
-func (c *CustomResourceDiscoveryController) updateCustomResource(obj, _ interface{}) {
+func (c *DiscoveryController) updateCustomResource(obj, _ interface{}) {
 	castObj := obj.(*apiextensions.CustomResource)
-	glog.V(4).Infof("Updating %s", castObj.Name)
+	glog.V(4).Infof("Updating customresource %s", castObj.Name)
 	c.enqueue(castObj)
 }
 
-func (c *CustomResourceDiscoveryController) deleteCustomResource(obj interface{}) {
+func (c *DiscoveryController) deleteCustomResource(obj interface{}) {
 	castObj, ok := obj.(*apiextensions.CustomResource)
 	if !ok {
 		tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
@@ -204,6 +206,6 @@ func (c *CustomResourceDiscoveryController) deleteCustomResource(obj interface{}
 			return
 		}
 	}
-	glog.V(4).Infof("Deleting %q", castObj.Name)
+	glog.V(4).Infof("Deleting customresource %q", castObj.Name)
 	c.enqueue(castObj)
 }

staging/src/k8s.io/kube-apiextensions-server/pkg/apiserver/customresource_handler.go

@@ -47,14 +47,14 @@ import (
 	"k8s.io/kube-apiextensions-server/pkg/registry/customresourcestorage"
 )
 
-// apisHandler serves the `/apis` endpoint.
+// customResourceHandler serves the `/apis` endpoint.
 // This is registered as a filter so that it never collides with any explictly registered endpoints
 type customResourceHandler struct {
-	versionDiscoveryHandler *customResourceVersionDiscoveryHandler
+	versionDiscoveryHandler *versionDiscoveryHandler
-	groupDiscoveryHandler   *customResourceGroupDiscoveryHandler
+	groupDiscoveryHandler   *groupDiscoveryHandler
 
-	storageMutationLock sync.Mutex
+	customStorageLock sync.Mutex
-	// customStorage contains a map[types.UID]*customResourceInfo
+	// customStorage contains a customResourceStorageMap
 	customStorage atomic.Value
 
 	requestContextMapper apirequest.RequestContextMapper
@@ -72,9 +72,12 @@ type customResourceInfo struct {
 	requestScope handlers.RequestScope
 }
 
+// customResourceStorageMap goes from customresource to its storage
+type customResourceStorageMap map[types.UID]*customResourceInfo
+
 func NewCustomResourceHandler(
-	versionDiscoveryHandler *customResourceVersionDiscoveryHandler,
+	versionDiscoveryHandler *versionDiscoveryHandler,
-	groupDiscoveryHandler *customResourceGroupDiscoveryHandler,
+	groupDiscoveryHandler *groupDiscoveryHandler,
 	requestContextMapper apirequest.RequestContextMapper,
 	customResourceLister listers.CustomResourceLister,
 	delegate http.Handler,
@@ -91,24 +94,27 @@ func NewCustomResourceHandler(
 		admission:               admission,
 	}
 
-	ret.customStorage.Store(map[types.UID]*customResourceInfo{})
+	ret.customStorage.Store(customResourceStorageMap{})
 	return ret
 }
 
 func (r *customResourceHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	ctx, ok := r.requestContextMapper.Get(req)
 	if !ok {
-		http.Error(w, "missing context", http.StatusInternalServerError)
+		// programmer error
+		panic("missing context")
 		return
 	}
 	requestInfo, ok := apirequest.RequestInfoFrom(ctx)
 	if !ok {
-		http.Error(w, "missing requestInfo", http.StatusInternalServerError)
+		// programmer error
+		panic("missing requestInfo")
 		return
 	}
 	if !requestInfo.IsResourceRequest {
 		pathParts := splitPath(requestInfo.Path)
 		// only match /apis/<group>/<version>
+		// only registered under /apis
 		if len(pathParts) == 3 {
 			r.versionDiscoveryHandler.ServeHTTP(w, req)
 			return
@@ -192,7 +198,7 @@ func (r *customResourceHandler) removeDeadStorage() {
 	// these don't have to be live.  A snapshot is fine
 	// if we wrongly delete, that's ok.  The rest storage will be recreated on the next request
 	// if we wrongly miss one, that's ok.  We'll get it next time
-	storageMap := r.customStorage.Load().(map[types.UID]*customResourceInfo)
+	storageMap := r.customStorage.Load().(customResourceStorageMap)
 	allCustomResources, err := r.customResourceLister.List(labels.Everything())
 	if err != nil {
 		utilruntime.HandleError(err)
@@ -212,21 +218,21 @@ func (r *customResourceHandler) removeDeadStorage() {
 		}
 	}
 
-	r.storageMutationLock.Lock()
+	r.customStorageLock.Lock()
-	defer r.storageMutationLock.Unlock()
+	defer r.customStorageLock.Unlock()
 
 	r.customStorage.Store(storageMap)
 }
 
 func (r *customResourceHandler) getServingInfoFor(customResource *apiextensions.CustomResource) *customResourceInfo {
-	storageMap := r.customStorage.Load().(map[types.UID]*customResourceInfo)
+	storageMap := r.customStorage.Load().(customResourceStorageMap)
 	ret, ok := storageMap[customResource.UID]
 	if ok {
 		return ret
 	}
 
-	r.storageMutationLock.Lock()
+	r.customStorageLock.Lock()
-	defer r.storageMutationLock.Unlock()
+	defer r.customStorageLock.Unlock()
 
 	ret, ok = storageMap[customResource.UID]
 	if ok {

staging/src/k8s.io/kube-apiextensions-server/test/integration/BUILD

@@ -10,7 +10,10 @@ load(
 go_test(
     name = "go_default_test",
     srcs = ["basic_test.go"],
-    tags = ["automanaged"],
+    tags = [
+        "automanaged",
+        "integration",
+    ],
     deps = [
         "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",

staging/src/k8s.io/kube-apiextensions-server/test/integration/apiserver.local.config/certificates/apiserver.crt

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDEzCCAfugAwIBAgIBATANBgkqhkiG9w0BAQsFADAfMR0wGwYDVQQDDBRsb2Nh
+bGhvc3RAMTQ5MzY2NDQ4OTAeFw0xNzA1MDExODQ4MDlaFw0xODA1MDExODQ4MDla
+MB8xHTAbBgNVBAMMFGxvY2FsaG9zdEAxNDkzNjY0NDg5MIIBIjANBgkqhkiG9w0B
+AQEFAAOCAQ8AMIIBCgKCAQEAy6tMZcDbG1J4vbX+YdiPswxqO+hX1r+i9+DFb1N/
+xyodBbprn1Mmd2k1lv3AkiZKm38v7dgzQ9/teA8Jm/1tyjjZSV/CxsZZWNuukPGU
+ykEtn4mvkb5tOI1159ieTBiL4mKx5VNq8DkIpy9CT22Ud9dHkJaxJHcIF601hXHg
+GIRla/6CRlkY/GFUItl1oij4sgzXRTS2pdv8lsmt2s7dXj737l10QCz9YDVuGSfu
+rYoHGwY5ofYYFWzscD7Ds4O0tPdu4mSPIu753K7nB3ilfBi+tUWcSXpw9wE4+hIF
+a1In8jnM+lw5/j/UoghrCtQ54BGWzpivPPXKv2dlNIOPiwIDAQABo1owWDAOBgNV
+HQ8BAf8EBAMCAqQwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDwYDVR0TAQH/BAUwAwEB
+/zAgBgNVHREEGTAXgglsb2NhbGhvc3SHBH8AAAGHBH8AAAEwDQYJKoZIhvcNAQEL
+BQADggEBAJbxTi/0Joxx/oja4QDksbWroip0qVJKh1ic7ryai52aSBTcHMF9pWiL
+047lL3sL0sN0YavXPUiow4PMTQm14W01ciwuZj5DCCaXnmnGtBy0fy8ifUdQoD/J
+9pvLQMWAsx+GP2XzY+KxYFQairKS7BehEF/d24TgNHPskgc2p2XgK3Z7Ipp7hQrj
+yZiTNromeULT12d5Zuwf+IeDp3aopGyhxCTOoc+RCz4MKLfKov40xjlaA4jVWazd
+ccHWnagwM5lDlXnmCqZRVvyOWaUulJCEzRFfRTHFxKgj6DSPNt00wHXNmQUvjhN/
+YXFAkfKQQEs3qQRXoHAXKquplnLgjyA=
+-----END CERTIFICATE-----

staging/src/k8s.io/kube-apiextensions-server/test/integration/apiserver.local.config/certificates/apiserver.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEAy6tMZcDbG1J4vbX+YdiPswxqO+hX1r+i9+DFb1N/xyodBbpr
+n1Mmd2k1lv3AkiZKm38v7dgzQ9/teA8Jm/1tyjjZSV/CxsZZWNuukPGUykEtn4mv
+kb5tOI1159ieTBiL4mKx5VNq8DkIpy9CT22Ud9dHkJaxJHcIF601hXHgGIRla/6C
+RlkY/GFUItl1oij4sgzXRTS2pdv8lsmt2s7dXj737l10QCz9YDVuGSfurYoHGwY5
+ofYYFWzscD7Ds4O0tPdu4mSPIu753K7nB3ilfBi+tUWcSXpw9wE4+hIFa1In8jnM
++lw5/j/UoghrCtQ54BGWzpivPPXKv2dlNIOPiwIDAQABAoIBABy4qWtoCP4PYUuP
+kLIHsiwTwh90on58Q+Uk43LRmaFihPk70tWDCleolJAYdMGneLn4869c383glEJs
+DHTdBlCQN8QrJvKVIiBvymxSRSNIkcB/0CyDaC+jc08gsyIUDBX+yQuH+fqqcFfz
+SCyfTWKhD0yKk6yKxK9iE7wf1PRf5uLtJD6x1vV0NBmsHH++feODjNVsHDRvnwy6
+3KXkgSvfCTQ7qnQPZ/MSsRxWRdMBCnhaQq9qRnJ8bv8XotrCsEG5laMybriyJYXX
+wvr9Dt04ciUD/g3qwIPy1ygMAKE9ya8hivSRURptZxz9SKCenWWihsfIzk4uyOi+
+sVDkJVECgYEA2981ZhbkruG5JhOsWVXTxDlXOrjI0pUqfej0WEp8lwdBMOrTXFs1
+cB8kdSocVC5GnMTbg6bqJrfglbNDOA7sUA8E8APLFFUAAvPwVfnrdDWk+0jK7atu
+2sixQGeIB97Y6ojWfSbjyA/0p3Z0zCTfP5SR6xt4hVWhInB11m5IpO0CgYEA7SKH
+bfWPZ0xfkFdM2X2cWQjt4zYHIUQqAKLPs2OjPTBC8PioNiZlvZq9bNseYR+eyIg6
+P9Kwe4KV/hzOSScf7JYpsN+YQ1+4Y+E63BkhAHXyz7y/vD+DA/Z83oKbelQtJqwq
+W+Zo1OGJrfZwEKK5JWN9HF+KI9Z1iMyZoyw8L1cCgYEAtaLDfj7TVBVs2qPN8U8R
+zjyAbyZP4IcRv0o+8OE345w+oqabTOScVK+lcpUDKhfAhamqniu5q5qjkYextBG/
+7rM5pP29OmKty8KxfJUlia73SA9udMD2pw68PzRIEBhsofPBHUqPSarEtcMJ4ctk
+EiYuFUdwXNXMc6Lr9eTNZlECgYEAsfFJIvAzjdY3l76KwmGJox4aNHdkXkgiJJwH
+s5s+8Tl34g8VWpzxl5e4MSkz4LmzktL2stHM8MGLAEZpXWdog0YjPsBqJ5R6byih
+3GtW4lufutbuIbqe+6hJB0eGmAL2ZqCmoJODcstTXyEf8rvIpw/C4DmpFT9mryKo
+31LgTr0CgYEAuxusmnR2vzZP/RjpjzmZcvIHf4xORG+SXlg3BXsSEd6+g3Rqiy5t
+Q0UkHHwYnYurBmJ2HL1LG9mZwU89D00F/4mJpJuWfwqtqvodIRZ7bimyGGbvKZ1t
+BGLmUssF5MYn75v7E5opxcc51aieW8nUQbop/PPMvWsYLrL/mcJNBpA=
+-----END RSA PRIVATE KEY-----

staging/src/k8s.io/kube-apiextensions-server/test/integration/basic_test.go

@@ -0,0 +1,189 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/watch"
+	apiextensionsv1alpha1 "k8s.io/kube-apiextensions-server/pkg/apis/apiextensions/v1alpha1"
+	"k8s.io/kube-apiextensions-server/test/integration/testserver"
+)
+
+func TestServerUp(t *testing.T) {
+	stopCh, _, _, err := testserver.StartDefaultServer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer close(stopCh)
+}
+
+func TestSimpleCRUD(t *testing.T) {
+	stopCh, apiExtensionClient, clientPool, err := testserver.StartDefaultServer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer close(stopCh)
+
+	noxuDefinition := testserver.NewNoxuCustomResourceDefinition()
+	noxuVersionClient, err := testserver.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, clientPool)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ns := "not-the-default"
+	noxuNamespacedResourceClient := noxuVersionClient.Resource(&metav1.APIResource{
+		Name:       noxuDefinition.Spec.Names.Plural,
+		Namespaced: noxuDefinition.Spec.Scope == apiextensionsv1alpha1.NamespaceScoped,
+	}, ns)
+	initialList, err := noxuNamespacedResourceClient.List(metav1.ListOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if e, a := 0, len(initialList.(*unstructured.UnstructuredList).Items); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+	initialListTypeMeta, err := meta.TypeAccessor(initialList)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if e, a := noxuDefinition.Spec.Group+"/"+noxuDefinition.Spec.Version, initialListTypeMeta.GetAPIVersion(); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+	if e, a := noxuDefinition.Spec.Names.ListKind, initialListTypeMeta.GetKind(); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+
+	initialListListMeta, err := meta.ListAccessor(initialList)
+	if err != nil {
+		t.Fatal(err)
+	}
+	noxuNamespacedWatch, err := noxuNamespacedResourceClient.Watch(metav1.ListOptions{ResourceVersion: initialListListMeta.GetResourceVersion()})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer noxuNamespacedWatch.Stop()
+
+	noxuInstanceToCreate := testserver.NewNoxuInstance(ns, "foo")
+	createdNoxuInstance, err := noxuNamespacedResourceClient.Create(noxuInstanceToCreate)
+	if err != nil {
+		t.Logf("%#v", createdNoxuInstance)
+		t.Fatal(err)
+	}
+	createdObjectMeta, err := meta.Accessor(createdNoxuInstance)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// it should have a UUID
+	if len(createdObjectMeta.GetUID()) == 0 {
+		t.Errorf("missing uuid: %#v", createdNoxuInstance)
+	}
+	createdTypeMeta, err := meta.TypeAccessor(createdNoxuInstance)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if e, a := noxuDefinition.Spec.Group+"/"+noxuDefinition.Spec.Version, createdTypeMeta.GetAPIVersion(); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+	if e, a := noxuDefinition.Spec.Names.Kind, createdTypeMeta.GetKind(); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+
+	select {
+	case watchEvent := <-noxuNamespacedWatch.ResultChan():
+		if e, a := watch.Added, watchEvent.Type; e != a {
+			t.Errorf("expected %v, got %v", e, a)
+			break
+		}
+		createdObjectMeta, err := meta.Accessor(watchEvent.Object)
+		if err != nil {
+			t.Fatal(err)
+		}
+		// it should have a UUID
+		if len(createdObjectMeta.GetUID()) == 0 {
+			t.Errorf("missing uuid: %#v", watchEvent.Object)
+		}
+		createdTypeMeta, err := meta.TypeAccessor(watchEvent.Object)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if e, a := noxuDefinition.Spec.Group+"/"+noxuDefinition.Spec.Version, createdTypeMeta.GetAPIVersion(); e != a {
+			t.Errorf("expected %v, got %v", e, a)
+		}
+		if e, a := noxuDefinition.Spec.Names.Kind, createdTypeMeta.GetKind(); e != a {
+			t.Errorf("expected %v, got %v", e, a)
+		}
+
+	case <-time.After(5 * time.Second):
+		t.Errorf("missing watch event")
+	}
+
+	gottenNoxuInstance, err := noxuNamespacedResourceClient.Get("foo")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if e, a := createdNoxuInstance, gottenNoxuInstance; !reflect.DeepEqual(e, a) {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+
+	listWithItem, err := noxuNamespacedResourceClient.List(metav1.ListOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if e, a := 1, len(listWithItem.(*unstructured.UnstructuredList).Items); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+	if e, a := *createdNoxuInstance, listWithItem.(*unstructured.UnstructuredList).Items[0]; !reflect.DeepEqual(e, a) {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+
+	if err := noxuNamespacedResourceClient.Delete("foo", nil); err != nil {
+		t.Fatal(err)
+	}
+
+	listWithoutItem, err := noxuNamespacedResourceClient.List(metav1.ListOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if e, a := 0, len(listWithoutItem.(*unstructured.UnstructuredList).Items); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+
+	select {
+	case watchEvent := <-noxuNamespacedWatch.ResultChan():
+		if e, a := watch.Deleted, watchEvent.Type; e != a {
+			t.Errorf("expected %v, got %v", e, a)
+			break
+		}
+		deletedObjectMeta, err := meta.Accessor(watchEvent.Object)
+		if err != nil {
+			t.Fatal(err)
+		}
+		// it should have a UUID
+		if e, a := createdObjectMeta.GetUID(), deletedObjectMeta.GetUID(); e != a {
+			t.Errorf("expected %v, got %v", e, a)
+		}
+
+	case <-time.After(5 * time.Second):
+		t.Errorf("missing watch event")
+	}
+}

staging/src/k8s.io/kube-apiextensions-server/test/integration/testserver/resources.go

@@ -0,0 +1,91 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testserver
+
+import (
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/dynamic"
+	apiextensionsv1alpha1 "k8s.io/kube-apiextensions-server/pkg/apis/apiextensions/v1alpha1"
+	"k8s.io/kube-apiextensions-server/pkg/client/clientset/clientset"
+)
+
+func NewNoxuCustomResourceDefinition() *apiextensionsv1alpha1.CustomResource {
+	return &apiextensionsv1alpha1.CustomResource{
+		ObjectMeta: metav1.ObjectMeta{Name: "noxus.mygroup.example.com"},
+		Spec: apiextensionsv1alpha1.CustomResourceSpec{
+			Group:   "mygroup.example.com",
+			Version: "v1alpha1",
+			Names: apiextensionsv1alpha1.CustomResourceNames{
+				Plural:   "noxus",
+				Singular: "nonenglishnoxu",
+				Kind:     "WishIHadChosenNoxu",
+				ListKind: "NoxuItemList",
+			},
+		},
+	}
+}
+
+func NewNoxuInstance(namespace, name string) *unstructured.Unstructured {
+	return &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"apiVersion": "mygroup.example.com/v1alpha1",
+			"kind":       "WishIHadChosenNoxu",
+			"metadata": map[string]interface{}{
+				"namespace": namespace,
+				"name":      name,
+			},
+			"content": map[string]interface{}{
+				"key": "value",
+			},
+		},
+	}
+}
+
+func CreateNewCustomResourceDefinition(customResource *apiextensionsv1alpha1.CustomResource, apiExtensionsClient clientset.Interface, clientPool dynamic.ClientPool) (*dynamic.Client, error) {
+	_, err := apiExtensionsClient.Apiextensions().CustomResources().Create(customResource)
+	if err != nil {
+		return nil, err
+	}
+
+	// wait until the resource appears in discovery
+	err = wait.PollImmediate(30*time.Millisecond, 30*time.Second, func() (bool, error) {
+		resourceList, err := apiExtensionsClient.Discovery().ServerResourcesForGroupVersion(customResource.Spec.Group + "/" + customResource.Spec.Version)
+		if err != nil {
+			return false, nil
+		}
+		for _, resource := range resourceList.APIResources {
+			if resource.Name == customResource.Spec.Names.Plural {
+				return true, nil
+			}
+		}
+		return false, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	dynamicClient, err := clientPool.ClientForGroupVersionResource(schema.GroupVersionResource{Group: customResource.Spec.Group, Version: customResource.Spec.Version, Resource: customResource.Spec.Names.Plural})
+	if err != nil {
+		return nil, err
+	}
+	return dynamicClient, nil
+}

staging/src/k8s.io/kube-apiextensions-server/test/integration/testserver/start.go

@@ -0,0 +1,178 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testserver
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/pborman/uuid"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
+	genericapiserver "k8s.io/apiserver/pkg/server"
+	"k8s.io/client-go/dynamic"
+	extensionsapiserver "k8s.io/kube-apiextensions-server/pkg/apiserver"
+	"k8s.io/kube-apiextensions-server/pkg/client/clientset/clientset"
+	"k8s.io/kube-apiextensions-server/pkg/cmd/server"
+)
+
+func DefaultServerConfig() (*extensionsapiserver.Config, error) {
+	port, err := FindFreeLocalPort()
+	if err != nil {
+		return nil, err
+	}
+
+	options := server.NewCustomResourcesServerOptions(os.Stdout, os.Stderr)
+	options.RecommendedOptions.Audit.Path = "-"
+	options.RecommendedOptions.SecureServing.BindPort = port
+	options.RecommendedOptions.Authentication.SkipInClusterLookup = true
+	options.RecommendedOptions.SecureServing.BindAddress = net.ParseIP("127.0.0.1")
+	etcdURL, ok := os.LookupEnv("KUBE_INTEGRATION_ETCD_URL")
+	if !ok {
+		etcdURL = "http://127.0.0.1:2379"
+	}
+	options.RecommendedOptions.Etcd.StorageConfig.ServerList = []string{etcdURL}
+	options.RecommendedOptions.Etcd.StorageConfig.Prefix = uuid.New()
+
+	// TODO stop copying this
+	// because there isn't currently a way to disable authentication or authorization from options
+	// explode options.Config here
+	genericConfig := genericapiserver.NewConfig(extensionsapiserver.Codecs)
+	genericConfig.Authenticator = nil
+	genericConfig.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer()
+
+	if err := options.RecommendedOptions.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{net.ParseIP("127.0.0.1")}); err != nil {
+		return nil, fmt.Errorf("error creating self-signed certificates: %v", err)
+	}
+	if err := options.RecommendedOptions.Etcd.ApplyTo(genericConfig); err != nil {
+		return nil, err
+	}
+	if err := options.RecommendedOptions.SecureServing.ApplyTo(genericConfig); err != nil {
+		return nil, err
+	}
+	if err := options.RecommendedOptions.Audit.ApplyTo(genericConfig); err != nil {
+		return nil, err
+	}
+	if err := options.RecommendedOptions.Features.ApplyTo(genericConfig); err != nil {
+		return nil, err
+	}
+
+	customResourceRESTOptionsGetter := extensionsapiserver.CustomResourceRESTOptionsGetter{
+		StorageConfig:           options.RecommendedOptions.Etcd.StorageConfig,
+		StoragePrefix:           options.RecommendedOptions.Etcd.StorageConfig.Prefix,
+		EnableWatchCache:        options.RecommendedOptions.Etcd.EnableWatchCache,
+		EnableGarbageCollection: options.RecommendedOptions.Etcd.EnableGarbageCollection,
+		DeleteCollectionWorkers: options.RecommendedOptions.Etcd.DeleteCollectionWorkers,
+	}
+	customResourceRESTOptionsGetter.StorageConfig.Codec = unstructured.UnstructuredJSONScheme
+	customResourceRESTOptionsGetter.StorageConfig.Copier = extensionsapiserver.UnstructuredCopier{}
+
+	config := &extensionsapiserver.Config{
+		GenericConfig:                   genericConfig,
+		CustomResourceRESTOptionsGetter: customResourceRESTOptionsGetter,
+	}
+
+	return config, nil
+}
+
+func StartServer(config *extensionsapiserver.Config) (chan struct{}, clientset.Interface, dynamic.ClientPool, error) {
+	stopCh := make(chan struct{})
+	server, err := config.Complete().New(genericapiserver.EmptyDelegate, stopCh)
+	if err != nil {
+		close(stopCh)
+		return nil, nil, nil, err
+	}
+	go func() {
+		err := server.GenericAPIServer.PrepareRun().Run(stopCh)
+		if err != nil {
+			close(stopCh)
+			panic(err)
+		}
+	}()
+
+	// wait until the server is healthy
+	err = wait.PollImmediate(30*time.Millisecond, 30*time.Second, func() (bool, error) {
+		healthClient, err := clientset.NewForConfig(server.GenericAPIServer.LoopbackClientConfig)
+		if err != nil {
+			return false, nil
+		}
+		healthResult := healthClient.Discovery().RESTClient().Get().AbsPath("/healthz").Do()
+		if healthResult.Error() != nil {
+			return false, nil
+		}
+		rawHealth, err := healthResult.Raw()
+		if err != nil {
+			return false, nil
+		}
+		if string(rawHealth) != "ok" {
+			return false, nil
+		}
+
+		return true, nil
+	})
+	if err != nil {
+		close(stopCh)
+		return nil, nil, nil, err
+	}
+
+	apiExtensionsClient, err := clientset.NewForConfig(server.GenericAPIServer.LoopbackClientConfig)
+	if err != nil {
+		close(stopCh)
+		return nil, nil, nil, err
+	}
+
+	bytes, _ := apiExtensionsClient.Discovery().RESTClient().Get().AbsPath("/apis/apiextensions.k8s.io/v1alpha1").DoRaw()
+	fmt.Print(string(bytes))
+
+	return stopCh, apiExtensionsClient, dynamic.NewDynamicClientPool(server.GenericAPIServer.LoopbackClientConfig), nil
+}
+
+func StartDefaultServer() (chan struct{}, clientset.Interface, dynamic.ClientPool, error) {
+	config, err := DefaultServerConfig()
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	return StartServer(config)
+}
+
+// FindFreeLocalPort returns the number of an available port number on
+// the loopback interface.  Useful for determining the port to launch
+// a server on.  Error handling required - there is a non-zero chance
+// that the returned port number will be bound by another process
+// after this function returns.
+func FindFreeLocalPort() (int, error) {
+	l, err := net.Listen("tcp", ":0")
+	if err != nil {
+		return 0, err
+	}
+	defer l.Close()
+	_, portStr, err := net.SplitHostPort(l.Addr().String())
+	if err != nil {
+		return 0, err
+	}
+	port, err := strconv.Atoi(portStr)
+	if err != nil {
+		return 0, err
+	}
+	return port, nil
+}