From fb9c10995366e8cccf450facf930d5bb509d7ad2 Mon Sep 17 00:00:00 2001 From: deads2k Date: Wed, 7 Dec 2016 10:55:33 -0500 Subject: [PATCH] add summarizing discovery controller and handlers --- .../artifacts/core-apiservices/legacy.yaml | 11 + .../core-apiservices/v1.autoscaling.yaml | 12 + .../artifacts/core-apiservices/v1.batch.yaml | 12 + .../v1alpha1.certificates.k8s.io.yaml | 12 + .../v1alpha1.rbac.authorization.k8s.io.yaml | 12 + .../core-apiservices/v1beta1.apps.yaml | 12 + .../v1beta1.authentication.k8s.io.yaml | 12 + .../v1beta1.authorization.k8s.io.yaml | 12 + .../core-apiservices/v1beta1.extensions.yaml | 12 + .../core-apiservices/v1beta1.policy.yaml | 12 + .../v1beta1.storage.k8s.io.yaml | 12 + cmd/kubernetes-discovery/pkg/apiserver/BUILD | 43 +- .../pkg/apiserver/apiserver.go | 113 ++++- .../pkg/apiserver/apiservice_controller.go | 162 +++++++ .../pkg/apiserver/handler_apis.go | 177 ++++++++ .../pkg/apiserver/handler_apis_test.go | 409 ++++++++++++++++++ hack/local-up-cluster.sh | 12 +- 17 files changed, 1042 insertions(+), 5 deletions(-) create mode 100644 cmd/kubernetes-discovery/artifacts/core-apiservices/legacy.yaml create mode 100644 cmd/kubernetes-discovery/artifacts/core-apiservices/v1.autoscaling.yaml create mode 100644 cmd/kubernetes-discovery/artifacts/core-apiservices/v1.batch.yaml create mode 100644 cmd/kubernetes-discovery/artifacts/core-apiservices/v1alpha1.certificates.k8s.io.yaml create mode 100644 cmd/kubernetes-discovery/artifacts/core-apiservices/v1alpha1.rbac.authorization.k8s.io.yaml create mode 100644 cmd/kubernetes-discovery/artifacts/core-apiservices/v1beta1.apps.yaml create mode 100644 cmd/kubernetes-discovery/artifacts/core-apiservices/v1beta1.authentication.k8s.io.yaml create mode 100644 cmd/kubernetes-discovery/artifacts/core-apiservices/v1beta1.authorization.k8s.io.yaml create mode 100644 cmd/kubernetes-discovery/artifacts/core-apiservices/v1beta1.extensions.yaml create mode 100644 cmd/kubernetes-discovery/artifacts/core-apiservices/v1beta1.policy.yaml create mode 100644 cmd/kubernetes-discovery/artifacts/core-apiservices/v1beta1.storage.k8s.io.yaml create mode 100644 cmd/kubernetes-discovery/pkg/apiserver/apiservice_controller.go create mode 100644 cmd/kubernetes-discovery/pkg/apiserver/handler_apis.go create mode 100644 cmd/kubernetes-discovery/pkg/apiserver/handler_apis_test.go diff --git a/cmd/kubernetes-discovery/artifacts/core-apiservices/legacy.yaml b/cmd/kubernetes-discovery/artifacts/core-apiservices/legacy.yaml new file mode 100644 index 00000000000..a3d89f53f9b --- /dev/null +++ b/cmd/kubernetes-discovery/artifacts/core-apiservices/legacy.yaml @@ -0,0 +1,11 @@ +apiVersion: apiregistration.k8s.io/v1alpha1 +kind: APIService +metadata: + name: v1. +spec: + version: v1 + service: + namespace: default + name: kubernetes + insecureSkipTLSVerify: true + priority: 100 diff --git a/cmd/kubernetes-discovery/artifacts/core-apiservices/v1.autoscaling.yaml b/cmd/kubernetes-discovery/artifacts/core-apiservices/v1.autoscaling.yaml new file mode 100644 index 00000000000..a604b8118c1 --- /dev/null +++ b/cmd/kubernetes-discovery/artifacts/core-apiservices/v1.autoscaling.yaml @@ -0,0 +1,12 @@ +apiVersion: apiregistration.k8s.io/v1alpha1 +kind: APIService +metadata: + name: v1.autoscaling +spec: + group: autoscaling + version: v1 + service: + namespace: default + name: kubernetes + insecureSkipTLSVerify: true + priority: 100 diff --git a/cmd/kubernetes-discovery/artifacts/core-apiservices/v1.batch.yaml b/cmd/kubernetes-discovery/artifacts/core-apiservices/v1.batch.yaml new file mode 100644 index 00000000000..3605b263f67 --- /dev/null +++ b/cmd/kubernetes-discovery/artifacts/core-apiservices/v1.batch.yaml @@ -0,0 +1,12 @@ +apiVersion: apiregistration.k8s.io/v1alpha1 +kind: APIService +metadata: + name: v1.batch +spec: + group: batch + version: v1 + service: + namespace: default + name: kubernetes + insecureSkipTLSVerify: true + priority: 100 diff --git a/cmd/kubernetes-discovery/artifacts/core-apiservices/v1alpha1.certificates.k8s.io.yaml b/cmd/kubernetes-discovery/artifacts/core-apiservices/v1alpha1.certificates.k8s.io.yaml new file mode 100644 index 00000000000..ec7a01a4440 --- /dev/null +++ b/cmd/kubernetes-discovery/artifacts/core-apiservices/v1alpha1.certificates.k8s.io.yaml @@ -0,0 +1,12 @@ +apiVersion: apiregistration.k8s.io/v1alpha1 +kind: APIService +metadata: + name: v1alpha1.certificates.k8s.io +spec: + group: certificates.k8s.io + version: v1alpha1 + service: + namespace: default + name: kubernetes + insecureSkipTLSVerify: true + priority: 100 diff --git a/cmd/kubernetes-discovery/artifacts/core-apiservices/v1alpha1.rbac.authorization.k8s.io.yaml b/cmd/kubernetes-discovery/artifacts/core-apiservices/v1alpha1.rbac.authorization.k8s.io.yaml new file mode 100644 index 00000000000..cfcf643fec0 --- /dev/null +++ b/cmd/kubernetes-discovery/artifacts/core-apiservices/v1alpha1.rbac.authorization.k8s.io.yaml @@ -0,0 +1,12 @@ +apiVersion: apiregistration.k8s.io/v1alpha1 +kind: APIService +metadata: + name: v1alpha1.rbac.authorization.k8s.io +spec: + group: rbac.authorization.k8s.io + version: v1alpha1 + service: + namespace: default + name: kubernetes + insecureSkipTLSVerify: true + priority: 100 diff --git a/cmd/kubernetes-discovery/artifacts/core-apiservices/v1beta1.apps.yaml b/cmd/kubernetes-discovery/artifacts/core-apiservices/v1beta1.apps.yaml new file mode 100644 index 00000000000..1620af3259d --- /dev/null +++ b/cmd/kubernetes-discovery/artifacts/core-apiservices/v1beta1.apps.yaml @@ -0,0 +1,12 @@ +apiVersion: apiregistration.k8s.io/v1alpha1 +kind: APIService +metadata: + name: v1beta1.apps +spec: + group: apps + version: v1beta1 + service: + namespace: default + name: kubernetes + insecureSkipTLSVerify: true + priority: 100 diff --git a/cmd/kubernetes-discovery/artifacts/core-apiservices/v1beta1.authentication.k8s.io.yaml b/cmd/kubernetes-discovery/artifacts/core-apiservices/v1beta1.authentication.k8s.io.yaml new file mode 100644 index 00000000000..d80e49f612f --- /dev/null +++ b/cmd/kubernetes-discovery/artifacts/core-apiservices/v1beta1.authentication.k8s.io.yaml @@ -0,0 +1,12 @@ +apiVersion: apiregistration.k8s.io/v1alpha1 +kind: APIService +metadata: + name: v1beta1.authentication.k8s.io +spec: + group: authentication.k8s.io + version: v1beta1 + service: + namespace: default + name: kubernetes + insecureSkipTLSVerify: true + priority: 100 diff --git a/cmd/kubernetes-discovery/artifacts/core-apiservices/v1beta1.authorization.k8s.io.yaml b/cmd/kubernetes-discovery/artifacts/core-apiservices/v1beta1.authorization.k8s.io.yaml new file mode 100644 index 00000000000..365f7904aa7 --- /dev/null +++ b/cmd/kubernetes-discovery/artifacts/core-apiservices/v1beta1.authorization.k8s.io.yaml @@ -0,0 +1,12 @@ +apiVersion: apiregistration.k8s.io/v1alpha1 +kind: APIService +metadata: + name: v1beta1.authorization.k8s.io +spec: + group: authorization.k8s.io + version: v1beta1 + service: + namespace: default + name: kubernetes + insecureSkipTLSVerify: true + priority: 100 diff --git a/cmd/kubernetes-discovery/artifacts/core-apiservices/v1beta1.extensions.yaml b/cmd/kubernetes-discovery/artifacts/core-apiservices/v1beta1.extensions.yaml new file mode 100644 index 00000000000..1deed18f97e --- /dev/null +++ b/cmd/kubernetes-discovery/artifacts/core-apiservices/v1beta1.extensions.yaml @@ -0,0 +1,12 @@ +apiVersion: apiregistration.k8s.io/v1alpha1 +kind: APIService +metadata: + name: v1beta1.extensions +spec: + group: extensions + version: v1beta1 + service: + namespace: default + name: kubernetes + insecureSkipTLSVerify: true + priority: 150 diff --git a/cmd/kubernetes-discovery/artifacts/core-apiservices/v1beta1.policy.yaml b/cmd/kubernetes-discovery/artifacts/core-apiservices/v1beta1.policy.yaml new file mode 100644 index 00000000000..c451ce67255 --- /dev/null +++ b/cmd/kubernetes-discovery/artifacts/core-apiservices/v1beta1.policy.yaml @@ -0,0 +1,12 @@ +apiVersion: apiregistration.k8s.io/v1alpha1 +kind: APIService +metadata: + name: v1beta1.policy +spec: + group: policy + version: v1beta1 + service: + namespace: default + name: kubernetes + insecureSkipTLSVerify: true + priority: 100 diff --git a/cmd/kubernetes-discovery/artifacts/core-apiservices/v1beta1.storage.k8s.io.yaml b/cmd/kubernetes-discovery/artifacts/core-apiservices/v1beta1.storage.k8s.io.yaml new file mode 100644 index 00000000000..9e72c87da09 --- /dev/null +++ b/cmd/kubernetes-discovery/artifacts/core-apiservices/v1beta1.storage.k8s.io.yaml @@ -0,0 +1,12 @@ +apiVersion: apiregistration.k8s.io/v1alpha1 +kind: APIService +metadata: + name: v1beta1.storage.k8s.io +spec: + group: storage.k8s.io + version: v1beta1 + service: + namespace: default + name: kubernetes + insecureSkipTLSVerify: true + priority: 100 diff --git a/cmd/kubernetes-discovery/pkg/apiserver/BUILD b/cmd/kubernetes-discovery/pkg/apiserver/BUILD index c09d4e2a679..f1dacc1fa55 100644 --- a/cmd/kubernetes-discovery/pkg/apiserver/BUILD +++ b/cmd/kubernetes-discovery/pkg/apiserver/BUILD @@ -12,16 +12,57 @@ load( go_library( name = "go_default_library", - srcs = ["apiserver.go"], + srcs = [ + "apiserver.go", + "apiservice_controller.go", + "handler_apis.go", + ], tags = ["automanaged"], deps = [ "//cmd/kubernetes-discovery/pkg/apis/apiregistration:go_default_library", "//cmd/kubernetes-discovery/pkg/apis/apiregistration/v1alpha1:go_default_library", + "//cmd/kubernetes-discovery/pkg/client/clientset_generated/internalclientset:go_default_library", + "//cmd/kubernetes-discovery/pkg/client/clientset_generated/release_1_5:go_default_library", + "//cmd/kubernetes-discovery/pkg/client/informers:go_default_library", + "//cmd/kubernetes-discovery/pkg/client/informers/apiregistration/internalversion:go_default_library", + "//cmd/kubernetes-discovery/pkg/client/listers/apiregistration/internalversion:go_default_library", "//cmd/kubernetes-discovery/pkg/registry/apiservice:go_default_library", + "//pkg/api:go_default_library", + "//pkg/api/errors:go_default_library", "//pkg/api/rest:go_default_library", + "//pkg/apis/meta/v1:go_default_library", + "//pkg/apiserver:go_default_library", + "//pkg/apiserver/filters:go_default_library", + "//pkg/auth/handlers:go_default_library", + "//pkg/client/cache:go_default_library", + "//pkg/controller:go_default_library", "//pkg/genericapiserver:go_default_library", + "//pkg/genericapiserver/filters:go_default_library", + "//pkg/labels:go_default_library", "//pkg/registry/generic:go_default_library", + "//pkg/runtime:go_default_library", "//pkg/runtime/schema:go_default_library", + "//pkg/util/runtime:go_default_library", + "//pkg/util/sets:go_default_library", + "//pkg/util/wait:go_default_library", + "//pkg/util/workqueue:go_default_library", "//pkg/version:go_default_library", + "//vendor:github.com/golang/glog", + ], +) + +go_test( + name = "go_default_test", + srcs = ["handler_apis_test.go"], + library = "go_default_library", + tags = ["automanaged"], + deps = [ + "//cmd/kubernetes-discovery/pkg/apis/apiregistration:go_default_library", + "//cmd/kubernetes-discovery/pkg/client/listers/apiregistration/internalversion:go_default_library", + "//pkg/api:go_default_library", + "//pkg/apis/meta/v1:go_default_library", + "//pkg/client/cache:go_default_library", + "//pkg/runtime:go_default_library", + "//pkg/util/diff:go_default_library", ], ) diff --git a/cmd/kubernetes-discovery/pkg/apiserver/apiserver.go b/cmd/kubernetes-discovery/pkg/apiserver/apiserver.go index 39d16a4846b..c211251a32f 100644 --- a/cmd/kubernetes-discovery/pkg/apiserver/apiserver.go +++ b/cmd/kubernetes-discovery/pkg/apiserver/apiserver.go @@ -17,17 +17,34 @@ limitations under the License. package apiserver import ( + "net/http" + "os" + "time" + + "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/rest" + apiserverfilters "k8s.io/kubernetes/pkg/apiserver/filters" + authhandlers "k8s.io/kubernetes/pkg/auth/handlers" "k8s.io/kubernetes/pkg/genericapiserver" + genericfilters "k8s.io/kubernetes/pkg/genericapiserver/filters" "k8s.io/kubernetes/pkg/registry/generic" "k8s.io/kubernetes/pkg/runtime/schema" + "k8s.io/kubernetes/pkg/util/sets" + "k8s.io/kubernetes/pkg/util/wait" "k8s.io/kubernetes/pkg/version" "k8s.io/kubernetes/cmd/kubernetes-discovery/pkg/apis/apiregistration" "k8s.io/kubernetes/cmd/kubernetes-discovery/pkg/apis/apiregistration/v1alpha1" + "k8s.io/kubernetes/cmd/kubernetes-discovery/pkg/client/clientset_generated/internalclientset" + clientset "k8s.io/kubernetes/cmd/kubernetes-discovery/pkg/client/clientset_generated/release_1_5" + "k8s.io/kubernetes/cmd/kubernetes-discovery/pkg/client/informers" + listers "k8s.io/kubernetes/cmd/kubernetes-discovery/pkg/client/listers/apiregistration/internalversion" apiservicestorage "k8s.io/kubernetes/cmd/kubernetes-discovery/pkg/registry/apiservice" ) +// legacyAPIServiceName is the fixed name of the only non-groupified API version +const legacyAPIServiceName = "v1." + // TODO move to genericapiserver or something like that // RESTOptionsGetter is used to construct storage for a particular resource type RESTOptionsGetter interface { @@ -44,6 +61,14 @@ type Config struct { // APIDiscoveryServer contains state for a Kubernetes cluster master/api server. type APIDiscoveryServer struct { GenericAPIServer *genericapiserver.GenericAPIServer + + // handledAPIServices tracks which APIServices have already been handled. Once endpoints are added, + // the listers that are used keep bits in sync automatically. + handledAPIServices sets.String + + // lister is used to add group handling for /apis/ discovery lookups based on + // controller state + lister listers.APIServiceLister } type completedConfig struct { @@ -67,26 +92,108 @@ func (c *Config) SkipComplete() completedConfig { // New returns a new instance of APIDiscoveryServer from the given config. func (c completedConfig) New() (*APIDiscoveryServer, error) { + informerFactory := informers.NewSharedInformerFactory( + internalclientset.NewForConfigOrDie(c.Config.GenericConfig.LoopbackClientConfig), + clientset.NewForConfigOrDie(c.Config.GenericConfig.LoopbackClientConfig), + 5*time.Minute, // this is effectively used as a refresh interval right now. Might want to do something nicer later on. + ) + + // most API servers don't need to do this, but we need a custom handler chain to handle the special /apis handling here + c.Config.GenericConfig.BuildHandlerChainsFunc = (&handlerChainConfig{ + informers: informerFactory, + }).handlerChain + genericServer, err := c.Config.GenericConfig.SkipComplete().New() // completion is done in Complete, no need for a second time if err != nil { return nil, err } s := &APIDiscoveryServer{ - GenericAPIServer: genericServer, + GenericAPIServer: genericServer, + handledAPIServices: sets.String{}, + lister: informerFactory.Apiregistration().InternalVersion().APIServices().Lister(), } apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(apiregistration.GroupName) apiGroupInfo.GroupMeta.GroupVersion = v1alpha1.SchemeGroupVersion - v1alpha1storage := map[string]rest.Storage{} v1alpha1storage["apiservices"] = apiservicestorage.NewREST(c.RESTOptionsGetter.NewFor(apiregistration.Resource("apiservices"))) - apiGroupInfo.VersionedResourcesStorageMap["v1alpha1"] = v1alpha1storage if err := s.GenericAPIServer.InstallAPIGroup(&apiGroupInfo); err != nil { return nil, err } + apiserviceRegistrationController := NewAPIServiceRegistrationController(informerFactory.Apiregistration().InternalVersion().APIServices(), s) + + s.GenericAPIServer.AddPostStartHook("start-informers", func(context genericapiserver.PostStartHookContext) error { + informerFactory.Start(wait.NeverStop) + return nil + }) + s.GenericAPIServer.AddPostStartHook("apiservice-registration-controller", func(context genericapiserver.PostStartHookContext) error { + apiserviceRegistrationController.Run(wait.NeverStop) + return nil + }) + return s, nil } + +// handlerChainConfig is the config used to build the custom handler chain for this api server +type handlerChainConfig struct { + informers informers.SharedInformerFactory +} + +// handlerChain is a method to build the handler chain for this API server. We need a custom handler chain so that we +// can have custom handling for `/apis`, since we're hosting discovery differently from anyone else and we're hosting +// the endpoints differently, since we're proxying all groups except for apiregistration.k8s.io. +func (h *handlerChainConfig) handlerChain(apiHandler http.Handler, c *genericapiserver.Config) (secure, insecure http.Handler) { + // add this as a filter so that we never collide with "already registered" failures on `/apis` + handler := WithAPIs(apiHandler, h.informers.Apiregistration().InternalVersion().APIServices()) + + handler = apiserverfilters.WithAuthorization(handler, c.RequestContextMapper, c.Authorizer) + handler = apiserverfilters.WithImpersonation(handler, c.RequestContextMapper, c.Authorizer) + // audit to stdout to help with debugging as we get this started + handler = apiserverfilters.WithAudit(handler, c.RequestContextMapper, os.Stdout) + handler = authhandlers.WithAuthentication(handler, c.RequestContextMapper, c.Authenticator, authhandlers.Unauthorized(c.SupportsBasicAuth)) + + handler = genericfilters.WithCORS(handler, c.CorsAllowedOriginList, nil, nil, nil, "true") + handler = genericfilters.WithPanicRecovery(handler, c.RequestContextMapper) + handler = genericfilters.WithTimeoutForNonLongRunningRequests(handler, c.RequestContextMapper, c.LongRunningFunc) + handler = genericfilters.WithMaxInFlightLimit(handler, c.MaxRequestsInFlight, c.MaxMutatingRequestsInFlight, c.RequestContextMapper, c.LongRunningFunc) + handler = apiserverfilters.WithRequestInfo(handler, genericapiserver.NewRequestInfoResolver(c), c.RequestContextMapper) + handler = api.WithRequestContext(handler, c.RequestContextMapper) + + return handler, nil +} + +// AddAPIService adds an API service. It is not thread-safe, so only call it on one thread at a time please. +// It's a slow moving API, so its ok to run the controller on a single thread +func (s *APIDiscoveryServer) AddAPIService(apiService *apiregistration.APIService) { + if s.handledAPIServices.Has(apiService.Name) { + return + } + // if we're dealing with the legacy group, we're done here + if apiService.Name == legacyAPIServiceName { + s.handledAPIServices.Insert(apiService.Name) + return + } + + // it's time to register the group discovery endpoint + groupPath := "/apis/" + apiService.Spec.Group + groupDiscoveryHandler := &apiGroupHandler{ + groupName: apiService.Spec.Group, + lister: s.lister, + } + // discovery is protected + s.GenericAPIServer.HandlerContainer.SecretRoutes.Handle(groupPath, groupDiscoveryHandler) + s.GenericAPIServer.HandlerContainer.SecretRoutes.Handle(groupPath+"/", groupDiscoveryHandler) + +} + +// RemoveAPIService removes the APIService from being handled. Later on it will disable the proxy endpoint. +// Right now it does nothing because our handler has to properly 404 itself since muxes don't unregister +func (s *APIDiscoveryServer) RemoveAPIService(apiServiceName string) { + if !s.handledAPIServices.Has(apiServiceName) { + return + } +} diff --git a/cmd/kubernetes-discovery/pkg/apiserver/apiservice_controller.go b/cmd/kubernetes-discovery/pkg/apiserver/apiservice_controller.go new file mode 100644 index 00000000000..9e8f5eeaeb2 --- /dev/null +++ b/cmd/kubernetes-discovery/pkg/apiserver/apiservice_controller.go @@ -0,0 +1,162 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiserver + +import ( + "fmt" + "time" + + "github.com/golang/glog" + + apierrors "k8s.io/kubernetes/pkg/api/errors" + "k8s.io/kubernetes/pkg/client/cache" + "k8s.io/kubernetes/pkg/controller" + utilruntime "k8s.io/kubernetes/pkg/util/runtime" + "k8s.io/kubernetes/pkg/util/wait" + "k8s.io/kubernetes/pkg/util/workqueue" + + "k8s.io/kubernetes/cmd/kubernetes-discovery/pkg/apis/apiregistration" + informers "k8s.io/kubernetes/cmd/kubernetes-discovery/pkg/client/informers/apiregistration/internalversion" + listers "k8s.io/kubernetes/cmd/kubernetes-discovery/pkg/client/listers/apiregistration/internalversion" +) + +type APIHandlerManager interface { + AddAPIService(apiServer *apiregistration.APIService) + RemoveAPIService(apiServerName string) +} + +type APIServiceRegistrationController struct { + apiHandlerManager APIHandlerManager + + apiServerLister listers.APIServiceLister + + // To allow injection for testing. + syncFn func(key string) error + + queue workqueue.RateLimitingInterface +} + +func NewAPIServiceRegistrationController(apiServerInformer informers.APIServiceInformer, apiHandlerManager APIHandlerManager) *APIServiceRegistrationController { + c := &APIServiceRegistrationController{ + apiHandlerManager: apiHandlerManager, + apiServerLister: apiServerInformer.Lister(), + queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "APIServiceRegistrationController"), + } + + apiServerInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: c.addAPIService, + UpdateFunc: c.updateAPIService, + DeleteFunc: c.deleteAPIService, + }) + + c.syncFn = c.sync + + return c +} + +func (c *APIServiceRegistrationController) sync(key string) error { + apiServer, err := c.apiServerLister.Get(key) + if apierrors.IsNotFound(err) { + c.apiHandlerManager.RemoveAPIService(key) + return nil + } + if err != nil { + return err + } + + c.apiHandlerManager.AddAPIService(apiServer) + return nil +} + +func (c *APIServiceRegistrationController) Run(stopCh <-chan struct{}) { + defer utilruntime.HandleCrash() + defer c.queue.ShutDown() + defer glog.Infof("Shutting down APIServiceRegistrationController") + + glog.Infof("Starting APIServiceRegistrationController") + + // only start one worker thread since its a slow moving API and the discovery server adding bits + // aren't threadsafe + go wait.Until(c.runWorker, time.Second, stopCh) + + <-stopCh +} + +func (c *APIServiceRegistrationController) runWorker() { + for c.processNextWorkItem() { + } +} + +// processNextWorkItem deals with one key off the queue. It returns false when it's time to quit. +func (c *APIServiceRegistrationController) processNextWorkItem() bool { + key, quit := c.queue.Get() + if quit { + return false + } + defer c.queue.Done(key) + + err := c.syncFn(key.(string)) + if err == nil { + c.queue.Forget(key) + return true + } + + utilruntime.HandleError(fmt.Errorf("%v failed with : %v", key, err)) + c.queue.AddRateLimited(key) + + return true +} + +func (c *APIServiceRegistrationController) enqueue(obj *apiregistration.APIService) { + key, err := controller.KeyFunc(obj) + if err != nil { + glog.Errorf("Couldn't get key for object %#v: %v", obj, err) + return + } + + c.queue.Add(key) +} + +func (c *APIServiceRegistrationController) addAPIService(obj interface{}) { + castObj := obj.(*apiregistration.APIService) + glog.V(4).Infof("Adding %s", castObj.Name) + c.enqueue(castObj) +} + +func (c *APIServiceRegistrationController) updateAPIService(obj, _ interface{}) { + castObj := obj.(*apiregistration.APIService) + glog.V(4).Infof("Updating %s", castObj.Name) + c.enqueue(castObj) +} + +func (c *APIServiceRegistrationController) deleteAPIService(obj interface{}) { + castObj, ok := obj.(*apiregistration.APIService) + if !ok { + tombstone, ok := obj.(cache.DeletedFinalStateUnknown) + if !ok { + glog.Errorf("Couldn't get object from tombstone %#v", obj) + return + } + castObj, ok = tombstone.Obj.(*apiregistration.APIService) + if !ok { + glog.Errorf("Tombstone contained object that is not expected %#v", obj) + return + } + } + glog.V(4).Infof("Deleting %q", castObj.Name) + c.enqueue(castObj) +} diff --git a/cmd/kubernetes-discovery/pkg/apiserver/handler_apis.go b/cmd/kubernetes-discovery/pkg/apiserver/handler_apis.go new file mode 100644 index 00000000000..0518c1c381a --- /dev/null +++ b/cmd/kubernetes-discovery/pkg/apiserver/handler_apis.go @@ -0,0 +1,177 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiserver + +import ( + "net/http" + "strings" + + "k8s.io/kubernetes/pkg/api" + apierrors "k8s.io/kubernetes/pkg/api/errors" + metav1 "k8s.io/kubernetes/pkg/apis/meta/v1" + "k8s.io/kubernetes/pkg/apiserver" + "k8s.io/kubernetes/pkg/labels" + "k8s.io/kubernetes/pkg/runtime" + + apiregistrationapi "k8s.io/kubernetes/cmd/kubernetes-discovery/pkg/apis/apiregistration" + apiregistrationv1alpha1api "k8s.io/kubernetes/cmd/kubernetes-discovery/pkg/apis/apiregistration/v1alpha1" + informers "k8s.io/kubernetes/cmd/kubernetes-discovery/pkg/client/informers/apiregistration/internalversion" + listers "k8s.io/kubernetes/cmd/kubernetes-discovery/pkg/client/listers/apiregistration/internalversion" +) + +// WithAPIs adds the handling for /apis and /apis/. +func WithAPIs(handler http.Handler, informer informers.APIServiceInformer) http.Handler { + apisHandler := &apisHandler{ + lister: informer.Lister(), + delegate: handler, + } + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + apisHandler.ServeHTTP(w, req) + }) +} + +// apisHandler servers the `/apis` endpoint. +// This is registered as a filter so that it never collides with any explictly registered endpoints +type apisHandler struct { + lister listers.APIServiceLister + + delegate http.Handler +} + +var discoveryGroup = metav1.APIGroup{ + Name: apiregistrationapi.GroupName, + Versions: []metav1.GroupVersionForDiscovery{ + { + GroupVersion: apiregistrationv1alpha1api.SchemeGroupVersion.String(), + Version: apiregistrationv1alpha1api.SchemeGroupVersion.Version, + }, + }, + PreferredVersion: metav1.GroupVersionForDiscovery{ + GroupVersion: apiregistrationv1alpha1api.SchemeGroupVersion.String(), + Version: apiregistrationv1alpha1api.SchemeGroupVersion.Version, + }, +} + +func (r *apisHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { + // if the URL is for OUR api group, serve it normally + if strings.HasPrefix(req.URL.Path+"/", "/apis/"+apiregistrationapi.GroupName+"/") { + r.delegate.ServeHTTP(w, req) + return + } + // don't handle URLs that aren't /apis + if req.URL.Path != "/apis" && req.URL.Path != "/apis/" { + r.delegate.ServeHTTP(w, req) + return + } + + discoveryGroupList := &metav1.APIGroupList{ + // always add OUR api group to the list first + Groups: []metav1.APIGroup{discoveryGroup}, + } + + apiServices, err := r.lister.List(labels.Everything()) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + apiServicesByGroup := apiregistrationapi.SortedByGroup(apiServices) + for _, apiGroupServers := range apiServicesByGroup { + // skip the legacy group + if len(apiGroupServers[0].Spec.Group) == 0 { + continue + } + discoveryGroupList.Groups = append(discoveryGroupList.Groups, *newDiscoveryAPIGroup(apiGroupServers)) + } + + json, err := runtime.Encode(api.Codecs.LegacyCodec(), discoveryGroupList) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if _, err := w.Write(json); err != nil { + panic(err) + } +} + +func newDiscoveryAPIGroup(apiServices []*apiregistrationapi.APIService) *metav1.APIGroup { + apiServicesByGroup := apiregistrationapi.SortedByGroup(apiServices)[0] + + discoveryGroup := &metav1.APIGroup{ + Name: apiServicesByGroup[0].Spec.Group, + PreferredVersion: metav1.GroupVersionForDiscovery{ + GroupVersion: apiServicesByGroup[0].Spec.Group + "/" + apiServicesByGroup[0].Spec.Version, + Version: apiServicesByGroup[0].Spec.Version, + }, + } + + for _, apiService := range apiServicesByGroup { + discoveryGroup.Versions = append(discoveryGroup.Versions, + metav1.GroupVersionForDiscovery{ + GroupVersion: apiService.Spec.Group + "/" + apiService.Spec.Version, + Version: apiService.Spec.Version, + }, + ) + } + + return discoveryGroup +} + +// apiGroupHandler servers the `/apis/` endpoint. +type apiGroupHandler struct { + groupName string + + lister listers.APIServiceLister +} + +func (r *apiGroupHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { + // don't handle URLs that aren't /apis/ + if req.URL.Path != "/apis/"+r.groupName && req.URL.Path != "/apis/"+r.groupName+"/" { + http.Error(w, "", http.StatusNotFound) + return + } + + apiServices, err := r.lister.List(labels.Everything()) + if statusErr, ok := err.(*apierrors.StatusError); ok && err != nil { + apiserver.WriteRawJSON(int(statusErr.Status().Code), statusErr.Status(), w) + return + } + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + apiServicesForGroup := []*apiregistrationapi.APIService{} + for _, apiService := range apiServices { + if apiService.Spec.Group == r.groupName { + apiServicesForGroup = append(apiServicesForGroup, apiService) + } + } + + if len(apiServicesForGroup) == 0 { + http.Error(w, "", http.StatusNotFound) + return + } + + json, err := runtime.Encode(api.Codecs.LegacyCodec(), newDiscoveryAPIGroup(apiServicesForGroup)) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if _, err := w.Write(json); err != nil { + panic(err) + } +} diff --git a/cmd/kubernetes-discovery/pkg/apiserver/handler_apis_test.go b/cmd/kubernetes-discovery/pkg/apiserver/handler_apis_test.go new file mode 100644 index 00000000000..606e6ac1333 --- /dev/null +++ b/cmd/kubernetes-discovery/pkg/apiserver/handler_apis_test.go @@ -0,0 +1,409 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiserver + +import ( + "io/ioutil" + "net/http" + "net/http/httptest" + "net/http/httputil" + "testing" + + "k8s.io/kubernetes/pkg/api" + metav1 "k8s.io/kubernetes/pkg/apis/meta/v1" + "k8s.io/kubernetes/pkg/client/cache" + "k8s.io/kubernetes/pkg/runtime" + "k8s.io/kubernetes/pkg/util/diff" + + "k8s.io/kubernetes/cmd/kubernetes-discovery/pkg/apis/apiregistration" + listers "k8s.io/kubernetes/cmd/kubernetes-discovery/pkg/client/listers/apiregistration/internalversion" +) + +type delegationHTTPHandler struct { + called bool +} + +func (d *delegationHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + d.called = true + w.WriteHeader(http.StatusOK) +} + +func TestAPIsDelegation(t *testing.T) { + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + delegate := &delegationHTTPHandler{} + handler := &apisHandler{ + lister: listers.NewAPIServiceLister(indexer), + delegate: delegate, + } + + server := httptest.NewServer(handler) + defer server.Close() + + pathToDelegation := map[string]bool{ + "/": true, + "/apis": false, + "/apis/": false, + "/apis/" + apiregistration.GroupName: true, + "/apis/" + apiregistration.GroupName + "/": true, + "/apis/" + apiregistration.GroupName + "/anything": true, + "/apis/" + apiregistration.GroupName + "/anything/again": true, + "/apis/something": true, + "/apis/something/nested": true, + "/apis/something/nested/deeper": true, + "/api": true, + "/api/v1": true, + "/version": true, + } + + for path, expectedDelegation := range pathToDelegation { + delegate.called = false + + resp, err := http.Get(server.URL + path) + if err != nil { + t.Errorf("%s: %v", path, err) + continue + } + if resp.StatusCode != http.StatusOK { + httputil.DumpResponse(resp, true) + t.Errorf("%s: %v", path, err) + continue + } + if e, a := expectedDelegation, delegate.called; e != a { + t.Errorf("%s: expected %v, got %v", path, e, a) + continue + } + } +} + +func TestAPIs(t *testing.T) { + tests := []struct { + name string + apiservices []*apiregistration.APIService + expected *metav1.APIGroupList + }{ + { + name: "empty", + apiservices: []*apiregistration.APIService{}, + expected: &metav1.APIGroupList{ + TypeMeta: metav1.TypeMeta{Kind: "APIGroupList", APIVersion: "v1"}, + Groups: []metav1.APIGroup{ + discoveryGroup, + }, + }, + }, + { + name: "simple add", + apiservices: []*apiregistration.APIService{ + { + ObjectMeta: api.ObjectMeta{Name: "v1.foo"}, + Spec: apiregistration.APIServiceSpec{ + Group: "foo", + Version: "v1", + Priority: 10, + }, + }, + { + ObjectMeta: api.ObjectMeta{Name: "v1.bar"}, + Spec: apiregistration.APIServiceSpec{ + Group: "bar", + Version: "v1", + Priority: 11, + }, + }, + }, + expected: &metav1.APIGroupList{ + TypeMeta: metav1.TypeMeta{Kind: "APIGroupList", APIVersion: "v1"}, + Groups: []metav1.APIGroup{ + discoveryGroup, + { + Name: "foo", + Versions: []metav1.GroupVersionForDiscovery{ + { + GroupVersion: "foo/v1", + Version: "v1", + }, + }, + PreferredVersion: metav1.GroupVersionForDiscovery{ + GroupVersion: "foo/v1", + Version: "v1", + }, + }, + { + Name: "bar", + Versions: []metav1.GroupVersionForDiscovery{ + { + GroupVersion: "bar/v1", + Version: "v1", + }, + }, + PreferredVersion: metav1.GroupVersionForDiscovery{ + GroupVersion: "bar/v1", + Version: "v1", + }, + }, + }, + }, + }, + { + name: "sorting", + apiservices: []*apiregistration.APIService{ + { + ObjectMeta: api.ObjectMeta{Name: "v1.foo"}, + Spec: apiregistration.APIServiceSpec{ + Group: "foo", + Version: "v1", + Priority: 20, + }, + }, + { + ObjectMeta: api.ObjectMeta{Name: "v2.bar"}, + Spec: apiregistration.APIServiceSpec{ + Group: "bar", + Version: "v2", + Priority: 11, + }, + }, + { + ObjectMeta: api.ObjectMeta{Name: "v2.foo"}, + Spec: apiregistration.APIServiceSpec{ + Group: "foo", + Version: "v2", + Priority: 1, + }, + }, + { + ObjectMeta: api.ObjectMeta{Name: "v1.bar"}, + Spec: apiregistration.APIServiceSpec{ + Group: "bar", + Version: "v1", + Priority: 11, + }, + }, + }, + expected: &metav1.APIGroupList{ + TypeMeta: metav1.TypeMeta{Kind: "APIGroupList", APIVersion: "v1"}, + Groups: []metav1.APIGroup{ + discoveryGroup, + { + Name: "foo", + Versions: []metav1.GroupVersionForDiscovery{ + { + GroupVersion: "foo/v2", + Version: "v2", + }, + { + GroupVersion: "foo/v1", + Version: "v1", + }, + }, + PreferredVersion: metav1.GroupVersionForDiscovery{ + GroupVersion: "foo/v2", + Version: "v2", + }, + }, + { + Name: "bar", + Versions: []metav1.GroupVersionForDiscovery{ + { + GroupVersion: "bar/v1", + Version: "v1", + }, + { + GroupVersion: "bar/v2", + Version: "v2", + }, + }, + PreferredVersion: metav1.GroupVersionForDiscovery{ + GroupVersion: "bar/v1", + Version: "v1", + }, + }, + }, + }, + }, + } + + for _, tc := range tests { + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + delegate := &delegationHTTPHandler{} + handler := &apisHandler{ + lister: listers.NewAPIServiceLister(indexer), + delegate: delegate, + } + for _, o := range tc.apiservices { + indexer.Add(o) + } + + server := httptest.NewServer(handler) + defer server.Close() + + resp, err := http.Get(server.URL + "/apis") + if err != nil { + t.Errorf("%s: %v", tc.name, err) + continue + } + bytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Errorf("%s: %v", tc.name, err) + continue + } + + actual := &metav1.APIGroupList{} + if err := runtime.DecodeInto(api.Codecs.UniversalDecoder(), bytes, actual); err != nil { + t.Errorf("%s: %v", tc.name, err) + continue + } + if !api.Semantic.DeepEqual(tc.expected, actual) { + t.Errorf("%s: %v", tc.name, diff.ObjectDiff(tc.expected, actual)) + continue + } + } +} + +func TestAPIGroupMissing(t *testing.T) { + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + handler := &apiGroupHandler{ + lister: listers.NewAPIServiceLister(indexer), + groupName: "foo", + } + + server := httptest.NewServer(handler) + defer server.Close() + + resp, err := http.Get(server.URL + "/apis/groupName/foo") + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("expected %v, got %v", resp.StatusCode, http.StatusNotFound) + } + + // foo still has no api services for it (like it was deleted), it should 404 + resp, err = http.Get(server.URL + "/apis/groupName/") + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("expected %v, got %v", resp.StatusCode, http.StatusNotFound) + } +} + +func TestAPIGroup(t *testing.T) { + tests := []struct { + name string + group string + apiservices []*apiregistration.APIService + expected *metav1.APIGroup + }{ + { + name: "sorting", + group: "foo", + apiservices: []*apiregistration.APIService{ + { + ObjectMeta: api.ObjectMeta{Name: "v1.foo"}, + Spec: apiregistration.APIServiceSpec{ + Group: "foo", + Version: "v1", + Priority: 20, + }, + }, + { + ObjectMeta: api.ObjectMeta{Name: "v2.bar"}, + Spec: apiregistration.APIServiceSpec{ + Group: "bar", + Version: "v2", + Priority: 11, + }, + }, + { + ObjectMeta: api.ObjectMeta{Name: "v2.foo"}, + Spec: apiregistration.APIServiceSpec{ + Group: "foo", + Version: "v2", + Priority: 1, + }, + }, + { + ObjectMeta: api.ObjectMeta{Name: "v1.bar"}, + Spec: apiregistration.APIServiceSpec{ + Group: "bar", + Version: "v1", + Priority: 11, + }, + }, + }, + expected: &metav1.APIGroup{ + TypeMeta: metav1.TypeMeta{Kind: "APIGroup", APIVersion: "v1"}, + Name: "foo", + Versions: []metav1.GroupVersionForDiscovery{ + { + GroupVersion: "foo/v2", + Version: "v2", + }, + { + GroupVersion: "foo/v1", + Version: "v1", + }, + }, + PreferredVersion: metav1.GroupVersionForDiscovery{ + GroupVersion: "foo/v2", + Version: "v2", + }, + }, + }, + } + + for _, tc := range tests { + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + handler := &apiGroupHandler{ + lister: listers.NewAPIServiceLister(indexer), + groupName: "foo", + } + for _, o := range tc.apiservices { + indexer.Add(o) + } + + server := httptest.NewServer(handler) + defer server.Close() + + resp, err := http.Get(server.URL + "/apis/" + tc.group) + if err != nil { + t.Errorf("%s: %v", tc.name, err) + continue + } + if resp.StatusCode != http.StatusOK { + httputil.DumpResponse(resp, true) + t.Errorf("%s", tc.name) + continue + } + bytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Errorf("%s: %v", tc.name, err) + continue + } + + actual := &metav1.APIGroup{} + if err := runtime.DecodeInto(api.Codecs.UniversalDecoder(), bytes, actual); err != nil { + t.Errorf("%s: %v", tc.name, err) + continue + } + if !api.Semantic.DeepEqual(tc.expected, actual) { + t.Errorf("%s: %v", tc.name, diff.ObjectDiff(tc.expected, actual)) + continue + } + } +} diff --git a/hack/local-up-cluster.sh b/hack/local-up-cluster.sh index c17a703035a..d9385874a1a 100755 --- a/hack/local-up-cluster.sh +++ b/hack/local-up-cluster.sh @@ -538,6 +538,9 @@ function start_discovery { return fi + ${CONTROLPLANE_SUDO} cp "${CERT_DIR}/admin.kubeconfig" "${CERT_DIR}/admin-discovery.kubeconfig" + ${CONTROLPLANE_SUDO} ${GO_OUT}/kubectl config set-cluster local-up-cluster --kubeconfig="${CERT_DIR}/admin-discovery.kubeconfig" --insecure-skip-tls-verify --server="https://${API_HOST}:${DISCOVERY_SECURE_PORT}" + DISCOVERY_SERVER_LOG=/tmp/kubernetes-discovery.log ${CONTROLPLANE_SUDO} "${GO_OUT}/kubernetes-discovery" \ --cert-dir="${CERT_DIR}" \ @@ -558,6 +561,9 @@ function start_discovery { # Wait for kubernetes-discovery to come up before launching the rest of the components. echo "Waiting for kubernetes-discovery to come up" kube::util::wait_for_url "https://${API_HOST}:${DISCOVERY_SECURE_PORT}/version" "kubernetes-discovery: " 1 ${WAIT_FOR_URL_API_SERVER} || exit 1 + + # create the "normal" api services for the core API server + ${CONTROLPLANE_SUDO} ${GO_OUT}/kubectl create -f "${KUBE_ROOT}/cmd/kubernetes-discovery/artifacts/core-apiservices" --kubeconfig="${CERT_DIR}/admin-discovery.kubeconfig" } @@ -832,7 +838,6 @@ if [[ "${START_MODE}" != "kubeletonly" ]]; then start_etcd set_service_accounts start_apiserver - start_discovery start_controller_manager start_kubeproxy start_kubedns @@ -842,6 +847,11 @@ if [[ "${START_MODE}" != "nokubelet" ]]; then start_kubelet fi +START_DISCOVERY=${START_DISCOVERY:-false} +if [[ "${START_DISCOVERY}" = true ]]; then + start_discovery +fi + print_success if [[ "${ENABLE_DAEMON}" = false ]]; then