partial

pkg/controlplane/controller/clusterauthenticationtrust/BUILD

@@ -0,0 +1,63 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["cluster_authentication_trust_controller.go"],
+    importpath = "k8s.io/kubernetes/pkg/master/controller/clusterauthenticationtrust",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/authentication/request/headerrequest:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:go_default_library",
+        "//staging/src/k8s.io/client-go/informers/core/v1:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
+        "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/cache:go_default_library",
+        "//staging/src/k8s.io/client-go/util/cert:go_default_library",
+        "//staging/src/k8s.io/client-go/util/workqueue:go_default_library",
+        "//vendor/k8s.io/klog/v2:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["cluster_authentication_trust_controller_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/authentication/request/headerrequest:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
+        "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library",
+        "//staging/src/k8s.io/client-go/testing:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/cache:go_default_library",
+        "//vendor/github.com/davecgh/go-spew/spew:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

pkg/controlplane/controller/clusterauthenticationtrust/cluster_authentication_trust_controller.go

@@ -0,0 +1,516 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package clusterauthenticationtrust
+
+import (
+	"bytes"
+	"context"
+	"crypto/x509"
+	"encoding/json"
+	"encoding/pem"
+	"fmt"
+	"reflect"
+	"strings"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/equality"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/authentication/request/headerrequest"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+	corev1informers "k8s.io/client-go/informers/core/v1"
+	"k8s.io/client-go/kubernetes"
+	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
+	corev1listers "k8s.io/client-go/listers/core/v1"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/util/cert"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog/v2"
+)
+
+const (
+	configMapNamespace = "kube-system"
+	configMapName      = "extension-apiserver-authentication"
+)
+
+// Controller holds the running state for the controller
+type Controller struct {
+	requiredAuthenticationData ClusterAuthenticationInfo
+
+	configMapLister corev1listers.ConfigMapLister
+	configMapClient corev1client.ConfigMapsGetter
+	namespaceClient corev1client.NamespacesGetter
+
+	// queue is where incoming work is placed to de-dup and to allow "easy" rate limited requeues on errors.
+	// we only ever place one entry in here, but it is keyed as usual: namespace/name
+	queue workqueue.RateLimitingInterface
+
+	// kubeSystemConfigMapInformer is tracked so that we can start these on Run
+	kubeSystemConfigMapInformer cache.SharedIndexInformer
+
+	// preRunCaches are the caches to sync before starting the work of this control loop
+	preRunCaches []cache.InformerSynced
+}
+
+// ClusterAuthenticationInfo holds the information that will included in public configmap.
+type ClusterAuthenticationInfo struct {
+	// ClientCA is the CA that can be used to verify the identity of normal clients
+	ClientCA dynamiccertificates.CAContentProvider
+
+	// RequestHeaderUsernameHeaders are the headers used by this kube-apiserver to determine username
+	RequestHeaderUsernameHeaders headerrequest.StringSliceProvider
+	// RequestHeaderGroupHeaders are the headers used by this kube-apiserver to determine groups
+	RequestHeaderGroupHeaders headerrequest.StringSliceProvider
+	// RequestHeaderExtraHeaderPrefixes are the headers used by this kube-apiserver to determine user.extra
+	RequestHeaderExtraHeaderPrefixes headerrequest.StringSliceProvider
+	// RequestHeaderAllowedNames are the sujbects allowed to act as a front proxy
+	RequestHeaderAllowedNames headerrequest.StringSliceProvider
+	// RequestHeaderCA is the CA that can be used to verify the front proxy
+	RequestHeaderCA dynamiccertificates.CAContentProvider
+}
+
+// NewClusterAuthenticationTrustController returns a controller that will maintain the kube-system configmap/extension-apiserver-authentication
+// that holds information about how to aggregated apiservers are recommended (but not required) to configure themselves.
+func NewClusterAuthenticationTrustController(requiredAuthenticationData ClusterAuthenticationInfo, kubeClient kubernetes.Interface) *Controller {
+	// we construct our own informer because we need such a small subset of the information available.  Just one namespace.
+	kubeSystemConfigMapInformer := corev1informers.NewConfigMapInformer(kubeClient, configMapNamespace, 12*time.Hour, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
+
+	c := &Controller{
+		requiredAuthenticationData:  requiredAuthenticationData,
+		configMapLister:             corev1listers.NewConfigMapLister(kubeSystemConfigMapInformer.GetIndexer()),
+		configMapClient:             kubeClient.CoreV1(),
+		namespaceClient:             kubeClient.CoreV1(),
+		queue:                       workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "cluster_authentication_trust_controller"),
+		preRunCaches:                []cache.InformerSynced{kubeSystemConfigMapInformer.HasSynced},
+		kubeSystemConfigMapInformer: kubeSystemConfigMapInformer,
+	}
+
+	kubeSystemConfigMapInformer.AddEventHandler(cache.FilteringResourceEventHandler{
+		FilterFunc: func(obj interface{}) bool {
+			if cast, ok := obj.(*corev1.ConfigMap); ok {
+				return cast.Name == configMapName
+			}
+			if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok {
+				if cast, ok := tombstone.Obj.(*corev1.ConfigMap); ok {
+					return cast.Name == configMapName
+				}
+			}
+			return true // always return true just in case.  The checks are fairly cheap
+		},
+		Handler: cache.ResourceEventHandlerFuncs{
+			// we have a filter, so any time we're called, we may as well queue. We only ever check one configmap
+			// so we don't have to be choosy about our key.
+			AddFunc: func(obj interface{}) {
+				c.queue.Add(keyFn())
+			},
+			UpdateFunc: func(oldObj, newObj interface{}) {
+				c.queue.Add(keyFn())
+			},
+			DeleteFunc: func(obj interface{}) {
+				c.queue.Add(keyFn())
+			},
+		},
+	})
+
+	return c
+}
+
+func (c *Controller) syncConfigMap() error {
+	originalAuthConfigMap, err := c.configMapLister.ConfigMaps(configMapNamespace).Get(configMapName)
+	if apierrors.IsNotFound(err) {
+		originalAuthConfigMap = &corev1.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{Namespace: configMapNamespace, Name: configMapName},
+		}
+	} else if err != nil {
+		return err
+	}
+	// keep the original to diff against later before updating
+	authConfigMap := originalAuthConfigMap.DeepCopy()
+
+	existingAuthenticationInfo, err := getClusterAuthenticationInfoFor(originalAuthConfigMap.Data)
+	if err != nil {
+		return err
+	}
+	combinedInfo, err := combinedClusterAuthenticationInfo(existingAuthenticationInfo, c.requiredAuthenticationData)
+	if err != nil {
+		return err
+	}
+	authConfigMap.Data, err = getConfigMapDataFor(combinedInfo)
+	if err != nil {
+		return err
+	}
+
+	if equality.Semantic.DeepEqual(authConfigMap, originalAuthConfigMap) {
+		klog.V(5).Info("no changes to configmap")
+		return nil
+	}
+	klog.V(2).Infof("writing updated authentication info to  %s configmaps/%s", configMapNamespace, configMapName)
+
+	if err := createNamespaceIfNeeded(c.namespaceClient, authConfigMap.Namespace); err != nil {
+		return err
+	}
+	if err := writeConfigMap(c.configMapClient, authConfigMap); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func createNamespaceIfNeeded(nsClient corev1client.NamespacesGetter, ns string) error {
+	if _, err := nsClient.Namespaces().Get(context.TODO(), ns, metav1.GetOptions{}); err == nil {
+		// the namespace already exists
+		return nil
+	}
+	newNs := &corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      ns,
+			Namespace: "",
+		},
+	}
+	_, err := nsClient.Namespaces().Create(context.TODO(), newNs, metav1.CreateOptions{})
+	if err != nil && apierrors.IsAlreadyExists(err) {
+		err = nil
+	}
+	return err
+}
+
+func writeConfigMap(configMapClient corev1client.ConfigMapsGetter, required *corev1.ConfigMap) error {
+	_, err := configMapClient.ConfigMaps(required.Namespace).Update(context.TODO(), required, metav1.UpdateOptions{})
+	if apierrors.IsNotFound(err) {
+		_, err := configMapClient.ConfigMaps(required.Namespace).Create(context.TODO(), required, metav1.CreateOptions{})
+		return err
+	}
+
+	// If the configmap is too big, clear the entire thing and count on this controller (or another one) to add the correct data back.
+	// We return the original error which causes the controller to re-queue.
+	// Too big means
+	//   1. request is so big the generic request catcher finds it
+	//   2. the content is so large that that the server sends a validation error "Too long: must have at most 1048576 characters"
+	if apierrors.IsRequestEntityTooLargeError(err) || (apierrors.IsInvalid(err) && strings.Contains(err.Error(), "Too long")) {
+		if deleteErr := configMapClient.ConfigMaps(required.Namespace).Delete(context.TODO(), required.Name, metav1.DeleteOptions{}); deleteErr != nil {
+			return deleteErr
+		}
+		return err
+	}
+
+	return err
+}
+
+// combinedClusterAuthenticationInfo combines two sets of authentication information into a new one
+func combinedClusterAuthenticationInfo(lhs, rhs ClusterAuthenticationInfo) (ClusterAuthenticationInfo, error) {
+	ret := ClusterAuthenticationInfo{
+		RequestHeaderAllowedNames:        combineUniqueStringSlices(lhs.RequestHeaderAllowedNames, rhs.RequestHeaderAllowedNames),
+		RequestHeaderExtraHeaderPrefixes: combineUniqueStringSlices(lhs.RequestHeaderExtraHeaderPrefixes, rhs.RequestHeaderExtraHeaderPrefixes),
+		RequestHeaderGroupHeaders:        combineUniqueStringSlices(lhs.RequestHeaderGroupHeaders, rhs.RequestHeaderGroupHeaders),
+		RequestHeaderUsernameHeaders:     combineUniqueStringSlices(lhs.RequestHeaderUsernameHeaders, rhs.RequestHeaderUsernameHeaders),
+	}
+
+	var err error
+	ret.ClientCA, err = combineCertLists(lhs.ClientCA, rhs.ClientCA)
+	if err != nil {
+		return ClusterAuthenticationInfo{}, err
+	}
+	ret.RequestHeaderCA, err = combineCertLists(lhs.RequestHeaderCA, rhs.RequestHeaderCA)
+	if err != nil {
+		return ClusterAuthenticationInfo{}, err
+	}
+
+	return ret, nil
+}
+
+func getConfigMapDataFor(authenticationInfo ClusterAuthenticationInfo) (map[string]string, error) {
+	data := map[string]string{}
+	if authenticationInfo.ClientCA != nil {
+		if caBytes := authenticationInfo.ClientCA.CurrentCABundleContent(); len(caBytes) > 0 {
+			data["client-ca-file"] = string(caBytes)
+		}
+	}
+
+	if authenticationInfo.RequestHeaderCA == nil {
+		return data, nil
+	}
+
+	if caBytes := authenticationInfo.RequestHeaderCA.CurrentCABundleContent(); len(caBytes) > 0 {
+		var err error
+
+		// encoding errors aren't going to get better, so just fail on them.
+		data["requestheader-username-headers"], err = jsonSerializeStringSlice(authenticationInfo.RequestHeaderUsernameHeaders.Value())
+		if err != nil {
+			return nil, err
+		}
+		data["requestheader-group-headers"], err = jsonSerializeStringSlice(authenticationInfo.RequestHeaderGroupHeaders.Value())
+		if err != nil {
+			return nil, err
+		}
+		data["requestheader-extra-headers-prefix"], err = jsonSerializeStringSlice(authenticationInfo.RequestHeaderExtraHeaderPrefixes.Value())
+		if err != nil {
+			return nil, err
+		}
+
+		data["requestheader-client-ca-file"] = string(caBytes)
+		data["requestheader-allowed-names"], err = jsonSerializeStringSlice(authenticationInfo.RequestHeaderAllowedNames.Value())
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return data, nil
+}
+
+func getClusterAuthenticationInfoFor(data map[string]string) (ClusterAuthenticationInfo, error) {
+	ret := ClusterAuthenticationInfo{}
+
+	var err error
+	ret.RequestHeaderGroupHeaders, err = jsonDeserializeStringSlice(data["requestheader-group-headers"])
+	if err != nil {
+		return ClusterAuthenticationInfo{}, err
+	}
+	ret.RequestHeaderExtraHeaderPrefixes, err = jsonDeserializeStringSlice(data["requestheader-extra-headers-prefix"])
+	if err != nil {
+		return ClusterAuthenticationInfo{}, err
+	}
+	ret.RequestHeaderAllowedNames, err = jsonDeserializeStringSlice(data["requestheader-allowed-names"])
+	if err != nil {
+		return ClusterAuthenticationInfo{}, err
+	}
+	ret.RequestHeaderUsernameHeaders, err = jsonDeserializeStringSlice(data["requestheader-username-headers"])
+	if err != nil {
+		return ClusterAuthenticationInfo{}, err
+	}
+
+	if caBundle := data["requestheader-client-ca-file"]; len(caBundle) > 0 {
+		ret.RequestHeaderCA, err = dynamiccertificates.NewStaticCAContent("existing", []byte(caBundle))
+		if err != nil {
+			return ClusterAuthenticationInfo{}, err
+		}
+	}
+
+	if caBundle := data["client-ca-file"]; len(caBundle) > 0 {
+		ret.ClientCA, err = dynamiccertificates.NewStaticCAContent("existing", []byte(caBundle))
+		if err != nil {
+			return ClusterAuthenticationInfo{}, err
+		}
+	}
+
+	return ret, nil
+}
+
+func jsonSerializeStringSlice(in []string) (string, error) {
+	out, err := json.Marshal(in)
+	if err != nil {
+		return "", err
+	}
+	return string(out), err
+}
+
+func jsonDeserializeStringSlice(in string) (headerrequest.StringSliceProvider, error) {
+	if len(in) == 0 {
+		return nil, nil
+	}
+
+	out := []string{}
+	if err := json.Unmarshal([]byte(in), &out); err != nil {
+		return nil, err
+	}
+	return headerrequest.StaticStringSlice(out), nil
+}
+
+func combineUniqueStringSlices(lhs, rhs headerrequest.StringSliceProvider) headerrequest.StringSliceProvider {
+	ret := []string{}
+	present := sets.String{}
+
+	if lhs != nil {
+		for _, curr := range lhs.Value() {
+			if present.Has(curr) {
+				continue
+			}
+			ret = append(ret, curr)
+			present.Insert(curr)
+		}
+	}
+
+	if rhs != nil {
+		for _, curr := range rhs.Value() {
+			if present.Has(curr) {
+				continue
+			}
+			ret = append(ret, curr)
+			present.Insert(curr)
+		}
+	}
+
+	return headerrequest.StaticStringSlice(ret)
+}
+
+func combineCertLists(lhs, rhs dynamiccertificates.CAContentProvider) (dynamiccertificates.CAContentProvider, error) {
+	certificates := []*x509.Certificate{}
+
+	if lhs != nil {
+		lhsCABytes := lhs.CurrentCABundleContent()
+		lhsCAs, err := cert.ParseCertsPEM(lhsCABytes)
+		if err != nil {
+			return nil, err
+		}
+		certificates = append(certificates, lhsCAs...)
+	}
+	if rhs != nil {
+		rhsCABytes := rhs.CurrentCABundleContent()
+		rhsCAs, err := cert.ParseCertsPEM(rhsCABytes)
+		if err != nil {
+			return nil, err
+		}
+		certificates = append(certificates, rhsCAs...)
+	}
+
+	certificates = filterExpiredCerts(certificates...)
+
+	finalCertificates := []*x509.Certificate{}
+	// now check for duplicates. n^2, but super simple
+	for i := range certificates {
+		found := false
+		for j := range finalCertificates {
+			if reflect.DeepEqual(certificates[i].Raw, finalCertificates[j].Raw) {
+				found = true
+				break
+			}
+		}
+		if !found {
+			finalCertificates = append(finalCertificates, certificates[i])
+		}
+	}
+
+	finalCABytes, err := encodeCertificates(finalCertificates...)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(finalCABytes) == 0 {
+		return nil, nil
+	}
+	// it makes sense for this list to be static because the combination of sources is only used just before writing and
+	// is recalculated
+	return dynamiccertificates.NewStaticCAContent("combined", finalCABytes)
+}
+
+// filterExpiredCerts checks are all certificates in the bundle valid, i.e. they have not expired.
+// The function returns new bundle with only valid certificates or error if no valid certificate is found.
+// We allow five minutes of slack for NotAfter comparisons
+func filterExpiredCerts(certs ...*x509.Certificate) []*x509.Certificate {
+	fiveMinutesAgo := time.Now().Add(-5 * time.Minute)
+
+	var validCerts []*x509.Certificate
+	for _, c := range certs {
+		if c.NotAfter.After(fiveMinutesAgo) {
+			validCerts = append(validCerts, c)
+		}
+	}
+
+	return validCerts
+}
+
+// Enqueue a method to allow separate control loops to cause the controller to trigger and reconcile content.
+func (c *Controller) Enqueue() {
+	c.queue.Add(keyFn())
+}
+
+// Run the controller until stopped.
+func (c *Controller) Run(threadiness int, stopCh <-chan struct{}) {
+	defer utilruntime.HandleCrash()
+	// make sure the work queue is shutdown which will trigger workers to end
+	defer c.queue.ShutDown()
+
+	klog.Infof("Starting cluster_authentication_trust_controller controller")
+	defer klog.Infof("Shutting down cluster_authentication_trust_controller controller")
+
+	// we have a personal informer that is narrowly scoped, start it.
+	go c.kubeSystemConfigMapInformer.Run(stopCh)
+
+	// wait for your secondary caches to fill before starting your work
+	if !cache.WaitForNamedCacheSync("cluster_authentication_trust_controller", stopCh, c.preRunCaches...) {
+		return
+	}
+
+	// only run one worker
+	go wait.Until(c.runWorker, time.Second, stopCh)
+
+	// checks are cheap.  run once a minute just to be sure we stay in sync in case fsnotify fails again
+	// start timer that rechecks every minute, just in case.  this also serves to prime the controller quickly.
+	_ = wait.PollImmediateUntil(1*time.Minute, func() (bool, error) {
+		c.queue.Add(keyFn())
+		return false, nil
+	}, stopCh)
+
+	// wait until we're told to stop
+	<-stopCh
+}
+
+func (c *Controller) runWorker() {
+	// hot loop until we're told to stop.  processNextWorkItem will automatically wait until there's work
+	// available, so we don't worry about secondary waits
+	for c.processNextWorkItem() {
+	}
+}
+
+// processNextWorkItem deals with one key off the queue.  It returns false when it's time to quit.
+func (c *Controller) processNextWorkItem() bool {
+	// pull the next work item from queue.  It should be a key we use to lookup something in a cache
+	key, quit := c.queue.Get()
+	if quit {
+		return false
+	}
+	// you always have to indicate to the queue that you've completed a piece of work
+	defer c.queue.Done(key)
+
+	// do your work on the key.  This method will contains your "do stuff" logic
+	err := c.syncConfigMap()
+	if err == nil {
+		// if you had no error, tell the queue to stop tracking history for your key.  This will
+		// reset things like failure counts for per-item rate limiting
+		c.queue.Forget(key)
+		return true
+	}
+
+	// there was a failure so be sure to report it.  This method allows for pluggable error handling
+	// which can be used for things like cluster-monitoring
+	utilruntime.HandleError(fmt.Errorf("%v failed with : %v", key, err))
+	// since we failed, we should requeue the item to work on later.  This method will add a backoff
+	// to avoid hotlooping on particular items (they're probably still not going to work right away)
+	// and overall controller protection (everything I've done is broken, this controller needs to
+	// calm down or it can starve other useful work) cases.
+	c.queue.AddRateLimited(key)
+
+	return true
+}
+
+func keyFn() string {
+	// this format matches DeletionHandlingMetaNamespaceKeyFunc for our single key
+	return configMapNamespace + "/" + configMapName
+}
+
+func encodeCertificates(certs ...*x509.Certificate) ([]byte, error) {
+	b := bytes.Buffer{}
+	for _, cert := range certs {
+		if err := pem.Encode(&b, &pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw}); err != nil {
+			return []byte{}, err
+		}
+	}
+	return b.Bytes(), nil
+}

pkg/controlplane/controller/clusterauthenticationtrust/cluster_authentication_trust_controller_test.go

@@ -0,0 +1,386 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package clusterauthenticationtrust
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/davecgh/go-spew/spew"
+
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/diff"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+	"k8s.io/apiserver/pkg/authentication/request/headerrequest"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+	"k8s.io/client-go/kubernetes/fake"
+	corev1listers "k8s.io/client-go/listers/core/v1"
+	clienttesting "k8s.io/client-go/testing"
+	"k8s.io/client-go/tools/cache"
+)
+
+var (
+	someRandomCA = []byte(`-----BEGIN CERTIFICATE-----
+MIIBqDCCAU2gAwIBAgIUfbqeieihh/oERbfvRm38XvS/xHAwCgYIKoZIzj0EAwIw
+GjEYMBYGA1UEAxMPSW50ZXJtZWRpYXRlLUNBMCAXDTE2MTAxMTA1MDYwMFoYDzIx
+MTYwOTE3MDUwNjAwWjAUMRIwEAYDVQQDEwlNeSBDbGllbnQwWTATBgcqhkjOPQIB
+BggqhkjOPQMBBwNCAARv6N4R/sjMR65iMFGNLN1GC/vd7WhDW6J4X/iAjkRLLnNb
+KbRG/AtOUZ+7upJ3BWIRKYbOabbQGQe2BbKFiap4o3UwczAOBgNVHQ8BAf8EBAMC
+BaAwEwYDVR0lBAwwCgYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU
+K/pZOWpNcYai6eHFpmJEeFpeQlEwHwYDVR0jBBgwFoAUX6nQlxjfWnP6aM1meO/Q
+a6b3a9kwCgYIKoZIzj0EAwIDSQAwRgIhAIWTKw/sjJITqeuNzJDAKU4xo1zL+xJ5
+MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps=
+-----END CERTIFICATE-----
+`)
+	anotherRandomCA = []byte(`-----BEGIN CERTIFICATE-----
+MIIDQDCCAiigAwIBAgIJANWw74P5KJk2MA0GCSqGSIb3DQEBCwUAMDQxMjAwBgNV
+BAMMKWdlbmVyaWNfd2ViaG9va19hZG1pc3Npb25fcGx1Z2luX3Rlc3RzX2NhMCAX
+DTE3MTExNjAwMDUzOVoYDzIyOTEwOTAxMDAwNTM5WjAjMSEwHwYDVQQDExh3ZWJo
+b29rLXRlc3QuZGVmYXVsdC5zdmMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
+AoIBAQDXd/nQ89a5H8ifEsigmMd01Ib6NVR3bkJjtkvYnTbdfYEBj7UzqOQtHoLa
+dIVmefny5uIHvj93WD8WDVPB3jX2JHrXkDTXd/6o6jIXHcsUfFTVLp6/bZ+Anqe0
+r/7hAPkzA2A7APyTWM3ZbEeo1afXogXhOJ1u/wz0DflgcB21gNho4kKTONXO3NHD
+XLpspFqSkxfEfKVDJaYAoMnYZJtFNsa2OvsmLnhYF8bjeT3i07lfwrhUZvP+7Gsp
+7UgUwc06WuNHjfx1s5e6ySzH0QioMD1rjYneqOvk0pKrMIhuAEWXqq7jlXcDtx1E
+j+wnYbVqqVYheHZ8BCJoVAAQGs9/AgMBAAGjZDBiMAkGA1UdEwQCMAAwCwYDVR0P
+BAQDAgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATApBgNVHREEIjAg
+hwR/AAABghh3ZWJob29rLXRlc3QuZGVmYXVsdC5zdmMwDQYJKoZIhvcNAQELBQAD
+ggEBAD/GKSPNyQuAOw/jsYZesb+RMedbkzs18sSwlxAJQMUrrXwlVdHrA8q5WhE6
+ABLqU1b8lQ8AWun07R8k5tqTmNvCARrAPRUqls/ryER+3Y9YEcxEaTc3jKNZFLbc
+T6YtcnkdhxsiO136wtiuatpYL91RgCmuSpR8+7jEHhuFU01iaASu7ypFrUzrKHTF
+bKwiLRQi1cMzVcLErq5CDEKiKhUkoDucyARFszrGt9vNIl/YCcBOkcNvM3c05Hn3
+M++C29JwS3Hwbubg6WO3wjFjoEhpCwU6qRYUz3MRp4tHO4kxKXx+oQnUiFnR7vW0
+YkNtGc1RUDHwecCTFpJtPb7Yu/E=
+-----END CERTIFICATE-----
+`)
+
+	someRandomCAProvider    dynamiccertificates.CAContentProvider
+	anotherRandomCAProvider dynamiccertificates.CAContentProvider
+)
+
+func init() {
+	var err error
+	someRandomCAProvider, err = dynamiccertificates.NewStaticCAContent("foo", someRandomCA)
+	if err != nil {
+		panic(err)
+	}
+	anotherRandomCAProvider, err = dynamiccertificates.NewStaticCAContent("bar", anotherRandomCA)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func TestWriteClientCAs(t *testing.T) {
+	tests := []struct {
+		name               string
+		clusterAuthInfo    ClusterAuthenticationInfo
+		preexistingObjs    []runtime.Object
+		expectedConfigMaps map[string]*corev1.ConfigMap
+		expectCreate       bool
+	}{
+		{
+			name: "basic",
+			clusterAuthInfo: ClusterAuthenticationInfo{
+				ClientCA:                         someRandomCAProvider,
+				RequestHeaderUsernameHeaders:     headerrequest.StaticStringSlice{"alfa", "bravo", "charlie"},
+				RequestHeaderGroupHeaders:        headerrequest.StaticStringSlice{"delta"},
+				RequestHeaderExtraHeaderPrefixes: headerrequest.StaticStringSlice{"echo", "foxtrot"},
+				RequestHeaderCA:                  anotherRandomCAProvider,
+				RequestHeaderAllowedNames:        headerrequest.StaticStringSlice{"first", "second"},
+			},
+			expectedConfigMaps: map[string]*corev1.ConfigMap{
+				"extension-apiserver-authentication": {
+					ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"},
+					Data: map[string]string{
+						"client-ca-file":                     string(someRandomCA),
+						"requestheader-username-headers":     `["alfa","bravo","charlie"]`,
+						"requestheader-group-headers":        `["delta"]`,
+						"requestheader-extra-headers-prefix": `["echo","foxtrot"]`,
+						"requestheader-client-ca-file":       string(anotherRandomCA),
+						"requestheader-allowed-names":        `["first","second"]`,
+					},
+				},
+			},
+			expectCreate: true,
+		},
+		{
+			name: "skip extension-apiserver-authentication",
+			clusterAuthInfo: ClusterAuthenticationInfo{
+				RequestHeaderCA:           anotherRandomCAProvider,
+				RequestHeaderAllowedNames: headerrequest.StaticStringSlice{"first", "second"},
+			},
+			expectedConfigMaps: map[string]*corev1.ConfigMap{
+				"extension-apiserver-authentication": {
+					ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"},
+					Data: map[string]string{
+						"requestheader-username-headers":     `[]`,
+						"requestheader-group-headers":        `[]`,
+						"requestheader-extra-headers-prefix": `[]`,
+						"requestheader-client-ca-file":       string(anotherRandomCA),
+						"requestheader-allowed-names":        `["first","second"]`,
+					},
+				},
+			},
+			expectCreate: true,
+		},
+		{
+			name: "skip extension-apiserver-authentication",
+			clusterAuthInfo: ClusterAuthenticationInfo{
+				ClientCA: someRandomCAProvider,
+			},
+			expectedConfigMaps: map[string]*corev1.ConfigMap{
+				"extension-apiserver-authentication": {
+					ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"},
+					Data: map[string]string{
+						"client-ca-file": string(someRandomCA),
+					},
+				},
+			},
+			expectCreate: true,
+		},
+		{
+			name: "empty allowed names",
+			clusterAuthInfo: ClusterAuthenticationInfo{
+				RequestHeaderCA: anotherRandomCAProvider,
+			},
+			expectedConfigMaps: map[string]*corev1.ConfigMap{
+				"extension-apiserver-authentication": {
+					ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"},
+					Data: map[string]string{
+						"requestheader-username-headers":     `[]`,
+						"requestheader-group-headers":        `[]`,
+						"requestheader-extra-headers-prefix": `[]`,
+						"requestheader-client-ca-file":       string(anotherRandomCA),
+						"requestheader-allowed-names":        `[]`,
+					},
+				},
+			},
+			expectCreate: true,
+		},
+		{
+			name: "overwrite extension-apiserver-authentication",
+			clusterAuthInfo: ClusterAuthenticationInfo{
+				ClientCA: someRandomCAProvider,
+			},
+			preexistingObjs: []runtime.Object{
+				&corev1.ConfigMap{
+					ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"},
+					Data: map[string]string{
+						"client-ca-file": string(anotherRandomCA),
+					},
+				},
+			},
+			expectedConfigMaps: map[string]*corev1.ConfigMap{
+				"extension-apiserver-authentication": {
+					ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"},
+					Data: map[string]string{
+						"client-ca-file": string(anotherRandomCA) + string(someRandomCA),
+					},
+				},
+			},
+		},
+		{
+			name: "overwrite extension-apiserver-authentication requestheader",
+			clusterAuthInfo: ClusterAuthenticationInfo{
+				RequestHeaderUsernameHeaders:     headerrequest.StaticStringSlice{},
+				RequestHeaderGroupHeaders:        headerrequest.StaticStringSlice{},
+				RequestHeaderExtraHeaderPrefixes: headerrequest.StaticStringSlice{},
+				RequestHeaderCA:                  anotherRandomCAProvider,
+				RequestHeaderAllowedNames:        headerrequest.StaticStringSlice{},
+			},
+			preexistingObjs: []runtime.Object{
+				&corev1.ConfigMap{
+					ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"},
+					Data: map[string]string{
+						"requestheader-username-headers":     `[]`,
+						"requestheader-group-headers":        `[]`,
+						"requestheader-extra-headers-prefix": `[]`,
+						"requestheader-client-ca-file":       string(someRandomCA),
+						"requestheader-allowed-names":        `[]`,
+					},
+				},
+			},
+			expectedConfigMaps: map[string]*corev1.ConfigMap{
+				"extension-apiserver-authentication": {
+					ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"},
+					Data: map[string]string{
+						"requestheader-username-headers":     `[]`,
+						"requestheader-group-headers":        `[]`,
+						"requestheader-extra-headers-prefix": `[]`,
+						"requestheader-client-ca-file":       string(someRandomCA) + string(anotherRandomCA),
+						"requestheader-allowed-names":        `[]`,
+					},
+				},
+			},
+		},
+		{
+			name: "namespace exists",
+			clusterAuthInfo: ClusterAuthenticationInfo{
+				ClientCA: someRandomCAProvider,
+			},
+			preexistingObjs: []runtime.Object{
+				&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: metav1.NamespaceSystem}},
+			},
+			expectedConfigMaps: map[string]*corev1.ConfigMap{
+				"extension-apiserver-authentication": {
+					ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"},
+					Data: map[string]string{
+						"client-ca-file": string(someRandomCA),
+					},
+				},
+			},
+			expectCreate: true,
+		},
+		{
+			name: "skip on no change",
+			clusterAuthInfo: ClusterAuthenticationInfo{
+				RequestHeaderUsernameHeaders:     headerrequest.StaticStringSlice{},
+				RequestHeaderGroupHeaders:        headerrequest.StaticStringSlice{},
+				RequestHeaderExtraHeaderPrefixes: headerrequest.StaticStringSlice{},
+				RequestHeaderCA:                  anotherRandomCAProvider,
+				RequestHeaderAllowedNames:        headerrequest.StaticStringSlice{},
+			},
+			preexistingObjs: []runtime.Object{
+				&corev1.ConfigMap{
+					ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"},
+					Data: map[string]string{
+						"requestheader-username-headers":     `[]`,
+						"requestheader-group-headers":        `[]`,
+						"requestheader-extra-headers-prefix": `[]`,
+						"requestheader-client-ca-file":       string(anotherRandomCA),
+						"requestheader-allowed-names":        `[]`,
+					},
+				},
+			},
+			expectedConfigMaps: map[string]*corev1.ConfigMap{},
+			expectCreate:       false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			client := fake.NewSimpleClientset(test.preexistingObjs...)
+			configMapIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
+			for _, obj := range test.preexistingObjs {
+				configMapIndexer.Add(obj)
+			}
+			configmapLister := corev1listers.NewConfigMapLister(configMapIndexer)
+
+			c := &Controller{
+				configMapLister:            configmapLister,
+				configMapClient:            client.CoreV1(),
+				namespaceClient:            client.CoreV1(),
+				requiredAuthenticationData: test.clusterAuthInfo,
+			}
+
+			err := c.syncConfigMap()
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			actualConfigMaps, updated := getFinalConfigMaps(t, client)
+			if !reflect.DeepEqual(test.expectedConfigMaps, actualConfigMaps) {
+				t.Fatalf("%s: %v", test.name, diff.ObjectReflectDiff(test.expectedConfigMaps, actualConfigMaps))
+			}
+			if test.expectCreate != updated {
+				t.Fatalf("%s: expected %v, got %v", test.name, test.expectCreate, updated)
+			}
+		})
+	}
+}
+
+func getFinalConfigMaps(t *testing.T, client *fake.Clientset) (map[string]*corev1.ConfigMap, bool) {
+	ret := map[string]*corev1.ConfigMap{}
+	created := false
+
+	for _, action := range client.Actions() {
+		t.Log(spew.Sdump(action))
+		if action.Matches("create", "configmaps") {
+			created = true
+			obj := action.(clienttesting.CreateAction).GetObject().(*corev1.ConfigMap)
+			ret[obj.Name] = obj
+		}
+		if action.Matches("update", "configmaps") {
+			obj := action.(clienttesting.UpdateAction).GetObject().(*corev1.ConfigMap)
+			ret[obj.Name] = obj
+		}
+	}
+	return ret, created
+}
+
+func TestWriteConfigMapDeleted(t *testing.T) {
+	// the basics are tested above, this checks the deletion logic when the ca bundles are too large
+	cm := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"},
+		Data: map[string]string{
+			"requestheader-username-headers":     `[]`,
+			"requestheader-group-headers":        `[]`,
+			"requestheader-extra-headers-prefix": `[]`,
+			"requestheader-client-ca-file":       string(anotherRandomCA),
+			"requestheader-allowed-names":        `[]`,
+		},
+	}
+
+	t.Run("request entity too large", func(t *testing.T) {
+		client := fake.NewSimpleClientset()
+		client.PrependReactor("update", "configmaps", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) {
+			return true, nil, apierrors.NewRequestEntityTooLargeError("way too big")
+		})
+		client.PrependReactor("delete", "configmaps", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) {
+			return true, nil, nil
+		})
+
+		err := writeConfigMap(client.CoreV1(), cm)
+		if err == nil || err.Error() != "Request entity too large: way too big" {
+			t.Fatal(err)
+		}
+		if len(client.Actions()) != 2 {
+			t.Fatal(client.Actions())
+		}
+		_, ok := client.Actions()[1].(clienttesting.DeleteAction)
+		if !ok {
+			t.Fatal(client.Actions())
+		}
+	})
+
+	t.Run("ca bundle too large", func(t *testing.T) {
+		client := fake.NewSimpleClientset()
+		client.PrependReactor("update", "configmaps", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) {
+			return true, nil, apierrors.NewInvalid(schema.GroupKind{Kind: "ConfigMap"}, cm.Name, field.ErrorList{field.TooLong(field.NewPath(""), cm, corev1.MaxSecretSize)})
+		})
+		client.PrependReactor("delete", "configmaps", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) {
+			return true, nil, nil
+		})
+
+		err := writeConfigMap(client.CoreV1(), cm)
+		if err == nil || err.Error() != `ConfigMap "extension-apiserver-authentication" is invalid: []: Too long: must have at most 1048576 bytes` {
+			t.Fatal(err)
+		}
+		if len(client.Actions()) != 2 {
+			t.Fatal(client.Actions())
+		}
+		_, ok := client.Actions()[1].(clienttesting.DeleteAction)
+		if !ok {
+			t.Fatal(client.Actions())
+		}
+	})
+
+}

pkg/controlplane/controller/crdregistration/BUILD

@@ -0,0 +1,54 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["crdregistration_controller.go"],
+    importpath = "k8s.io/kubernetes/pkg/master/controller/crdregistration",
+    deps = [
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions/apiextensions/v1:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/cache:go_default_library",
+        "//staging/src/k8s.io/client-go/util/workqueue:go_default_library",
+        "//staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1:go_default_library",
+        "//vendor/k8s.io/klog/v2:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["crdregistration_controller_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/cache:go_default_library",
+        "//staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1:go_default_library",
+    ],
+)

pkg/controlplane/controller/crdregistration/crdregistration_controller.go

@@ -0,0 +1,225 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crdregistration
+
+import (
+	"fmt"
+	"time"
+
+	"k8s.io/klog/v2"
+
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	crdinformers "k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions/apiextensions/v1"
+	crdlisters "k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
+)
+
+// AutoAPIServiceRegistration is an interface which callers can re-declare locally and properly cast to for
+// adding and removing APIServices
+type AutoAPIServiceRegistration interface {
+	// AddAPIServiceToSync adds an API service to auto-register.
+	AddAPIServiceToSync(in *v1.APIService)
+	// RemoveAPIServiceToSync removes an API service to auto-register.
+	RemoveAPIServiceToSync(name string)
+}
+
+type crdRegistrationController struct {
+	crdLister crdlisters.CustomResourceDefinitionLister
+	crdSynced cache.InformerSynced
+
+	apiServiceRegistration AutoAPIServiceRegistration
+
+	syncHandler func(groupVersion schema.GroupVersion) error
+
+	syncedInitialSet chan struct{}
+
+	// queue is where incoming work is placed to de-dup and to allow "easy" rate limited requeues on errors
+	// this is actually keyed by a groupVersion
+	queue workqueue.RateLimitingInterface
+}
+
+// NewCRDRegistrationController returns a controller which will register CRD GroupVersions with the auto APIService registration
+// controller so they automatically stay in sync.
+func NewCRDRegistrationController(crdinformer crdinformers.CustomResourceDefinitionInformer, apiServiceRegistration AutoAPIServiceRegistration) *crdRegistrationController {
+	c := &crdRegistrationController{
+		crdLister:              crdinformer.Lister(),
+		crdSynced:              crdinformer.Informer().HasSynced,
+		apiServiceRegistration: apiServiceRegistration,
+		syncedInitialSet:       make(chan struct{}),
+		queue:                  workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "crd_autoregistration_controller"),
+	}
+	c.syncHandler = c.handleVersionUpdate
+
+	crdinformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc: func(obj interface{}) {
+			cast := obj.(*apiextensionsv1.CustomResourceDefinition)
+			c.enqueueCRD(cast)
+		},
+		UpdateFunc: func(oldObj, newObj interface{}) {
+			// Enqueue both old and new object to make sure we remove and add appropriate API services.
+			// The working queue will resolve any duplicates and only changes will stay in the queue.
+			c.enqueueCRD(oldObj.(*apiextensionsv1.CustomResourceDefinition))
+			c.enqueueCRD(newObj.(*apiextensionsv1.CustomResourceDefinition))
+		},
+		DeleteFunc: func(obj interface{}) {
+			cast, ok := obj.(*apiextensionsv1.CustomResourceDefinition)
+			if !ok {
+				tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
+				if !ok {
+					klog.V(2).Infof("Couldn't get object from tombstone %#v", obj)
+					return
+				}
+				cast, ok = tombstone.Obj.(*apiextensionsv1.CustomResourceDefinition)
+				if !ok {
+					klog.V(2).Infof("Tombstone contained unexpected object: %#v", obj)
+					return
+				}
+			}
+			c.enqueueCRD(cast)
+		},
+	})
+
+	return c
+}
+
+func (c *crdRegistrationController) Run(threadiness int, stopCh <-chan struct{}) {
+	defer utilruntime.HandleCrash()
+	// make sure the work queue is shutdown which will trigger workers to end
+	defer c.queue.ShutDown()
+
+	klog.Infof("Starting crd-autoregister controller")
+	defer klog.Infof("Shutting down crd-autoregister controller")
+
+	// wait for your secondary caches to fill before starting your work
+	if !cache.WaitForNamedCacheSync("crd-autoregister", stopCh, c.crdSynced) {
+		return
+	}
+
+	// process each item in the list once
+	if crds, err := c.crdLister.List(labels.Everything()); err != nil {
+		utilruntime.HandleError(err)
+	} else {
+		for _, crd := range crds {
+			for _, version := range crd.Spec.Versions {
+				if err := c.syncHandler(schema.GroupVersion{Group: crd.Spec.Group, Version: version.Name}); err != nil {
+					utilruntime.HandleError(err)
+				}
+			}
+		}
+	}
+	close(c.syncedInitialSet)
+
+	// start up your worker threads based on threadiness.  Some controllers have multiple kinds of workers
+	for i := 0; i < threadiness; i++ {
+		// runWorker will loop until "something bad" happens.  The .Until will then rekick the worker
+		// after one second
+		go wait.Until(c.runWorker, time.Second, stopCh)
+	}
+
+	// wait until we're told to stop
+	<-stopCh
+}
+
+// WaitForInitialSync blocks until the initial set of CRD resources has been processed
+func (c *crdRegistrationController) WaitForInitialSync() {
+	<-c.syncedInitialSet
+}
+
+func (c *crdRegistrationController) runWorker() {
+	// hot loop until we're told to stop.  processNextWorkItem will automatically wait until there's work
+	// available, so we don't worry about secondary waits
+	for c.processNextWorkItem() {
+	}
+}
+
+// processNextWorkItem deals with one key off the queue.  It returns false when it's time to quit.
+func (c *crdRegistrationController) processNextWorkItem() bool {
+	// pull the next work item from queue.  It should be a key we use to lookup something in a cache
+	key, quit := c.queue.Get()
+	if quit {
+		return false
+	}
+	// you always have to indicate to the queue that you've completed a piece of work
+	defer c.queue.Done(key)
+
+	// do your work on the key.  This method will contains your "do stuff" logic
+	err := c.syncHandler(key.(schema.GroupVersion))
+	if err == nil {
+		// if you had no error, tell the queue to stop tracking history for your key.  This will
+		// reset things like failure counts for per-item rate limiting
+		c.queue.Forget(key)
+		return true
+	}
+
+	// there was a failure so be sure to report it.  This method allows for pluggable error handling
+	// which can be used for things like cluster-monitoring
+	utilruntime.HandleError(fmt.Errorf("%v failed with : %v", key, err))
+	// since we failed, we should requeue the item to work on later.  This method will add a backoff
+	// to avoid hotlooping on particular items (they're probably still not going to work right away)
+	// and overall controller protection (everything I've done is broken, this controller needs to
+	// calm down or it can starve other useful work) cases.
+	c.queue.AddRateLimited(key)
+
+	return true
+}
+
+func (c *crdRegistrationController) enqueueCRD(crd *apiextensionsv1.CustomResourceDefinition) {
+	for _, version := range crd.Spec.Versions {
+		c.queue.Add(schema.GroupVersion{Group: crd.Spec.Group, Version: version.Name})
+	}
+}
+
+func (c *crdRegistrationController) handleVersionUpdate(groupVersion schema.GroupVersion) error {
+	apiServiceName := groupVersion.Version + "." + groupVersion.Group
+
+	// check all CRDs.  There shouldn't that many, but if we have problems later we can index them
+	crds, err := c.crdLister.List(labels.Everything())
+	if err != nil {
+		return err
+	}
+	for _, crd := range crds {
+		if crd.Spec.Group != groupVersion.Group {
+			continue
+		}
+		for _, version := range crd.Spec.Versions {
+			if version.Name != groupVersion.Version || !version.Served {
+				continue
+			}
+
+			c.apiServiceRegistration.AddAPIServiceToSync(&v1.APIService{
+				ObjectMeta: metav1.ObjectMeta{Name: apiServiceName},
+				Spec: v1.APIServiceSpec{
+					Group:                groupVersion.Group,
+					Version:              groupVersion.Version,
+					GroupPriorityMinimum: 1000, // CRDs should have relatively low priority
+					VersionPriority:      100,  // CRDs will be sorted by kube-like versions like any other APIService with the same VersionPriority
+				},
+			})
+			return nil
+		}
+	}
+
+	c.apiServiceRegistration.RemoveAPIServiceToSync(apiServiceName)
+	return nil
+}

pkg/controlplane/controller/crdregistration/crdregistration_controller_test.go

@@ -0,0 +1,129 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crdregistration
+
+import (
+	"reflect"
+	"testing"
+
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	crdlisters "k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/tools/cache"
+	apiregistration "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
+)
+
+func TestHandleVersionUpdate(t *testing.T) {
+	tests := []struct {
+		name         string
+		startingCRDs []*apiextensionsv1.CustomResourceDefinition
+		version      schema.GroupVersion
+
+		expectedAdded   []*apiregistration.APIService
+		expectedRemoved []string
+	}{
+		{
+			name: "simple add crd",
+			startingCRDs: []*apiextensionsv1.CustomResourceDefinition{
+				{
+					Spec: apiextensionsv1.CustomResourceDefinitionSpec{
+						Group: "group.com",
+						// Version field is deprecated and crd registration won't rely on it at all.
+						// defaulting route will fill up Versions field if user only provided version field.
+						Versions: []apiextensionsv1.CustomResourceDefinitionVersion{
+							{
+								Name:    "v1",
+								Served:  true,
+								Storage: true,
+							},
+						},
+					},
+				},
+			},
+			version: schema.GroupVersion{Group: "group.com", Version: "v1"},
+
+			expectedAdded: []*apiregistration.APIService{
+				{
+					ObjectMeta: metav1.ObjectMeta{Name: "v1.group.com"},
+					Spec: apiregistration.APIServiceSpec{
+						Group:                "group.com",
+						Version:              "v1",
+						GroupPriorityMinimum: 1000,
+						VersionPriority:      100,
+					},
+				},
+			},
+		},
+		{
+			name: "simple remove crd",
+			startingCRDs: []*apiextensionsv1.CustomResourceDefinition{
+				{
+					Spec: apiextensionsv1.CustomResourceDefinitionSpec{
+						Group: "group.com",
+						Versions: []apiextensionsv1.CustomResourceDefinitionVersion{
+							{
+								Name:    "v1",
+								Served:  true,
+								Storage: true,
+							},
+						},
+					},
+				},
+			},
+			version: schema.GroupVersion{Group: "group.com", Version: "v2"},
+
+			expectedRemoved: []string{"v2.group.com"},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			registration := &fakeAPIServiceRegistration{}
+			crdCache := cache.NewIndexer(cache.DeletionHandlingMetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
+			crdLister := crdlisters.NewCustomResourceDefinitionLister(crdCache)
+			c := crdRegistrationController{
+				crdLister:              crdLister,
+				apiServiceRegistration: registration,
+			}
+			for i := range test.startingCRDs {
+				crdCache.Add(test.startingCRDs[i])
+			}
+
+			c.handleVersionUpdate(test.version)
+
+			if !reflect.DeepEqual(test.expectedAdded, registration.added) {
+				t.Errorf("%s expected %v, got %v", test.name, test.expectedAdded, registration.added)
+			}
+			if !reflect.DeepEqual(test.expectedRemoved, registration.removed) {
+				t.Errorf("%s expected %v, got %v", test.name, test.expectedRemoved, registration.removed)
+			}
+		})
+	}
+}
+
+type fakeAPIServiceRegistration struct {
+	added   []*apiregistration.APIService
+	removed []string
+}
+
+func (a *fakeAPIServiceRegistration) AddAPIServiceToSync(in *apiregistration.APIService) {
+	a.added = append(a.added, in)
+}
+func (a *fakeAPIServiceRegistration) RemoveAPIServiceToSync(name string) {
+	a.removed = append(a.removed, name)
+}