dynamic reload cluster authentication info for aggregated API servers

test/integration/examples/BUILD

@@ -15,8 +15,8 @@ go_test(
     ],
     tags = ["integration"],
     deps = [
-        "//cmd/kube-apiserver/app:go_default_library",
         "//cmd/kube-apiserver/app/options:go_default_library",
+        "//cmd/kube-apiserver/app/testing:go_default_library",
         "//pkg/master:go_default_library",
         "//pkg/master/reconcilers:go_default_library",
         "//staging/src/k8s.io/api/admissionregistration/v1beta1:go_default_library",
@@ -27,7 +27,7 @@ go_test(
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/apis/audit:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library",
-        "//staging/src/k8s.io/apiserver/pkg/server:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library",
         "//staging/src/k8s.io/client-go/discovery:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
@@ -35,15 +35,12 @@ go_test(
         "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library",
         "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library",
         "//staging/src/k8s.io/client-go/util/cert:go_default_library",
-        "//staging/src/k8s.io/client-go/util/keyutil:go_default_library",
         "//staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1:go_default_library",
         "//staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset:go_default_library",
-        "//staging/src/k8s.io/kube-aggregator/pkg/cmd/server:go_default_library",
         "//staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1:go_default_library",
         "//staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1beta1:go_default_library",
         "//staging/src/k8s.io/sample-apiserver/pkg/cmd/server:go_default_library",
         "//test/integration/framework:go_default_library",
-        "//test/utils:go_default_library",
         "//vendor/github.com/stretchr/testify/assert:go_default_library",
     ],
 )

test/integration/examples/apiserver_test.go

@@ -17,7 +17,6 @@ limitations under the License.
 package apiserver
 
 import (
-	"crypto/x509"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -25,7 +24,7 @@ import (
 	"net/http"
 	"os"
 	"path"
-	"sync/atomic"
+	"reflect"
 	"testing"
 	"time"
 
@@ -35,293 +34,82 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/wait"
-	genericapiserver "k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
 	genericapiserveroptions "k8s.io/apiserver/pkg/server/options"
-	discovery "k8s.io/client-go/discovery"
+	"k8s.io/client-go/discovery"
 	client "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 	"k8s.io/client-go/util/cert"
-	"k8s.io/client-go/util/keyutil"
 	apiregistrationv1beta1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1"
 	aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
-	kubeaggregatorserver "k8s.io/kube-aggregator/pkg/cmd/server"
-	"k8s.io/kubernetes/cmd/kube-apiserver/app"
-	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
+	kastesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
 	"k8s.io/kubernetes/test/integration/framework"
-	testutil "k8s.io/kubernetes/test/utils"
 	wardlev1alpha1 "k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1"
 	wardlev1beta1 "k8s.io/sample-apiserver/pkg/apis/wardle/v1beta1"
 	sampleserver "k8s.io/sample-apiserver/pkg/cmd/server"
 )
 
 func TestAggregatedAPIServer(t *testing.T) {
+	// makes the kube-apiserver very responsive.  it's normally a minute
+	dynamiccertificates.FileRefreshDuration = 1 * time.Second
+
 	stopCh := make(chan struct{})
 	defer close(stopCh)
 
-	certDir, _ := ioutil.TempDir("", "test-integration-apiserver")
-	defer os.RemoveAll(certDir)
-	_, defaultServiceClusterIPRange, _ := net.ParseCIDR("10.0.0.0/24")
-	proxySigningKey, err := testutil.NewPrivateKey()
-	if err != nil {
-		t.Fatal(err)
-	}
-	proxySigningCert, err := cert.NewSelfSignedCACert(cert.Config{CommonName: "front-proxy-ca"}, proxySigningKey)
-	if err != nil {
-		t.Fatal(err)
-	}
-	proxyCACertFile, _ := ioutil.TempFile(certDir, "proxy-ca.crt")
-	if err := ioutil.WriteFile(proxyCACertFile.Name(), testutil.EncodeCertPEM(proxySigningCert), 0644); err != nil {
-		t.Fatal(err)
-	}
-	clientSigningKey, err := testutil.NewPrivateKey()
-	if err != nil {
-		t.Fatal(err)
-	}
-	clientSigningCert, err := cert.NewSelfSignedCACert(cert.Config{CommonName: "client-ca"}, clientSigningKey)
-	if err != nil {
-		t.Fatal(err)
-	}
-	clientCACertFile, _ := ioutil.TempFile(certDir, "client-ca.crt")
-	if err := ioutil.WriteFile(clientCACertFile.Name(), testutil.EncodeCertPEM(clientSigningCert), 0644); err != nil {
-		t.Fatal(err)
-	}
-
-	kubeClientConfigValue := atomic.Value{}
-	go func() {
-		listener, _, err := genericapiserveroptions.CreateListener("tcp", "127.0.0.1:0")
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		kubeAPIServerOptions := options.NewServerRunOptions()
-		kubeAPIServerOptions.SecureServing.Listener = listener
-		kubeAPIServerOptions.SecureServing.BindAddress = net.ParseIP("127.0.0.1")
-		kubeAPIServerOptions.SecureServing.ServerCert.CertDirectory = certDir
-		kubeAPIServerOptions.InsecureServing.BindPort = 0
-		kubeAPIServerOptions.Etcd.StorageConfig.Transport.ServerList = []string{framework.GetEtcdURL()}
-		kubeAPIServerOptions.ServiceClusterIPRanges = defaultServiceClusterIPRange.String()
-		kubeAPIServerOptions.Authentication.RequestHeader.UsernameHeaders = []string{"X-Remote-User"}
-		kubeAPIServerOptions.Authentication.RequestHeader.GroupHeaders = []string{"X-Remote-Group"}
-		kubeAPIServerOptions.Authentication.RequestHeader.ExtraHeaderPrefixes = []string{"X-Remote-Extra-"}
-		kubeAPIServerOptions.Authentication.RequestHeader.AllowedNames = []string{"kube-aggregator"}
-		kubeAPIServerOptions.Authentication.RequestHeader.ClientCAFile = proxyCACertFile.Name()
-		kubeAPIServerOptions.Authentication.ClientCert.ClientCA = clientCACertFile.Name()
-		kubeAPIServerOptions.Authorization.Modes = []string{"RBAC"}
-		completedOptions, err := app.Complete(kubeAPIServerOptions)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		tunneler, proxyTransport, err := app.CreateNodeDialer(completedOptions)
-		if err != nil {
-			t.Fatal(err)
-		}
-		kubeAPIServerConfig, _, _, _, err := app.CreateKubeAPIServerConfig(completedOptions, tunneler, proxyTransport)
-		if err != nil {
-			t.Fatal(err)
-		}
-		// Adjust the loopback config for external use (external server name and CA)
-		kubeAPIServerClientConfig := rest.CopyConfig(kubeAPIServerConfig.GenericConfig.LoopbackClientConfig)
-		kubeAPIServerClientConfig.CAFile = path.Join(certDir, "apiserver.crt")
-		kubeAPIServerClientConfig.CAData = nil
-		kubeAPIServerClientConfig.ServerName = ""
-		kubeClientConfigValue.Store(kubeAPIServerClientConfig)
-
-		kubeAPIServer, err := app.CreateKubeAPIServer(kubeAPIServerConfig, genericapiserver.NewEmptyDelegate())
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		if err := kubeAPIServer.GenericAPIServer.PrepareRun().Run(wait.NeverStop); err != nil {
-			t.Fatal(err)
-		}
-	}()
-
-	// just use json because everyone speaks it
-	err = wait.PollImmediate(time.Second, time.Minute, func() (done bool, err error) {
-		obj := kubeClientConfigValue.Load()
-		if obj == nil {
-			return false, nil
-		}
-		kubeClientConfig := kubeClientConfigValue.Load().(*rest.Config)
-		kubeClientConfig.ContentType = ""
-		kubeClientConfig.AcceptContentTypes = ""
-		kubeClient, err := client.NewForConfig(kubeClientConfig)
-		if err != nil {
-			// this happens because we race the API server start
-			t.Log(err)
-			return false, nil
-		}
-
-		healthStatus := 0
-		kubeClient.Discovery().RESTClient().Get().AbsPath("/healthz").Do().StatusCode(&healthStatus)
-		if healthStatus != http.StatusOK {
-			return false, nil
-		}
-
-		return true, nil
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// after this point we won't be mutating, so the race detector will be fine
-	kubeClientConfig := kubeClientConfigValue.Load().(*rest.Config)
-
-	// write a kubeconfig out for starting other API servers with delegated auth.  remember, no in-cluster config
-	adminKubeConfig := createKubeConfig(kubeClientConfig)
-	kubeconfigFile, _ := ioutil.TempFile("", "")
-	defer os.Remove(kubeconfigFile.Name())
-	clientcmd.WriteToFile(*adminKubeConfig, kubeconfigFile.Name())
-	wardleCertDir, _ := ioutil.TempDir("", "test-integration-wardle-server")
-	defer os.RemoveAll(wardleCertDir)
-	wardlePort := new(int32)
+	testServer := kastesting.StartTestServerOrDie(t, nil, nil, framework.SharedEtcd())
+	defer testServer.TearDownFn()
+	kubeClientConfig := rest.CopyConfig(testServer.ClientConfig)
+	// force json because everything speaks it
+	kubeClientConfig.ContentType = ""
+	kubeClientConfig.AcceptContentTypes = ""
+	kubeClient := client.NewForConfigOrDie(kubeClientConfig)
+	aggregatorClient := aggregatorclient.NewForConfigOrDie(kubeClientConfig)
 
 	// start the wardle server to prove we can aggregate it
+	wardleToKASKubeConfigFile := writeKubeConfigForWardleServerToKASConnection(t, rest.CopyConfig(kubeClientConfig))
+	defer os.Remove(wardleToKASKubeConfigFile)
+	wardleCertDir, _ := ioutil.TempDir("", "test-integration-wardle-server")
+	defer os.RemoveAll(wardleCertDir)
+	listener, wardlePort, err := genericapiserveroptions.CreateListener("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
 	go func() {
-		listener, port, err := genericapiserveroptions.CreateListener("tcp", "127.0.0.1:0")
-		if err != nil {
-			t.Fatal(err)
-		}
-		atomic.StoreInt32(wardlePort, int32(port))
-
 		o := sampleserver.NewWardleServerOptions(os.Stdout, os.Stderr)
 		o.RecommendedOptions.SecureServing.Listener = listener
 		o.RecommendedOptions.SecureServing.BindAddress = net.ParseIP("127.0.0.1")
 		wardleCmd := sampleserver.NewCommandStartWardleServer(o, stopCh)
 		wardleCmd.SetArgs([]string{
-			"--requestheader-username-headers=X-Remote-User",
-			"--requestheader-group-headers=X-Remote-Group",
-			"--requestheader-extra-headers-prefix=X-Remote-Extra-",
-			"--requestheader-client-ca-file=" + proxyCACertFile.Name(),
-			"--requestheader-allowed-names=kube-aggregator",
-			"--authentication-kubeconfig", kubeconfigFile.Name(),
-			"--authorization-kubeconfig", kubeconfigFile.Name(),
+			"--authentication-kubeconfig", wardleToKASKubeConfigFile,
+			"--authorization-kubeconfig", wardleToKASKubeConfigFile,
 			"--etcd-servers", framework.GetEtcdURL(),
 			"--cert-dir", wardleCertDir,
-			"--kubeconfig", kubeconfigFile.Name(),
+			"--kubeconfig", wardleToKASKubeConfigFile,
 		})
 		if err := wardleCmd.Execute(); err != nil {
 			t.Fatal(err)
 		}
 	}()
-
-	wardleClientConfig := rest.AnonymousClientConfig(kubeClientConfig)
-	wardleClientConfig.CAFile = path.Join(wardleCertDir, "apiserver.crt")
-	wardleClientConfig.CAData = nil
-	wardleClientConfig.ServerName = ""
-	wardleClientConfig.BearerToken = kubeClientConfig.BearerToken
-	var wardleClient client.Interface
-	err = wait.PollImmediate(100*time.Millisecond, 10*time.Second, func() (done bool, err error) {
-		wardleClientConfig.Host = fmt.Sprintf("https://127.0.0.1:%d", atomic.LoadInt32(wardlePort))
-		wardleClient, err = client.NewForConfig(wardleClientConfig)
-		if err != nil {
-			// this happens because we race the API server start
-			t.Log(err)
-			return false, nil
-		}
-		healthStatus := 0
-		wardleClient.Discovery().RESTClient().Get().AbsPath("/healthz").Do().StatusCode(&healthStatus)
-		if healthStatus != http.StatusOK {
-			return false, nil
-		}
-		return true, nil
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// start the aggregator
-	aggregatorCertDir, _ := ioutil.TempDir("", "test-integration-aggregator")
-	defer os.RemoveAll(aggregatorCertDir)
-	proxyClientKey, err := testutil.NewPrivateKey()
-	if err != nil {
-		t.Fatal(err)
-	}
-	proxyClientCert, err := testutil.NewSignedCert(
-		&cert.Config{
-			CommonName: "kube-aggregator",
-			Usages:     []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
-		},
-		proxyClientKey, proxySigningCert, proxySigningKey,
-	)
-	proxyClientCertFile, _ := ioutil.TempFile(aggregatorCertDir, "proxy-client.crt")
-	proxyClientKeyFile, _ := ioutil.TempFile(aggregatorCertDir, "proxy-client.key")
-	if err := ioutil.WriteFile(proxyClientCertFile.Name(), testutil.EncodeCertPEM(proxyClientCert), 0600); err != nil {
-		t.Fatal(err)
-	}
-	proxyClientKeyPEM, err := keyutil.MarshalPrivateKeyToPEM(proxyClientKey)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := ioutil.WriteFile(proxyClientKeyFile.Name(), proxyClientKeyPEM, 0644); err != nil {
-		t.Fatal(err)
-	}
-	aggregatorPort := new(int32)
-
-	go func() {
-		listener, port, err := genericapiserveroptions.CreateListener("tcp", "127.0.0.1:0")
-		if err != nil {
-			t.Fatal(err)
-		}
-		atomic.StoreInt32(aggregatorPort, int32(port))
-
-		o := kubeaggregatorserver.NewDefaultOptions(os.Stdout, os.Stderr)
-		o.RecommendedOptions.SecureServing.Listener = listener
-		o.RecommendedOptions.SecureServing.BindAddress = net.ParseIP("127.0.0.1")
-		aggregatorCmd := kubeaggregatorserver.NewCommandStartAggregator(o, stopCh)
-		aggregatorCmd.SetArgs([]string{
-			"--requestheader-username-headers", "",
-			"--proxy-client-cert-file", proxyClientCertFile.Name(),
-			"--proxy-client-key-file", proxyClientKeyFile.Name(),
-			"--kubeconfig", kubeconfigFile.Name(),
-			"--authentication-kubeconfig", kubeconfigFile.Name(),
-			"--authorization-kubeconfig", kubeconfigFile.Name(),
-			"--etcd-servers", framework.GetEtcdURL(),
-			"--cert-dir", aggregatorCertDir,
-		})
-
-		if err := aggregatorCmd.Execute(); err != nil {
-			t.Fatal(err)
-		}
-	}()
-
-	aggregatorClientConfig := rest.AnonymousClientConfig(kubeClientConfig)
-	aggregatorClientConfig.CAFile = path.Join(aggregatorCertDir, "apiserver.crt")
-	aggregatorClientConfig.CAData = nil
-	aggregatorClientConfig.ServerName = ""
-	aggregatorClientConfig.BearerToken = kubeClientConfig.BearerToken
-	var aggregatorDiscoveryClient client.Interface
-	err = wait.PollImmediate(100*time.Millisecond, 10*time.Second, func() (done bool, err error) {
-		aggregatorClientConfig.Host = fmt.Sprintf("https://127.0.0.1:%d", atomic.LoadInt32(aggregatorPort))
-		aggregatorDiscoveryClient, err = client.NewForConfig(aggregatorClientConfig)
-		if err != nil {
-			// this happens if we race the API server for writing the cert
-			return false, nil
-		}
-		healthStatus := 0
-		aggregatorDiscoveryClient.Discovery().RESTClient().Get().AbsPath("/healthz").Do().StatusCode(&healthStatus)
-		if healthStatus != http.StatusOK {
-			return false, nil
-		}
-		return true, nil
-	})
+	directWardleClientConfig, err := waitForWardleRunning(t, kubeClientConfig, wardleCertDir, wardlePort)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	// now we're finally ready to test. These are what's run by default now
+	wardleClient, err := client.NewForConfig(directWardleClientConfig)
+	if err != nil {
+		t.Fatal(err)
+	}
 	testAPIGroupList(t, wardleClient.Discovery().RESTClient())
 	testAPIGroup(t, wardleClient.Discovery().RESTClient())
 	testAPIResourceList(t, wardleClient.Discovery().RESTClient())
 
-	wardleCA, err := ioutil.ReadFile(wardleClientConfig.CAFile)
+	wardleCA, err := ioutil.ReadFile(directWardleClientConfig.CAFile)
 	if err != nil {
 		t.Fatal(err)
 	}
-	aggregatorClient := aggregatorclient.NewForConfigOrDie(aggregatorClientConfig)
 	_, err = aggregatorClient.ApiregistrationV1beta1().APIServices().Create(&apiregistrationv1beta1.APIService{
 		ObjectMeta: metav1.ObjectMeta{Name: "v1alpha1.wardle.example.com"},
 		Spec: apiregistrationv1beta1.APIServiceSpec{
@@ -342,39 +130,154 @@ func TestAggregatedAPIServer(t *testing.T) {
 
 	// wait for the unavailable API service to be processed with updated status
 	err = wait.Poll(100*time.Millisecond, 5*time.Second, func() (done bool, err error) {
-		_, err = aggregatorDiscoveryClient.Discovery().ServerResources()
+		_, err = kubeClient.Discovery().ServerResources()
 		hasExpectedError := checkWardleUnavailableDiscoveryError(t, err)
 		return hasExpectedError, nil
 	})
 	if err != nil {
 		t.Fatal(err)
 	}
+	// TODO figure out how to turn on enough of services and dns to run more
 
-	_, err = aggregatorClient.ApiregistrationV1beta1().APIServices().Create(&apiregistrationv1beta1.APIService{
-		ObjectMeta: metav1.ObjectMeta{Name: "v1."},
-		Spec: apiregistrationv1beta1.APIServiceSpec{
-			// register this as a local service so it doesn't try to lookup the default kubernetes service
-			// which will have an unroutable IP address since it's fake.
-			Group:                "",
-			Version:              "v1",
-			GroupPriorityMinimum: 100,
-			VersionPriority:      100,
-		},
-	})
+	// Now we want to verify that the client CA bundles properly reflect the values for the cluster-authentication
+	firstKubeCANames, err := cert.GetClientCANamesForURL(kubeClientConfig.Host)
 	if err != nil {
 		t.Fatal(err)
 	}
-
-	// this is ugly, but sleep just a little bit so that the watch is probably observed.  Since nothing will actually be added to discovery
-	// (the service is missing), we don't have an external signal.
-	time.Sleep(100 * time.Millisecond)
-	_, err = aggregatorDiscoveryClient.Discovery().ServerResources()
-	hasExpectedError := checkWardleUnavailableDiscoveryError(t, err)
-	if !hasExpectedError {
-		t.Fatalf("Discovery call didn't return expected error: %v", err)
+	t.Log(firstKubeCANames)
+	firstWardleCANames, err := cert.GetClientCANamesForURL(directWardleClientConfig.Host)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log(firstWardleCANames)
+	if !reflect.DeepEqual(firstKubeCANames, firstWardleCANames) {
+		t.Fatal("names don't match")
 	}
 
-	// TODO figure out how to turn on enough of services and dns to run more
+	// now we update the client-ca nd request-header-client-ca-file and the kas will consume it, update the configmap
+	// and then the wardle server will detect and update too.
+	if err := ioutil.WriteFile(path.Join(testServer.TmpDir, "client-ca.crt"), differentClientCA, 0644); err != nil {
+		t.Fatal(err)
+	}
+	if err := ioutil.WriteFile(path.Join(testServer.TmpDir, "proxy-ca.crt"), differentFrontProxyCA, 0644); err != nil {
+		t.Fatal(err)
+	}
+	// wait for it to be picked up.  there's a test in certreload_test.go that ensure this works
+	time.Sleep(4 * time.Second)
+
+	// Now we want to verify that the client CA bundles properly updated to reflect the new values written for the kube-apiserver
+	secondKubeCANames, err := cert.GetClientCANamesForURL(kubeClientConfig.Host)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log(secondKubeCANames)
+	for i := range firstKubeCANames {
+		if firstKubeCANames[i] == secondKubeCANames[i] {
+			t.Errorf("ca bundles should change")
+		}
+	}
+	secondWardleCANames, err := cert.GetClientCANamesForURL(directWardleClientConfig.Host)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log(secondWardleCANames)
+
+	// second wardle should contain all the certs, first and last
+	numMatches := 0
+	for _, needle := range firstKubeCANames {
+		for _, haystack := range secondWardleCANames {
+			if needle == haystack {
+				numMatches++
+				break
+			}
+		}
+	}
+	for _, needle := range secondKubeCANames {
+		for _, haystack := range secondWardleCANames {
+			if needle == haystack {
+				numMatches++
+				break
+			}
+		}
+	}
+	if numMatches != 4 {
+		t.Fatal("names don't match")
+	}
+
+}
+
+func waitForWardleRunning(t *testing.T, wardleToKASKubeConfig *rest.Config, wardleCertDir string, wardlePort int) (*rest.Config, error) {
+	directWardleClientConfig := rest.AnonymousClientConfig(rest.CopyConfig(wardleToKASKubeConfig))
+	directWardleClientConfig.CAFile = path.Join(wardleCertDir, "apiserver.crt")
+	directWardleClientConfig.CAData = nil
+	directWardleClientConfig.ServerName = ""
+	directWardleClientConfig.BearerToken = wardleToKASKubeConfig.BearerToken
+	var wardleClient client.Interface
+	lastHealthContent := []byte{}
+	var lastHealthErr error
+	err := wait.PollImmediate(100*time.Millisecond, 10*time.Second, func() (done bool, err error) {
+		if _, err := os.Stat(directWardleClientConfig.CAFile); os.IsNotExist(err) { // wait until the file trust is created
+			lastHealthErr = err
+			return false, nil
+		}
+		directWardleClientConfig.Host = fmt.Sprintf("https://127.0.0.1:%d", wardlePort)
+		wardleClient, err = client.NewForConfig(directWardleClientConfig)
+		if err != nil {
+			// this happens because we race the API server start
+			t.Log(err)
+			return false, nil
+		}
+		healthStatus := 0
+		result := wardleClient.Discovery().RESTClient().Get().AbsPath("/healthz").Do().StatusCode(&healthStatus)
+		lastHealthContent, lastHealthErr = result.Raw()
+		if healthStatus != http.StatusOK {
+			return false, nil
+		}
+		return true, nil
+	})
+	if err != nil {
+		t.Log(string(lastHealthContent))
+		t.Log(lastHealthErr)
+		return nil, err
+	}
+
+	return directWardleClientConfig, nil
+}
+
+func writeKubeConfigForWardleServerToKASConnection(t *testing.T, kubeClientConfig *rest.Config) string {
+	// write a kubeconfig out for starting other API servers with delegated auth.  remember, no in-cluster config
+	// the loopback client config uses a loopback cert with different SNI.  We need to use the "real"
+	// cert, so we'll hope we aren't hacked during a unit test and instead load it from the server we started.
+	wardleToKASKubeClientConfig := rest.CopyConfig(kubeClientConfig)
+
+	servingCerts, _, err := cert.GetServingCertificatesForURL(wardleToKASKubeClientConfig.Host, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	encodedServing, err := cert.EncodeCertificates(servingCerts...)
+	if err != nil {
+		t.Fatal(err)
+	}
+	wardleToKASKubeClientConfig.CAData = encodedServing
+
+	for _, v := range servingCerts {
+		t.Logf("Client: Server public key is %v\n", dynamiccertificates.GetHumanCertDetail(v))
+	}
+	certs, err := cert.ParseCertsPEM(wardleToKASKubeClientConfig.CAData)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, curr := range certs {
+		t.Logf("CA bundle %v\n", dynamiccertificates.GetHumanCertDetail(curr))
+	}
+
+	adminKubeConfig := createKubeConfig(wardleToKASKubeClientConfig)
+	wardleToKASKubeConfigFile, _ := ioutil.TempFile("", "")
+	if err := clientcmd.WriteToFile(*adminKubeConfig, wardleToKASKubeConfigFile.Name()); err != nil {
+		t.Fatal(err)
+	}
+
+	return wardleToKASKubeConfigFile.Name()
 }
 
 func checkWardleUnavailableDiscoveryError(t *testing.T, err error) bool {
@@ -510,3 +413,41 @@ func testAPIResourceList(t *testing.T, client rest.Interface) {
 	assert.Equal(t, "flunders", apiResourceList.APIResources[1].Name)
 	assert.True(t, apiResourceList.APIResources[1].Namespaced)
 }
+
+var (
+	// I have no idea what these certs are, they just need to be different
+	differentClientCA = []byte(`-----BEGIN CERTIFICATE-----
+MIIDQDCCAiigAwIBAgIJANWw74P5KJk2MA0GCSqGSIb3DQEBCwUAMDQxMjAwBgNV
+BAMMKWdlbmVyaWNfd2ViaG9va19hZG1pc3Npb25fcGx1Z2luX3Rlc3RzX2NhMCAX
+DTE3MTExNjAwMDUzOVoYDzIyOTEwOTAxMDAwNTM5WjAjMSEwHwYDVQQDExh3ZWJo
+b29rLXRlc3QuZGVmYXVsdC5zdmMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
+AoIBAQDXd/nQ89a5H8ifEsigmMd01Ib6NVR3bkJjtkvYnTbdfYEBj7UzqOQtHoLa
+dIVmefny5uIHvj93WD8WDVPB3jX2JHrXkDTXd/6o6jIXHcsUfFTVLp6/bZ+Anqe0
+r/7hAPkzA2A7APyTWM3ZbEeo1afXogXhOJ1u/wz0DflgcB21gNho4kKTONXO3NHD
+XLpspFqSkxfEfKVDJaYAoMnYZJtFNsa2OvsmLnhYF8bjeT3i07lfwrhUZvP+7Gsp
+7UgUwc06WuNHjfx1s5e6ySzH0QioMD1rjYneqOvk0pKrMIhuAEWXqq7jlXcDtx1E
+j+wnYbVqqVYheHZ8BCJoVAAQGs9/AgMBAAGjZDBiMAkGA1UdEwQCMAAwCwYDVR0P
+BAQDAgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATApBgNVHREEIjAg
+hwR/AAABghh3ZWJob29rLXRlc3QuZGVmYXVsdC5zdmMwDQYJKoZIhvcNAQELBQAD
+ggEBAD/GKSPNyQuAOw/jsYZesb+RMedbkzs18sSwlxAJQMUrrXwlVdHrA8q5WhE6
+ABLqU1b8lQ8AWun07R8k5tqTmNvCARrAPRUqls/ryER+3Y9YEcxEaTc3jKNZFLbc
+T6YtcnkdhxsiO136wtiuatpYL91RgCmuSpR8+7jEHhuFU01iaASu7ypFrUzrKHTF
+bKwiLRQi1cMzVcLErq5CDEKiKhUkoDucyARFszrGt9vNIl/YCcBOkcNvM3c05Hn3
+M++C29JwS3Hwbubg6WO3wjFjoEhpCwU6qRYUz3MRp4tHO4kxKXx+oQnUiFnR7vW0
+YkNtGc1RUDHwecCTFpJtPb7Yu/E=
+-----END CERTIFICATE-----
+`)
+	differentFrontProxyCA = []byte(`-----BEGIN CERTIFICATE-----
+MIIBqDCCAU2gAwIBAgIUfbqeieihh/oERbfvRm38XvS/xHAwCgYIKoZIzj0EAwIw
+GjEYMBYGA1UEAxMPSW50ZXJtZWRpYXRlLUNBMCAXDTE2MTAxMTA1MDYwMFoYDzIx
+MTYwOTE3MDUwNjAwWjAUMRIwEAYDVQQDEwlNeSBDbGllbnQwWTATBgcqhkjOPQIB
+BggqhkjOPQMBBwNCAARv6N4R/sjMR65iMFGNLN1GC/vd7WhDW6J4X/iAjkRLLnNb
+KbRG/AtOUZ+7upJ3BWIRKYbOabbQGQe2BbKFiap4o3UwczAOBgNVHQ8BAf8EBAMC
+BaAwEwYDVR0lBAwwCgYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU
+K/pZOWpNcYai6eHFpmJEeFpeQlEwHwYDVR0jBBgwFoAUX6nQlxjfWnP6aM1meO/Q
+a6b3a9kwCgYIKoZIzj0EAwIDSQAwRgIhAIWTKw/sjJITqeuNzJDAKU4xo1zL+xJ5
+MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps=
+-----END CERTIFICATE-----
+
+`)
+)