From 5a15ae03f2f5411b68c47b5b8914f7f13eae4e6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanislav=20L=C3=A1zni=C4=8Dka?= Date: Mon, 20 Feb 2023 15:13:02 +0100 Subject: [PATCH] test:integration: split Wardle test server run Split running the Wardle aggregated API into preparation and running phase. This allows reusing the prepared options and makes it possible for us to introduce additional hooks into the server authorization flow. --- test/integration/examples/apiserver_test.go | 126 +++++++++++++------- 1 file changed, 85 insertions(+), 41 deletions(-) diff --git a/test/integration/examples/apiserver_test.go b/test/integration/examples/apiserver_test.go index a658a5378b8..6646556a788 100644 --- a/test/integration/examples/apiserver_test.go +++ b/test/integration/examples/apiserver_test.go @@ -237,10 +237,7 @@ func TestAggregatedAPIServer(t *testing.T) { }) } -func testAggregatedAPIServer(t *testing.T, enableWardleFeatureGate bool, emulationVersion string) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) - t.Cleanup(cancel) - +func prepareAggregatedWardleAPIServer(ctx context.Context, t *testing.T, namespace string) (*kastesting.TestServer, *sampleserver.WardleServerOptions, int) { // makes the kube-apiserver very responsive. it's normally a minute dynamiccertificates.FileRefreshDuration = 1 * time.Second @@ -253,25 +250,20 @@ func testAggregatedAPIServer(t *testing.T, enableWardleFeatureGate bool, emulati t.Cleanup(app.SetServiceResolverForTests(staticURLServiceResolver(fmt.Sprintf("https://127.0.0.1:%d", wardlePort)))) testServer := kastesting.StartTestServerOrDie(t, &kastesting.TestServerInstanceOptions{EnableCertAuth: true, BinaryVersion: "1.32"}, nil, framework.SharedEtcd()) - defer testServer.TearDownFn() - kubeClientConfig := rest.CopyConfig(testServer.ClientConfig) - // force json because everything speaks it - kubeClientConfig.ContentType = "" - kubeClientConfig.AcceptContentTypes = "" - kubeClient := client.NewForConfigOrDie(kubeClientConfig) - aggregatorClient := aggregatorclient.NewForConfigOrDie(kubeClientConfig) - wardleClient := wardlev1alpha1client.NewForConfigOrDie(kubeClientConfig) + t.Cleanup(func() { testServer.TearDownFn() }) + + kubeClient := client.NewForConfigOrDie(getKubeConfig(testServer)) // create the bare minimum resources required to be able to get the API service into an available state _, err = kubeClient.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: "kube-wardle", + Name: namespace, }, }, metav1.CreateOptions{}) if err != nil { t.Fatal(err) } - _, err = kubeClient.CoreV1().Services("kube-wardle").Create(ctx, &corev1.Service{ + _, err = kubeClient.CoreV1().Services(namespace).Create(ctx, &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "api", }, @@ -284,48 +276,64 @@ func testAggregatedAPIServer(t *testing.T, enableWardleFeatureGate bool, emulati t.Fatal(err) } + wardleOptions := sampleserver.NewWardleServerOptions(os.Stdout, os.Stderr) + // ensure this is a SAN on the generated cert for service FQDN + wardleOptions.AlternateDNS = []string{ + fmt.Sprintf("api.%s.svc", namespace), + } + wardleOptions.RecommendedOptions.SecureServing.Listener = listener + wardleOptions.RecommendedOptions.SecureServing.BindAddress = netutils.ParseIPSloppy("127.0.0.1") + + return testServer, wardleOptions, wardlePort +} + +func runPreparedWardleServer( + ctx context.Context, + t *testing.T, + wardleOptions *sampleserver.WardleServerOptions, + certDir string, + wardlePort int, + flunderBanningFeatureGate bool, + emulationVersion string, + kubeConfig *rest.Config, +) *rest.Config { + // start the wardle server to prove we can aggregate it - wardleToKASKubeConfigFile := writeKubeConfigForWardleServerToKASConnection(t, rest.CopyConfig(kubeClientConfig)) - defer os.Remove(wardleToKASKubeConfigFile) - wardleCertDir, _ := os.MkdirTemp("", "test-integration-wardle-server") - defer os.RemoveAll(wardleCertDir) + wardleToKASKubeConfigFile := writeKubeConfigForWardleServerToKASConnection(t, rest.CopyConfig(kubeConfig)) + t.Cleanup(func() { os.Remove(wardleToKASKubeConfigFile) }) + go func() { - o := sampleserver.NewWardleServerOptions(os.Stdout, os.Stderr) - // ensure this is a SAN on the generated cert for service FQDN - o.AlternateDNS = []string{ - "api.kube-wardle.svc", - } - o.RecommendedOptions.SecureServing.Listener = listener - o.RecommendedOptions.SecureServing.BindAddress = netutils.ParseIPSloppy("127.0.0.1") - wardleCmd := sampleserver.NewCommandStartWardleServer(ctx, o) args := []string{ "--authentication-kubeconfig", wardleToKASKubeConfigFile, "--authorization-kubeconfig", wardleToKASKubeConfigFile, "--etcd-servers", framework.GetEtcdURL(), - "--cert-dir", wardleCertDir, + "--cert-dir", certDir, "--kubeconfig", wardleToKASKubeConfigFile, "--emulated-version", fmt.Sprintf("wardle=%s", emulationVersion), } - if enableWardleFeatureGate { + if flunderBanningFeatureGate { args = append(args, "--feature-gates", "wardle:BanFlunder=true") } + wardleCmd := sampleserver.NewCommandStartWardleServer(ctx, wardleOptions) wardleCmd.SetArgs(args) if err := wardleCmd.Execute(); err != nil { t.Error(err) } }() - directWardleClientConfig, err := waitForWardleRunning(ctx, t, kubeClientConfig, wardleCertDir, wardlePort) + + directWardleClientConfig, err := waitForWardleRunning(ctx, t, kubeConfig, certDir, wardlePort) if err != nil { t.Fatal(err) } - // now we're finally ready to test. These are what's run by default now - wardleDirectClient := client.NewForConfigOrDie(directWardleClientConfig) - testAPIGroupList(ctx, t, wardleDirectClient.Discovery().RESTClient()) - testAPIGroup(ctx, t, wardleDirectClient.Discovery().RESTClient()) - testAPIResourceList(ctx, t, wardleDirectClient.Discovery().RESTClient()) + return directWardleClientConfig +} - wardleCA, err := os.ReadFile(directWardleClientConfig.CAFile) +func waitForWardleAPIServiceReady(ctx context.Context, t *testing.T, kubeConfig *rest.Config, wardleCertDir string, namespace string) { + kubeClient := client.NewForConfigOrDie(kubeConfig) + aggregatorClient := aggregatorclient.NewForConfigOrDie(kubeConfig) + + wardleCA, err := os.ReadFile(wardleCAFilePath(wardleCertDir)) if err != nil { t.Fatal(err) } @@ -333,7 +341,7 @@ func testAggregatedAPIServer(t *testing.T, enableWardleFeatureGate bool, emulati ObjectMeta: metav1.ObjectMeta{Name: "v1alpha1.wardle.example.com"}, Spec: apiregistrationv1.APIServiceSpec{ Service: &apiregistrationv1.ServiceReference{ - Namespace: "kube-wardle", + Namespace: namespace, Name: "api", }, Group: "wardle.example.com", @@ -397,9 +405,34 @@ func testAggregatedAPIServer(t *testing.T, enableWardleFeatureGate bool, emulati if err != nil { t.Fatal(err) } +} + +func testAggregatedAPIServer(t *testing.T, flunderBanningFeatureGate bool, emulationVersion string) { + const testNamespace = "kube-wardle" + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + t.Cleanup(cancel) + + testKAS, wardleOptions, wardlePort := prepareAggregatedWardleAPIServer(ctx, t, testNamespace) + kubeClientConfig := getKubeConfig(testKAS) + + wardleCertDir, _ := os.MkdirTemp("", "test-integration-wardle-server") + defer os.RemoveAll(wardleCertDir) + + directWardleClientConfig := runPreparedWardleServer(ctx, t, wardleOptions, wardleCertDir, wardlePort, flunderBanningFeatureGate, emulationVersion, kubeClientConfig) + + // now we're finally ready to test. These are what's run by default now + wardleDirectClient := client.NewForConfigOrDie(directWardleClientConfig) + testAPIGroupList(ctx, t, wardleDirectClient.Discovery().RESTClient()) + testAPIGroup(ctx, t, wardleDirectClient.Discovery().RESTClient()) + testAPIResourceList(ctx, t, wardleDirectClient.Discovery().RESTClient()) + + wardleClient := wardlev1alpha1client.NewForConfigOrDie(kubeClientConfig) + + waitForWardleAPIServiceReady(ctx, t, kubeClientConfig, wardleCertDir, testNamespace) // perform simple CRUD operations against the wardle resources - _, err = wardleClient.Fischers().Create(ctx, &wardlev1alpha1.Fischer{ + _, err := wardleClient.Fischers().Create(ctx, &wardlev1alpha1.Fischer{ ObjectMeta: metav1.ObjectMeta{ Name: "panda", }, @@ -426,7 +459,7 @@ func testAggregatedAPIServer(t *testing.T, enableWardleFeatureGate bool, emulati Name: "badname", }, }, metav1.CreateOptions{}) - banFlunder := enableWardleFeatureGate || emulationVersion == "1.2" + banFlunder := flunderBanningFeatureGate || emulationVersion == "1.2" if banFlunder && err == nil { t.Fatal("expect flunder:badname not admitted when wardle feature gates are specified") } @@ -498,10 +531,10 @@ func testAggregatedAPIServer(t *testing.T, enableWardleFeatureGate bool, emulati // now we update the client-ca nd request-header-client-ca-file and the kas will consume it, update the configmap // and then the wardle server will detect and update too. - if err := os.WriteFile(path.Join(testServer.TmpDir, "client-ca.crt"), differentClientCA, 0644); err != nil { + if err := os.WriteFile(path.Join(testKAS.TmpDir, "client-ca.crt"), differentClientCA, 0644); err != nil { t.Fatal(err) } - if err := os.WriteFile(path.Join(testServer.TmpDir, "proxy-ca.crt"), differentFrontProxyCA, 0644); err != nil { + if err := os.WriteFile(path.Join(testKAS.TmpDir, "proxy-ca.crt"), differentFrontProxyCA, 0644); err != nil { t.Fatal(err) } // wait for it to be picked up. there's a test in certreload_test.go that ensure this works @@ -547,9 +580,18 @@ func testAggregatedAPIServer(t *testing.T, enableWardleFeatureGate bool, emulati } } +func getKubeConfig(testServer *kastesting.TestServer) *rest.Config { + kubeClientConfig := rest.CopyConfig(testServer.ClientConfig) + // force json because everything speaks it + kubeClientConfig.ContentType = "" + kubeClientConfig.AcceptContentTypes = "" + + return kubeClientConfig +} + func waitForWardleRunning(ctx context.Context, t *testing.T, wardleToKASKubeConfig *rest.Config, wardleCertDir string, wardlePort int) (*rest.Config, error) { directWardleClientConfig := rest.AnonymousClientConfig(rest.CopyConfig(wardleToKASKubeConfig)) - directWardleClientConfig.CAFile = path.Join(wardleCertDir, "apiserver.crt") + directWardleClientConfig.CAFile = wardleCAFilePath(wardleCertDir) directWardleClientConfig.CAData = nil directWardleClientConfig.ServerName = "" directWardleClientConfig.BearerToken = wardleToKASKubeConfig.BearerToken @@ -585,6 +627,8 @@ func waitForWardleRunning(ctx context.Context, t *testing.T, wardleToKASKubeConf return directWardleClientConfig, nil } +func wardleCAFilePath(wardleCertDir string) string { return path.Join(wardleCertDir, "apiserver.crt") } + func TestAggregatedAPIServerRejectRedirectResponse(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) t.Cleanup(cancel)