From fd788f3476bb1e398004a7d66bdbe1569c0015e9 Mon Sep 17 00:00:00 2001 From: Jean Rouge Date: Wed, 28 Aug 2019 19:47:09 -0700 Subject: [PATCH] Adding an e2e test on GMSA support The previously existing e2e GMSA test really only tests a small part of the whole GMSA set up process, namely that once the API has inlined the GMSA contents in the pod's spec, and sent that to a worker's kubelet, then the kubelet passes that down to the runtime. This new test, in contrast, really tests the whole thing, i.e. deploying the admission webhook, then deploying a GMSA custom resource, and using that resource within a pod. The downside of this test though, is that it does need to make a lot of assumptions about the cluster it runs against, notably that it runs on a worker node that's already been joined to a working Active Directory domain (there are other assumptions, all documented at the beginning of the test file); for that reason, it is only intended to ever be run against an AKS cluster with the custom AKS extension from https://github.com/kubernetes-sigs/windows-testing/pull/98. Note that this test doesn't aim at testing every edge-case, such as a pod trying to use a GMSA it doesn't have access to; the webhook has its own tests for these. This test's goal is to ensure the happy path doesn't break. Signed-off-by: Jean Rouge --- test/e2e/windows/BUILD | 6 +- test/e2e/windows/gmsa_full.go | 396 ++++++++++++++++++ test/e2e/windows/{gmsa.go => gmsa_kubelet.go} | 13 +- test/e2e/windows/utils.go | 44 ++ 4 files changed, 454 insertions(+), 5 deletions(-) create mode 100644 test/e2e/windows/gmsa_full.go rename test/e2e/windows/{gmsa.go => gmsa_kubelet.go} (91%) create mode 100644 test/e2e/windows/utils.go diff --git a/test/e2e/windows/BUILD b/test/e2e/windows/BUILD index 1dc47dff9f9..028521381e2 100644 --- a/test/e2e/windows/BUILD +++ b/test/e2e/windows/BUILD @@ -8,12 +8,14 @@ go_library( "density.go", "dns.go", "framework.go", - "gmsa.go", + "gmsa_full.go", + "gmsa_kubelet.go", "hybrid_network.go", "memory_limits.go", "networking.go", "security_context.go", "service.go", + "utils.go", "volumes.go", ], importpath = "k8s.io/kubernetes/test/e2e/windows", @@ -21,6 +23,7 @@ go_library( deps = [ "//pkg/kubelet/apis/config:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/api/rbac/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", @@ -43,6 +46,7 @@ go_library( "//test/utils/image:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", + "//vendor/github.com/pkg/errors:go_default_library", ], ) diff --git a/test/e2e/windows/gmsa_full.go b/test/e2e/windows/gmsa_full.go new file mode 100644 index 00000000000..98345143a63 --- /dev/null +++ b/test/e2e/windows/gmsa_full.go @@ -0,0 +1,396 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This test ensures that the whole GMSA process works as intended. +// However, it does require a pretty heavy weight set up to run correctly; +// in particular, it does make a number of assumptions about the cluster it +// runs against: +// * there exists a Windows worker node with the agentpool=windowsgmsa label on it +// * that node is joined to a working Active Directory domain. +// * a GMSA account has been created in that AD domain, and then installed on that +// same worker. +// * a valid k8s manifest file containing a single CRD definition has been generated using +// https://github.com/kubernetes-sigs/windows-gmsa/blob/master/scripts/GenerateCredentialSpecResource.ps1 +// with the credential specs of that GMSA account, or type GMSACredentialSpec and named gmsa-e2e; +// and that manifest file has been written to C:\gmsa\gmsa-cred-spec-gmsa-e2e.yml +// on that same worker node. +// * the API has both MutatingAdmissionWebhook and ValidatingAdmissionWebhook +// admission controllers enabled. +// * the cluster comprises at least one Linux node that accepts workloads - it +// can be the master, but any other Linux node is fine too. This is needed for +// the webhook's pod. +// All these assumptions are fulfilled by an AKS extension when setting up the AKS +// cluster we run daily e2e tests against, but they do make running this test +// outside of that very specific context pretty hard. + +package windows + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path" + "strings" + "time" + + v1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + e2elog "k8s.io/kubernetes/test/e2e/framework/log" + imageutils "k8s.io/kubernetes/test/utils/image" + + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" + "github.com/pkg/errors" +) + +const ( + // gmsaFullNodeLabel is the label we expect to find on at least one node + // that is then expected to fulfill all the expectations explained above. + gmsaFullNodeLabel = "agentpool=windowsgmsa" + + // gmsaCrdManifestPath is where we expect to find the manifest file for + // the GMSA cred spec on that node - see explanations above. + gmsaCrdManifestPath = `C:\gmsa\gmsa-cred-spec-gmsa-e2e.yml` + + // gmsaCustomResourceName is the expected name of the GMSA custom resource + // defined at gmsaCrdManifestPath + gmsaCustomResourceName = "gmsa-e2e" + + // gmsaWebhookDeployScriptURL is the URL of the deploy script for the GMSA webook + // TODO(wk8): we should pin versions. + gmsaWebhookDeployScriptURL = "https://raw.githubusercontent.com/kubernetes-sigs/windows-gmsa/master/admission-webhook/deploy/deploy-gmsa-webhook.sh" +) + +var _ = SIGDescribe("[Feature:Windows] [Feature:WindowsGMSA] GMSA Full [Slow]", func() { + f := framework.NewDefaultFramework("gmsa-full-test-windows") + + ginkgo.Describe("GMSA support", func() { + ginkgo.It("works end to end", func() { + defer ginkgo.GinkgoRecover() + + ginkgo.By("finding the worker node that fulfills this test's assumptions") + nodes := findPreconfiguredGmsaNodes(f.ClientSet) + if len(nodes) != 1 { + framework.Skipf("Expected to find exactly one node with the %q label, found %d", gmsaFullNodeLabel, len(nodes)) + } + node := nodes[0] + + ginkgo.By("retrieving the contents of the GMSACredentialSpec custom resource manifest from the node") + crdManifestContents := retrieveCRDManifestFileContents(f, node) + + ginkgo.By("downloading the GMSA webhook deploy script") + deployScriptPath, err := downloadFile(gmsaWebhookDeployScriptURL) + defer func() { os.Remove(deployScriptPath) }() + if err != nil { + e2elog.Failf(err.Error()) + } + + ginkgo.By("deploying the GMSA webhook") + webhookCleanUp, err := deployGmsaWebhook(f, deployScriptPath) + defer webhookCleanUp() + if err != nil { + e2elog.Failf(err.Error()) + } + + ginkgo.By("creating the GMSA custom resource") + customResourceCleanup, err := createGmsaCustomResource(crdManifestContents) + defer customResourceCleanup() + if err != nil { + e2elog.Failf(err.Error()) + } + + ginkgo.By("creating an RBAC role to grant use access to that GMSA resource") + rbacRoleName, rbacRoleCleanup, err := createRBACRoleForGmsa(f) + defer rbacRoleCleanup() + if err != nil { + e2elog.Failf(err.Error()) + } + + ginkgo.By("creating a service account") + serviceAccountName := createServiceAccount(f) + + ginkgo.By("binding the RBAC role to the service account") + bindRBACRoleToServiceAccount(f, serviceAccountName, rbacRoleName) + + ginkgo.By("creating a pod using the GMSA cred spec") + podName := createPodWithGmsa(f, serviceAccountName) + + // nltest /QUERY will only return successfully if there is a GMSA + // identity configured, _and_ it succeeds in contacting the AD controller + // and authenticating with it. + ginkgo.By("checking that nltest /QUERY returns successfully") + var output string + gomega.Eventually(func() bool { + output, err = runKubectlExecInNamespace(f.Namespace.Name, podName, "nltest", "/QUERY") + return err == nil + }, 1*time.Minute, 1*time.Second).Should(gomega.BeTrue()) + + expectedSubstr := "The command completed successfully" + if !strings.Contains(output, expectedSubstr) { + e2elog.Failf("Expected %q to contain %q", output, expectedSubstr) + } + }) + }) +}) + +// findPreconfiguredGmsaNode finds node with the gmsaFullNodeLabel label on it. +func findPreconfiguredGmsaNodes(c clientset.Interface) []v1.Node { + nodeOpts := metav1.ListOptions{ + LabelSelector: gmsaFullNodeLabel, + } + nodes, err := c.CoreV1().Nodes().List(nodeOpts) + if err != nil { + e2elog.Failf("Unable to list nodes: %v", err) + } + return nodes.Items +} + +// retrieveCRDManifestFileContents retrieves the contents of the file +// at gmsaCrdManifestPath on node; it does so by scheduling a single pod +// on nodes with the gmsaFullNodeLabel label with that file's directory +// mounted on it, and then exec-ing into that pod to retrieve the file's +// contents. +func retrieveCRDManifestFileContents(f *framework.Framework, node v1.Node) string { + podName := "retrieve-gmsa-crd-contents" + // we can't use filepath.Dir here since the test itself runs on a Linux machine + splitPath := strings.Split(gmsaCrdManifestPath, `\`) + dirPath := strings.Join(splitPath[:len(splitPath)-1], `\`) + volumeName := "retrieve-gmsa-crd-contents-volume" + + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: f.Namespace.Name, + }, + Spec: v1.PodSpec{ + NodeSelector: node.Labels, + Containers: []v1.Container{ + { + Name: podName, + Image: imageutils.GetPauseImageName(), + VolumeMounts: []v1.VolumeMount{ + { + Name: volumeName, + MountPath: dirPath, + }, + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: volumeName, + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: dirPath, + }, + }, + }, + }, + }, + } + f.PodClient().CreateSync(pod) + + // using powershell and using forward slashes avoids the nightmare of having to properly + // escape quotes and backward slashes + output, err := runKubectlExecInNamespace(f.Namespace.Name, podName, "powershell", "Get-Content", strings.ReplaceAll(gmsaCrdManifestPath, `\`, "/")) + if err != nil { + e2elog.Failf("failed to retrieve the contents of %q on node %q: %v", gmsaCrdManifestPath, node.Name, err) + } + + // Windows to linux new lines + return strings.ReplaceAll(output, "\r\n", "\n") +} + +// deployGmsaWebhook deploys the GMSA webhook, and returns a cleanup function +// to be called when done with testing, that removes the temp files it's created +// on disks as well as the API resources it's created. +func deployGmsaWebhook(f *framework.Framework, deployScriptPath string) (func(), error) { + cleanUpFunc := func() {} + + tempDir, err := ioutil.TempDir("", "") + if err != nil { + return cleanUpFunc, errors.Wrapf(err, "unable to create temp dir") + } + + manifestsFile := path.Join(tempDir, "manifests.yml") + name := "gmsa-webhook" + namespace := f.Namespace.Name + "-webhook" + certsDir := path.Join(tempDir, "certs") + + // regardless of whether the deployment succeeded, let's do a best effort at cleanup + cleanUpFunc = func() { + framework.RunKubectl("delete", "--filename", manifestsFile) + framework.RunKubectl("delete", "CustomResourceDefinition", "gmsacredentialspecs.windows.k8s.io") + framework.RunKubectl("delete", "CertificateSigningRequest", fmt.Sprintf("%s.%s", name, namespace)) + os.RemoveAll(tempDir) + } + + cmd := exec.Command("bash", deployScriptPath, + "--file", manifestsFile, + "--name", name, + "--namespace", namespace, + "--certs-dir", certsDir) + + output, err := cmd.CombinedOutput() + if err == nil { + e2elog.Logf("GMSA webhook successfully deployed, output:\n%s", string(output)) + } else { + err = errors.Wrapf(err, "unable to deploy GMSA webhook, output:\n%s", string(output)) + } + + return cleanUpFunc, err +} + +// createGmsaCustomResource creates the GMSA API object from the contents +// of the manifest file retrieved from the worker node. +// It returns a function to clean up both the temp file it creates and +// the API object it creates when done with testing. +func createGmsaCustomResource(crdManifestContents string) (func(), error) { + cleanUpFunc := func() {} + + tempFile, err := ioutil.TempFile("", "") + if err != nil { + return cleanUpFunc, errors.Wrapf(err, "unable to create temp file") + } + defer tempFile.Close() + + cleanUpFunc = func() { + framework.RunKubectl("delete", "--filename", tempFile.Name()) + os.Remove(tempFile.Name()) + } + + _, err = tempFile.WriteString(crdManifestContents) + if err != nil { + err = errors.Wrapf(err, "unable to write GMSA contents to %q", tempFile.Name()) + return cleanUpFunc, err + } + + output, err := framework.RunKubectl("apply", "--filename", tempFile.Name()) + if err != nil { + err = errors.Wrapf(err, "unable to create custom resource, output:\n%s", output) + } + + return cleanUpFunc, err +} + +// createRBACRoleForGmsa creates an RBAC cluster role to grant use +// access to our test credential spec. +// It returns the role's name, as well as a function to delete it when done. +func createRBACRoleForGmsa(f *framework.Framework) (string, func(), error) { + roleName := f.Namespace.Name + "-rbac-role" + + role := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: roleName, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"windows.k8s.io"}, + Resources: []string{"gmsacredentialspecs"}, + Verbs: []string{"use"}, + ResourceNames: []string{gmsaCustomResourceName}, + }, + }, + } + + cleanUpFunc := func() { + f.ClientSet.RbacV1().ClusterRoles().Delete(roleName, &metav1.DeleteOptions{}) + } + + _, err := f.ClientSet.RbacV1().ClusterRoles().Create(role) + if err != nil { + err = errors.Wrapf(err, "unable to create RBAC cluster role %q", roleName) + } + + return roleName, cleanUpFunc, err +} + +// createServiceAccount creates a service account, and returns its name. +func createServiceAccount(f *framework.Framework) string { + accountName := f.Namespace.Name + "-sa" + account := &v1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: accountName, + Namespace: f.Namespace.Name, + }, + } + if _, err := f.ClientSet.CoreV1().ServiceAccounts(f.Namespace.Name).Create(account); err != nil { + e2elog.Failf("unable to create service account %q: %v", accountName, err) + } + return accountName +} + +// bindRBACRoleToServiceAccount binds the given RBAC cluster role to the given service account. +func bindRBACRoleToServiceAccount(f *framework.Framework, serviceAccountName, rbacRoleName string) { + binding := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: f.Namespace.Name + "-rbac-binding", + Namespace: f.Namespace.Name, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: serviceAccountName, + Namespace: f.Namespace.Name, + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: rbacRoleName, + }, + } + f.ClientSet.RbacV1().RoleBindings(f.Namespace.Name).Create(binding) +} + +// createPodWithGmsa creates a pod using the test GMSA cred spec, and returns its name. +func createPodWithGmsa(f *framework.Framework, serviceAccountName string) string { + podName := "pod-with-gmsa" + credSpecName := gmsaCustomResourceName + + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: f.Namespace.Name, + }, + Spec: v1.PodSpec{ + ServiceAccountName: serviceAccountName, + Containers: []v1.Container{ + { + Name: podName, + Image: imageutils.GetPauseImageName(), + }, + }, + SecurityContext: &v1.PodSecurityContext{ + WindowsOptions: &v1.WindowsSecurityContextOptions{ + GMSACredentialSpecName: &credSpecName, + }, + }, + }, + } + f.PodClient().CreateSync(pod) + + return podName +} + +func runKubectlExecInNamespace(namespace string, args ...string) (string, error) { + namespaceOption := fmt.Sprintf("--namespace=%s", namespace) + return framework.RunKubectl(append([]string{"exec", namespaceOption}, args...)...) +} diff --git a/test/e2e/windows/gmsa.go b/test/e2e/windows/gmsa_kubelet.go similarity index 91% rename from test/e2e/windows/gmsa.go rename to test/e2e/windows/gmsa_kubelet.go index 239411df667..596330363fd 100644 --- a/test/e2e/windows/gmsa.go +++ b/test/e2e/windows/gmsa_kubelet.go @@ -14,6 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. */ +// This test only makes sure that the kubelet correctly passes down GMSA cred specs +// down to the runtime. +// It's a much lighter test than gmsa_full, that tests the whole GMSA process, but +// also requires a heavy weight setup to run. + package windows import ( @@ -31,15 +36,15 @@ import ( "github.com/onsi/gomega" ) -var _ = SIGDescribe("[Feature:Windows] [Feature:WindowsGMSA] GMSA [Slow]", func() { - f := framework.NewDefaultFramework("gmsa-test-windows") +var _ = SIGDescribe("[Feature:Windows] [Feature:WindowsGMSA] GMSA Kubelet [Slow]", func() { + f := framework.NewDefaultFramework("gmsa-kubelet-test-windows") ginkgo.Describe("kubelet GMSA support", func() { ginkgo.Context("when creating a pod with correct GMSA credential specs", func() { ginkgo.It("passes the credential specs down to the Pod's containers", func() { defer ginkgo.GinkgoRecover() - podName := "with-correct-gmsa-annotations" + podName := "with-correct-gmsa-specs" container1Name := "container1" podDomain := "acme.com" @@ -75,7 +80,7 @@ var _ = SIGDescribe("[Feature:Windows] [Feature:WindowsGMSA] GMSA [Slow]", func( }, } - ginkgo.By("creating a pod with correct GMSA annotations") + ginkgo.By("creating a pod with correct GMSA specs") f.PodClient().CreateSync(pod) ginkgo.By("checking the domain reported by nltest in the containers") diff --git a/test/e2e/windows/utils.go b/test/e2e/windows/utils.go new file mode 100644 index 00000000000..c89c9d00ae2 --- /dev/null +++ b/test/e2e/windows/utils.go @@ -0,0 +1,44 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package windows + +import ( + "io" + "io/ioutil" + "net/http" + + "github.com/pkg/errors" +) + +// downloadFile saves a remote URL to a local temp file, and returns its path. +// It's the caller's responsibility to clean up the temp file when done. +func downloadFile(url string) (string, error) { + response, err := http.Get(url) + if err != nil { + return "", errors.Wrapf(err, "unable to download from %q", url) + } + defer response.Body.Close() + + tempFile, err := ioutil.TempFile("", "") + if err != nil { + return "", errors.Wrapf(err, "unable to create temp file") + } + defer tempFile.Close() + + _, err = io.Copy(tempFile, response.Body) + return tempFile.Name(), err +}