Merge pull request #82109 from wk8/wk8/gmsa-e2e

Adding an e2e test on GMSA support

test/e2e/windows/BUILD

@@ -8,12 +8,14 @@ go_library(
         "density.go",
         "dns.go",
         "framework.go",
-        "gmsa.go",
+        "gmsa_full.go",
+        "gmsa_kubelet.go",
         "hybrid_network.go",
         "memory_limits.go",
         "networking.go",
         "security_context.go",
         "service.go",
+        "utils.go",
         "volumes.go",
     ],
     importpath = "k8s.io/kubernetes/test/e2e/windows",
@@ -21,6 +23,7 @@ go_library(
     deps = [
         "//pkg/kubelet/apis/config:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/api/rbac/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
@@ -43,6 +46,7 @@ go_library(
         "//test/utils/image:go_default_library",
         "//vendor/github.com/onsi/ginkgo:go_default_library",
         "//vendor/github.com/onsi/gomega:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
     ],
 )
 

test/e2e/windows/gmsa_full.go

@@ -0,0 +1,396 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// This test ensures that the whole GMSA process works as intended.
+// However, it does require a pretty heavy weight set up to run correctly;
+// in particular, it does make a number of assumptions about the cluster it
+// runs against:
+//  * there exists a Windows worker node with the agentpool=windowsgmsa label on it
+//  * that node is joined to a working Active Directory domain.
+//  * a GMSA account has been created in that AD domain, and then installed on that
+//    same worker.
+//  * a valid k8s manifest file containing a single CRD definition has been generated using
+//    https://github.com/kubernetes-sigs/windows-gmsa/blob/master/scripts/GenerateCredentialSpecResource.ps1
+//    with the credential specs of that GMSA account, or type GMSACredentialSpec and named gmsa-e2e;
+//    and that manifest file has been written to C:\gmsa\gmsa-cred-spec-gmsa-e2e.yml
+//    on that same worker node.
+//  * the API has both MutatingAdmissionWebhook and ValidatingAdmissionWebhook
+//    admission controllers enabled.
+//  * the cluster comprises at least one Linux node that accepts workloads - it
+//    can be the master, but any other Linux node is fine too. This is needed for
+//    the webhook's pod.
+// All these assumptions are fulfilled by an AKS extension when setting up the AKS
+// cluster we run daily e2e tests against, but they do make running this test
+// outside of that very specific context pretty hard.
+
+package windows
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+	"time"
+
+	v1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+	e2elog "k8s.io/kubernetes/test/e2e/framework/log"
+	imageutils "k8s.io/kubernetes/test/utils/image"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+	"github.com/pkg/errors"
+)
+
+const (
+	// gmsaFullNodeLabel is the label we expect to find on at least one node
+	// that is then expected to fulfill all the expectations explained above.
+	gmsaFullNodeLabel = "agentpool=windowsgmsa"
+
+	// gmsaCrdManifestPath is where we expect to find the manifest file for
+	// the GMSA cred spec on that node - see explanations above.
+	gmsaCrdManifestPath = `C:\gmsa\gmsa-cred-spec-gmsa-e2e.yml`
+
+	// gmsaCustomResourceName is the expected name of the GMSA custom resource
+	// defined at gmsaCrdManifestPath
+	gmsaCustomResourceName = "gmsa-e2e"
+
+	// gmsaWebhookDeployScriptURL is the URL of the deploy script for the GMSA webook
+	// TODO(wk8): we should pin versions.
+	gmsaWebhookDeployScriptURL = "https://raw.githubusercontent.com/kubernetes-sigs/windows-gmsa/master/admission-webhook/deploy/deploy-gmsa-webhook.sh"
+)
+
+var _ = SIGDescribe("[Feature:Windows] [Feature:WindowsGMSA] GMSA Full [Slow]", func() {
+	f := framework.NewDefaultFramework("gmsa-full-test-windows")
+
+	ginkgo.Describe("GMSA support", func() {
+		ginkgo.It("works end to end", func() {
+			defer ginkgo.GinkgoRecover()
+
+			ginkgo.By("finding the worker node that fulfills this test's assumptions")
+			nodes := findPreconfiguredGmsaNodes(f.ClientSet)
+			if len(nodes) != 1 {
+				framework.Skipf("Expected to find exactly one node with the %q label, found %d", gmsaFullNodeLabel, len(nodes))
+			}
+			node := nodes[0]
+
+			ginkgo.By("retrieving the contents of the GMSACredentialSpec custom resource manifest from the node")
+			crdManifestContents := retrieveCRDManifestFileContents(f, node)
+
+			ginkgo.By("downloading the GMSA webhook deploy script")
+			deployScriptPath, err := downloadFile(gmsaWebhookDeployScriptURL)
+			defer func() { os.Remove(deployScriptPath) }()
+			if err != nil {
+				e2elog.Failf(err.Error())
+			}
+
+			ginkgo.By("deploying the GMSA webhook")
+			webhookCleanUp, err := deployGmsaWebhook(f, deployScriptPath)
+			defer webhookCleanUp()
+			if err != nil {
+				e2elog.Failf(err.Error())
+			}
+
+			ginkgo.By("creating the GMSA custom resource")
+			customResourceCleanup, err := createGmsaCustomResource(crdManifestContents)
+			defer customResourceCleanup()
+			if err != nil {
+				e2elog.Failf(err.Error())
+			}
+
+			ginkgo.By("creating an RBAC role to grant use access to that GMSA resource")
+			rbacRoleName, rbacRoleCleanup, err := createRBACRoleForGmsa(f)
+			defer rbacRoleCleanup()
+			if err != nil {
+				e2elog.Failf(err.Error())
+			}
+
+			ginkgo.By("creating a service account")
+			serviceAccountName := createServiceAccount(f)
+
+			ginkgo.By("binding the RBAC role to the service account")
+			bindRBACRoleToServiceAccount(f, serviceAccountName, rbacRoleName)
+
+			ginkgo.By("creating a pod using the GMSA cred spec")
+			podName := createPodWithGmsa(f, serviceAccountName)
+
+			// nltest /QUERY will only return successfully if there is a GMSA
+			// identity configured, _and_ it succeeds in contacting the AD controller
+			// and authenticating with it.
+			ginkgo.By("checking that nltest /QUERY returns successfully")
+			var output string
+			gomega.Eventually(func() bool {
+				output, err = runKubectlExecInNamespace(f.Namespace.Name, podName, "nltest", "/QUERY")
+				return err == nil
+			}, 1*time.Minute, 1*time.Second).Should(gomega.BeTrue())
+
+			expectedSubstr := "The command completed successfully"
+			if !strings.Contains(output, expectedSubstr) {
+				e2elog.Failf("Expected %q to contain %q", output, expectedSubstr)
+			}
+		})
+	})
+})
+
+// findPreconfiguredGmsaNode finds node with the gmsaFullNodeLabel label on it.
+func findPreconfiguredGmsaNodes(c clientset.Interface) []v1.Node {
+	nodeOpts := metav1.ListOptions{
+		LabelSelector: gmsaFullNodeLabel,
+	}
+	nodes, err := c.CoreV1().Nodes().List(nodeOpts)
+	if err != nil {
+		e2elog.Failf("Unable to list nodes: %v", err)
+	}
+	return nodes.Items
+}
+
+// retrieveCRDManifestFileContents retrieves the contents of the file
+// at gmsaCrdManifestPath on node; it does so by scheduling a single pod
+// on nodes with the gmsaFullNodeLabel label with that file's directory
+// mounted on it, and then exec-ing into that pod to retrieve the file's
+// contents.
+func retrieveCRDManifestFileContents(f *framework.Framework, node v1.Node) string {
+	podName := "retrieve-gmsa-crd-contents"
+	// we can't use filepath.Dir here since the test itself runs on a Linux machine
+	splitPath := strings.Split(gmsaCrdManifestPath, `\`)
+	dirPath := strings.Join(splitPath[:len(splitPath)-1], `\`)
+	volumeName := "retrieve-gmsa-crd-contents-volume"
+
+	pod := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      podName,
+			Namespace: f.Namespace.Name,
+		},
+		Spec: v1.PodSpec{
+			NodeSelector: node.Labels,
+			Containers: []v1.Container{
+				{
+					Name:  podName,
+					Image: imageutils.GetPauseImageName(),
+					VolumeMounts: []v1.VolumeMount{
+						{
+							Name:      volumeName,
+							MountPath: dirPath,
+						},
+					},
+				},
+			},
+			Volumes: []v1.Volume{
+				{
+					Name: volumeName,
+					VolumeSource: v1.VolumeSource{
+						HostPath: &v1.HostPathVolumeSource{
+							Path: dirPath,
+						},
+					},
+				},
+			},
+		},
+	}
+	f.PodClient().CreateSync(pod)
+
+	// using powershell and using forward slashes avoids the nightmare of having to properly
+	// escape quotes and backward slashes
+	output, err := runKubectlExecInNamespace(f.Namespace.Name, podName, "powershell", "Get-Content", strings.ReplaceAll(gmsaCrdManifestPath, `\`, "/"))
+	if err != nil {
+		e2elog.Failf("failed to retrieve the contents of %q on node %q: %v", gmsaCrdManifestPath, node.Name, err)
+	}
+
+	// Windows to linux new lines
+	return strings.ReplaceAll(output, "\r\n", "\n")
+}
+
+// deployGmsaWebhook deploys the GMSA webhook, and returns a cleanup function
+// to be called when done with testing, that removes the temp files it's created
+// on disks as well as the API resources it's created.
+func deployGmsaWebhook(f *framework.Framework, deployScriptPath string) (func(), error) {
+	cleanUpFunc := func() {}
+
+	tempDir, err := ioutil.TempDir("", "")
+	if err != nil {
+		return cleanUpFunc, errors.Wrapf(err, "unable to create temp dir")
+	}
+
+	manifestsFile := path.Join(tempDir, "manifests.yml")
+	name := "gmsa-webhook"
+	namespace := f.Namespace.Name + "-webhook"
+	certsDir := path.Join(tempDir, "certs")
+
+	// regardless of whether the deployment succeeded, let's do a best effort at cleanup
+	cleanUpFunc = func() {
+		framework.RunKubectl("delete", "--filename", manifestsFile)
+		framework.RunKubectl("delete", "CustomResourceDefinition", "gmsacredentialspecs.windows.k8s.io")
+		framework.RunKubectl("delete", "CertificateSigningRequest", fmt.Sprintf("%s.%s", name, namespace))
+		os.RemoveAll(tempDir)
+	}
+
+	cmd := exec.Command("bash", deployScriptPath,
+		"--file", manifestsFile,
+		"--name", name,
+		"--namespace", namespace,
+		"--certs-dir", certsDir)
+
+	output, err := cmd.CombinedOutput()
+	if err == nil {
+		e2elog.Logf("GMSA webhook successfully deployed, output:\n%s", string(output))
+	} else {
+		err = errors.Wrapf(err, "unable to deploy GMSA webhook, output:\n%s", string(output))
+	}
+
+	return cleanUpFunc, err
+}
+
+// createGmsaCustomResource creates the GMSA API object from the contents
+// of the manifest file retrieved from the worker node.
+// It returns a function to clean up both the temp file it creates and
+// the API object it creates when done with testing.
+func createGmsaCustomResource(crdManifestContents string) (func(), error) {
+	cleanUpFunc := func() {}
+
+	tempFile, err := ioutil.TempFile("", "")
+	if err != nil {
+		return cleanUpFunc, errors.Wrapf(err, "unable to create temp file")
+	}
+	defer tempFile.Close()
+
+	cleanUpFunc = func() {
+		framework.RunKubectl("delete", "--filename", tempFile.Name())
+		os.Remove(tempFile.Name())
+	}
+
+	_, err = tempFile.WriteString(crdManifestContents)
+	if err != nil {
+		err = errors.Wrapf(err, "unable to write GMSA contents to %q", tempFile.Name())
+		return cleanUpFunc, err
+	}
+
+	output, err := framework.RunKubectl("apply", "--filename", tempFile.Name())
+	if err != nil {
+		err = errors.Wrapf(err, "unable to create custom resource, output:\n%s", output)
+	}
+
+	return cleanUpFunc, err
+}
+
+// createRBACRoleForGmsa creates an RBAC cluster role to grant use
+// access to our test credential spec.
+// It returns the role's name, as well as a function to delete it when done.
+func createRBACRoleForGmsa(f *framework.Framework) (string, func(), error) {
+	roleName := f.Namespace.Name + "-rbac-role"
+
+	role := &rbacv1.ClusterRole{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: roleName,
+		},
+		Rules: []rbacv1.PolicyRule{
+			{
+				APIGroups:     []string{"windows.k8s.io"},
+				Resources:     []string{"gmsacredentialspecs"},
+				Verbs:         []string{"use"},
+				ResourceNames: []string{gmsaCustomResourceName},
+			},
+		},
+	}
+
+	cleanUpFunc := func() {
+		f.ClientSet.RbacV1().ClusterRoles().Delete(roleName, &metav1.DeleteOptions{})
+	}
+
+	_, err := f.ClientSet.RbacV1().ClusterRoles().Create(role)
+	if err != nil {
+		err = errors.Wrapf(err, "unable to create RBAC cluster role %q", roleName)
+	}
+
+	return roleName, cleanUpFunc, err
+}
+
+// createServiceAccount creates a service account, and returns its name.
+func createServiceAccount(f *framework.Framework) string {
+	accountName := f.Namespace.Name + "-sa"
+	account := &v1.ServiceAccount{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      accountName,
+			Namespace: f.Namespace.Name,
+		},
+	}
+	if _, err := f.ClientSet.CoreV1().ServiceAccounts(f.Namespace.Name).Create(account); err != nil {
+		e2elog.Failf("unable to create service account %q: %v", accountName, err)
+	}
+	return accountName
+}
+
+// bindRBACRoleToServiceAccount binds the given RBAC cluster role to the given service account.
+func bindRBACRoleToServiceAccount(f *framework.Framework, serviceAccountName, rbacRoleName string) {
+	binding := &rbacv1.RoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      f.Namespace.Name + "-rbac-binding",
+			Namespace: f.Namespace.Name,
+		},
+		Subjects: []rbacv1.Subject{
+			{
+				Kind:      "ServiceAccount",
+				Name:      serviceAccountName,
+				Namespace: f.Namespace.Name,
+			},
+		},
+		RoleRef: rbacv1.RoleRef{
+			APIGroup: "rbac.authorization.k8s.io",
+			Kind:     "ClusterRole",
+			Name:     rbacRoleName,
+		},
+	}
+	f.ClientSet.RbacV1().RoleBindings(f.Namespace.Name).Create(binding)
+}
+
+// createPodWithGmsa creates a pod using the test GMSA cred spec, and returns its name.
+func createPodWithGmsa(f *framework.Framework, serviceAccountName string) string {
+	podName := "pod-with-gmsa"
+	credSpecName := gmsaCustomResourceName
+
+	pod := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      podName,
+			Namespace: f.Namespace.Name,
+		},
+		Spec: v1.PodSpec{
+			ServiceAccountName: serviceAccountName,
+			Containers: []v1.Container{
+				{
+					Name:  podName,
+					Image: imageutils.GetPauseImageName(),
+				},
+			},
+			SecurityContext: &v1.PodSecurityContext{
+				WindowsOptions: &v1.WindowsSecurityContextOptions{
+					GMSACredentialSpecName: &credSpecName,
+				},
+			},
+		},
+	}
+	f.PodClient().CreateSync(pod)
+
+	return podName
+}
+
+func runKubectlExecInNamespace(namespace string, args ...string) (string, error) {
+	namespaceOption := fmt.Sprintf("--namespace=%s", namespace)
+	return framework.RunKubectl(append([]string{"exec", namespaceOption}, args...)...)
+}

test/e2e/windows/gmsa_kubelet.go

@@ -14,6 +14,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+// This test only makes sure that the kubelet correctly passes down GMSA cred specs
+// down to the runtime.
+// It's a much lighter test than gmsa_full, that tests the whole GMSA process, but
+// also requires a heavy weight setup to run.
+
 package windows
 
 import (
@@ -31,15 +36,15 @@ import (
 	"github.com/onsi/gomega"
 )
 
-var _ = SIGDescribe("[Feature:Windows] [Feature:WindowsGMSA] GMSA [Slow]", func() {
+var _ = SIGDescribe("[Feature:Windows] [Feature:WindowsGMSA] GMSA Kubelet [Slow]", func() {
-	f := framework.NewDefaultFramework("gmsa-test-windows")
+	f := framework.NewDefaultFramework("gmsa-kubelet-test-windows")
 
 	ginkgo.Describe("kubelet GMSA support", func() {
 		ginkgo.Context("when creating a pod with correct GMSA credential specs", func() {
 			ginkgo.It("passes the credential specs down to the Pod's containers", func() {
 				defer ginkgo.GinkgoRecover()
 
-				podName := "with-correct-gmsa-annotations"
+				podName := "with-correct-gmsa-specs"
 
 				container1Name := "container1"
 				podDomain := "acme.com"
@@ -75,7 +80,7 @@ var _ = SIGDescribe("[Feature:Windows] [Feature:WindowsGMSA] GMSA [Slow]", func(
 					},
 				}
 
-				ginkgo.By("creating a pod with correct GMSA annotations")
+				ginkgo.By("creating a pod with correct GMSA specs")
 				f.PodClient().CreateSync(pod)
 
 				ginkgo.By("checking the domain reported by nltest in the containers")

test/e2e/windows/utils.go

@@ -0,0 +1,44 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package windows
+
+import (
+	"io"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/pkg/errors"
+)
+
+// downloadFile saves a remote URL to a local temp file, and returns its path.
+// It's the caller's responsibility to clean up the temp file when done.
+func downloadFile(url string) (string, error) {
+	response, err := http.Get(url)
+	if err != nil {
+		return "", errors.Wrapf(err, "unable to download from %q", url)
+	}
+	defer response.Body.Close()
+
+	tempFile, err := ioutil.TempFile("", "")
+	if err != nil {
+		return "", errors.Wrapf(err, "unable to create temp file")
+	}
+	defer tempFile.Close()
+
+	_, err = io.Copy(tempFile, response.Body)
+	return tempFile.Name(), err
+}