github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-30 06:54:01 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #82109 from wk8/wk8/gmsa-e2e

Browse Source 
Adding an e2e test on GMSA support
This commit is contained in:
Kubernetes Prow Robot 2019-08-30 08:00:57 -07:00 committed by GitHub
commit e176e47719
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 454 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
test/e2e/windows/BUILD
View File

@ -8,12 +8,14 @@ go_library(
"density.go",
"dns.go",
"framework.go",
"gmsa.go",
"gmsa_full.go",
"gmsa_kubelet.go",
"hybrid_network.go",
"memory_limits.go",
"networking.go",
"security_context.go",
"service.go",
"utils.go",
"volumes.go",
],
importpath = "k8s.io/kubernetes/test/e2e/windows",
@ -21,6 +23,7 @@ go_library(
deps = [
"//pkg/kubelet/apis/config:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/api/rbac/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
@ -43,6 +46,7 @@ go_library(
"//test/utils/image:go_default_library",
"//vendor/github.com/onsi/ginkgo:go_default_library",
"//vendor/github.com/onsi/gomega:go_default_library",
"//vendor/github.com/pkg/errors:go_default_library",
],
)

396
test/e2e/windows/gmsa_full.go Normal file
View File

@ -0,0 +1,396 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// This test ensures that the whole GMSA process works as intended.
// However, it does require a pretty heavy weight set up to run correctly;
// in particular, it does make a number of assumptions about the cluster it
// runs against:
// * there exists a Windows worker node with the agentpool=windowsgmsa label on it
// * that node is joined to a working Active Directory domain.
// * a GMSA account has been created in that AD domain, and then installed on that
// same worker.
// * a valid k8s manifest file containing a single CRD definition has been generated using
// https://github.com/kubernetes-sigs/windows-gmsa/blob/master/scripts/GenerateCredentialSpecResource.ps1
// with the credential specs of that GMSA account, or type GMSACredentialSpec and named gmsa-e2e;
// and that manifest file has been written to C:\gmsa\gmsa-cred-spec-gmsa-e2e.yml
// on that same worker node.
// * the API has both MutatingAdmissionWebhook and ValidatingAdmissionWebhook
// admission controllers enabled.
// * the cluster comprises at least one Linux node that accepts workloads - it
// can be the master, but any other Linux node is fine too. This is needed for
// the webhook's pod.
// All these assumptions are fulfilled by an AKS extension when setting up the AKS
// cluster we run daily e2e tests against, but they do make running this test
// outside of that very specific context pretty hard.
package windows
import (
"fmt"
"io/ioutil"
"os"
"os/exec"
"path"
"strings"
"time"
v1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
e2elog "k8s.io/kubernetes/test/e2e/framework/log"
imageutils "k8s.io/kubernetes/test/utils/image"
"github.com/onsi/ginkgo"
"github.com/onsi/gomega"
"github.com/pkg/errors"
)
const (
// gmsaFullNodeLabel is the label we expect to find on at least one node
// that is then expected to fulfill all the expectations explained above.
gmsaFullNodeLabel = "agentpool=windowsgmsa"
// gmsaCrdManifestPath is where we expect to find the manifest file for
// the GMSA cred spec on that node - see explanations above.
gmsaCrdManifestPath = `C:\gmsa\gmsa-cred-spec-gmsa-e2e.yml`
// gmsaCustomResourceName is the expected name of the GMSA custom resource
// defined at gmsaCrdManifestPath
gmsaCustomResourceName = "gmsa-e2e"
// gmsaWebhookDeployScriptURL is the URL of the deploy script for the GMSA webook
// TODO(wk8): we should pin versions.
gmsaWebhookDeployScriptURL = "https://raw.githubusercontent.com/kubernetes-sigs/windows-gmsa/master/admission-webhook/deploy/deploy-gmsa-webhook.sh"
)
var _ = SIGDescribe("[Feature:Windows] [Feature:WindowsGMSA] GMSA Full [Slow]", func() {
f := framework.NewDefaultFramework("gmsa-full-test-windows")
ginkgo.Describe("GMSA support", func() {
ginkgo.It("works end to end", func() {
defer ginkgo.GinkgoRecover()
ginkgo.By("finding the worker node that fulfills this test's assumptions")
nodes := findPreconfiguredGmsaNodes(f.ClientSet)
if len(nodes) != 1 {
framework.Skipf("Expected to find exactly one node with the %q label, found %d", gmsaFullNodeLabel, len(nodes))
}
node := nodes[0]
ginkgo.By("retrieving the contents of the GMSACredentialSpec custom resource manifest from the node")
crdManifestContents := retrieveCRDManifestFileContents(f, node)
ginkgo.By("downloading the GMSA webhook deploy script")
deployScriptPath, err := downloadFile(gmsaWebhookDeployScriptURL)
defer func() { os.Remove(deployScriptPath) }()
if err != nil {
e2elog.Failf(err.Error())
}
ginkgo.By("deploying the GMSA webhook")
webhookCleanUp, err := deployGmsaWebhook(f, deployScriptPath)
defer webhookCleanUp()
if err != nil {
e2elog.Failf(err.Error())
}
ginkgo.By("creating the GMSA custom resource")
customResourceCleanup, err := createGmsaCustomResource(crdManifestContents)
defer customResourceCleanup()
if err != nil {
e2elog.Failf(err.Error())
}
ginkgo.By("creating an RBAC role to grant use access to that GMSA resource")
rbacRoleName, rbacRoleCleanup, err := createRBACRoleForGmsa(f)
defer rbacRoleCleanup()
if err != nil {
e2elog.Failf(err.Error())
}
ginkgo.By("creating a service account")
serviceAccountName := createServiceAccount(f)
ginkgo.By("binding the RBAC role to the service account")
bindRBACRoleToServiceAccount(f, serviceAccountName, rbacRoleName)
ginkgo.By("creating a pod using the GMSA cred spec")
podName := createPodWithGmsa(f, serviceAccountName)
// nltest /QUERY will only return successfully if there is a GMSA
// identity configured, _and_ it succeeds in contacting the AD controller
// and authenticating with it.
ginkgo.By("checking that nltest /QUERY returns successfully")
var output string
gomega.Eventually(func() bool {
output, err = runKubectlExecInNamespace(f.Namespace.Name, podName, "nltest", "/QUERY")
return err == nil
}, 1*time.Minute, 1*time.Second).Should(gomega.BeTrue())
expectedSubstr := "The command completed successfully"
if !strings.Contains(output, expectedSubstr) {
e2elog.Failf("Expected %q to contain %q", output, expectedSubstr)
}
})
})
})
// findPreconfiguredGmsaNode finds node with the gmsaFullNodeLabel label on it.
func findPreconfiguredGmsaNodes(c clientset.Interface) []v1.Node {
nodeOpts := metav1.ListOptions{
LabelSelector: gmsaFullNodeLabel,
}
nodes, err := c.CoreV1().Nodes().List(nodeOpts)
if err != nil {
e2elog.Failf("Unable to list nodes: %v", err)
}
return nodes.Items
}
// retrieveCRDManifestFileContents retrieves the contents of the file
// at gmsaCrdManifestPath on node; it does so by scheduling a single pod
// on nodes with the gmsaFullNodeLabel label with that file's directory
// mounted on it, and then exec-ing into that pod to retrieve the file's
// contents.
func retrieveCRDManifestFileContents(f *framework.Framework, node v1.Node) string {
podName := "retrieve-gmsa-crd-contents"
// we can't use filepath.Dir here since the test itself runs on a Linux machine
splitPath := strings.Split(gmsaCrdManifestPath, `\`)
dirPath := strings.Join(splitPath[:len(splitPath)-1], `\`)
volumeName := "retrieve-gmsa-crd-contents-volume"
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: podName,
Namespace: f.Namespace.Name,
},
Spec: v1.PodSpec{
NodeSelector: node.Labels,
Containers: []v1.Container{
{
Name: podName,
Image: imageutils.GetPauseImageName(),
VolumeMounts: []v1.VolumeMount{
{
Name: volumeName,
MountPath: dirPath,
},
},
},
},
Volumes: []v1.Volume{
{
Name: volumeName,
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: dirPath,
},
},
},
},
},
}
f.PodClient().CreateSync(pod)
// using powershell and using forward slashes avoids the nightmare of having to properly
// escape quotes and backward slashes
output, err := runKubectlExecInNamespace(f.Namespace.Name, podName, "powershell", "Get-Content", strings.ReplaceAll(gmsaCrdManifestPath, `\`, "/"))
if err != nil {
e2elog.Failf("failed to retrieve the contents of %q on node %q: %v", gmsaCrdManifestPath, node.Name, err)
}
// Windows to linux new lines
return strings.ReplaceAll(output, "\r\n", "\n")
}
// deployGmsaWebhook deploys the GMSA webhook, and returns a cleanup function
// to be called when done with testing, that removes the temp files it's created
// on disks as well as the API resources it's created.
func deployGmsaWebhook(f *framework.Framework, deployScriptPath string) (func(), error) {
cleanUpFunc := func() {}
tempDir, err := ioutil.TempDir("", "")
if err != nil {
return cleanUpFunc, errors.Wrapf(err, "unable to create temp dir")
}
manifestsFile := path.Join(tempDir, "manifests.yml")
name := "gmsa-webhook"
namespace := f.Namespace.Name + "-webhook"
certsDir := path.Join(tempDir, "certs")
// regardless of whether the deployment succeeded, let's do a best effort at cleanup
cleanUpFunc = func() {
framework.RunKubectl("delete", "--filename", manifestsFile)
framework.RunKubectl("delete", "CustomResourceDefinition", "gmsacredentialspecs.windows.k8s.io")
framework.RunKubectl("delete", "CertificateSigningRequest", fmt.Sprintf("%s.%s", name, namespace))
os.RemoveAll(tempDir)
}
cmd := exec.Command("bash", deployScriptPath,
"--file", manifestsFile,
"--name", name,
"--namespace", namespace,
"--certs-dir", certsDir)
output, err := cmd.CombinedOutput()
if err == nil {
e2elog.Logf("GMSA webhook successfully deployed, output:\n%s", string(output))
} else {
err = errors.Wrapf(err, "unable to deploy GMSA webhook, output:\n%s", string(output))
}
return cleanUpFunc, err
}
// createGmsaCustomResource creates the GMSA API object from the contents
// of the manifest file retrieved from the worker node.
// It returns a function to clean up both the temp file it creates and
// the API object it creates when done with testing.
func createGmsaCustomResource(crdManifestContents string) (func(), error) {
cleanUpFunc := func() {}
tempFile, err := ioutil.TempFile("", "")
if err != nil {
return cleanUpFunc, errors.Wrapf(err, "unable to create temp file")
}
defer tempFile.Close()
cleanUpFunc = func() {
framework.RunKubectl("delete", "--filename", tempFile.Name())
os.Remove(tempFile.Name())
}
_, err = tempFile.WriteString(crdManifestContents)
if err != nil {
err = errors.Wrapf(err, "unable to write GMSA contents to %q", tempFile.Name())
return cleanUpFunc, err
}
output, err := framework.RunKubectl("apply", "--filename", tempFile.Name())
if err != nil {
err = errors.Wrapf(err, "unable to create custom resource, output:\n%s", output)
}
return cleanUpFunc, err
}
// createRBACRoleForGmsa creates an RBAC cluster role to grant use
// access to our test credential spec.
// It returns the role's name, as well as a function to delete it when done.
func createRBACRoleForGmsa(f *framework.Framework) (string, func(), error) {
roleName := f.Namespace.Name + "-rbac-role"
role := &rbacv1.ClusterRole{
ObjectMeta: metav1.ObjectMeta{
Name: roleName,
},
Rules: []rbacv1.PolicyRule{
{
APIGroups: []string{"windows.k8s.io"},
Resources: []string{"gmsacredentialspecs"},
Verbs: []string{"use"},
ResourceNames: []string{gmsaCustomResourceName},
},
},
}
cleanUpFunc := func() {
f.ClientSet.RbacV1().ClusterRoles().Delete(roleName, &metav1.DeleteOptions{})
}
_, err := f.ClientSet.RbacV1().ClusterRoles().Create(role)
if err != nil {
err = errors.Wrapf(err, "unable to create RBAC cluster role %q", roleName)
}
return roleName, cleanUpFunc, err
}
// createServiceAccount creates a service account, and returns its name.
func createServiceAccount(f *framework.Framework) string {
accountName := f.Namespace.Name + "-sa"
account := &v1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: accountName,
Namespace: f.Namespace.Name,
},
}
if _, err := f.ClientSet.CoreV1().ServiceAccounts(f.Namespace.Name).Create(account); err != nil {
e2elog.Failf("unable to create service account %q: %v", accountName, err)
}
return accountName
}
// bindRBACRoleToServiceAccount binds the given RBAC cluster role to the given service account.
func bindRBACRoleToServiceAccount(f *framework.Framework, serviceAccountName, rbacRoleName string) {
binding := &rbacv1.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: f.Namespace.Name + "-rbac-binding",
Namespace: f.Namespace.Name,
},
Subjects: []rbacv1.Subject{
{
Kind: "ServiceAccount",
Name: serviceAccountName,
Namespace: f.Namespace.Name,
},
},
RoleRef: rbacv1.RoleRef{
APIGroup: "rbac.authorization.k8s.io",
Kind: "ClusterRole",
Name: rbacRoleName,
},
}
f.ClientSet.RbacV1().RoleBindings(f.Namespace.Name).Create(binding)
}
// createPodWithGmsa creates a pod using the test GMSA cred spec, and returns its name.
func createPodWithGmsa(f *framework.Framework, serviceAccountName string) string {
podName := "pod-with-gmsa"
credSpecName := gmsaCustomResourceName
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: podName,
Namespace: f.Namespace.Name,
},
Spec: v1.PodSpec{
ServiceAccountName: serviceAccountName,
Containers: []v1.Container{
{
Name: podName,
Image: imageutils.GetPauseImageName(),
},
},
SecurityContext: &v1.PodSecurityContext{
WindowsOptions: &v1.WindowsSecurityContextOptions{
GMSACredentialSpecName: &credSpecName,
},
},
},
}
f.PodClient().CreateSync(pod)
return podName
}
func runKubectlExecInNamespace(namespace string, args ...string) (string, error) {
namespaceOption := fmt.Sprintf("--namespace=%s", namespace)
return framework.RunKubectl(append([]string{"exec", namespaceOption}, args...)...)
}

13
test/e2e/windows/gmsa.go → test/e2e/windows/gmsa_kubelet.go
View File

@ -14,6 +14,11 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
// This test only makes sure that the kubelet correctly passes down GMSA cred specs
// down to the runtime.
// It's a much lighter test than gmsa_full, that tests the whole GMSA process, but
// also requires a heavy weight setup to run.
package windows
import (
@ -31,15 +36,15 @@ import (
"github.com/onsi/gomega"
)
var _ = SIGDescribe("[Feature:Windows] [Feature:WindowsGMSA] GMSA [Slow]", func() {
f := framework.NewDefaultFramework("gmsa-test-windows")
var _ = SIGDescribe("[Feature:Windows] [Feature:WindowsGMSA] GMSA Kubelet [Slow]", func() {
f := framework.NewDefaultFramework("gmsa-kubelet-test-windows")
ginkgo.Describe("kubelet GMSA support", func() {
ginkgo.Context("when creating a pod with correct GMSA credential specs", func() {
ginkgo.It("passes the credential specs down to the Pod's containers", func() {
defer ginkgo.GinkgoRecover()
podName := "with-correct-gmsa-annotations"
podName := "with-correct-gmsa-specs"
container1Name := "container1"
podDomain := "acme.com"
@ -75,7 +80,7 @@ var _ = SIGDescribe("[Feature:Windows] [Feature:WindowsGMSA] GMSA [Slow]", func(
},
}
ginkgo.By("creating a pod with correct GMSA annotations")
ginkgo.By("creating a pod with correct GMSA specs")
f.PodClient().CreateSync(pod)
ginkgo.By("checking the domain reported by nltest in the containers")

44
test/e2e/windows/utils.go Normal file
View File

@ -0,0 +1,44 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package windows
import (
"io"
"io/ioutil"
"net/http"
"github.com/pkg/errors"
)
// downloadFile saves a remote URL to a local temp file, and returns its path.
// It's the caller's responsibility to clean up the temp file when done.
func downloadFile(url string) (string, error) {
response, err := http.Get(url)
if err != nil {
return "", errors.Wrapf(err, "unable to download from %q", url)
}
defer response.Body.Close()
tempFile, err := ioutil.TempFile("", "")
if err != nil {
return "", errors.Wrapf(err, "unable to create temp file")
}
defer tempFile.Close()
_, err = io.Copy(tempFile, response.Body)
return tempFile.Name(), err
}