From ff5558edc33f6042171ac3034e87e08926dab1f5 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 21 Feb 2022 15:45:15 +0100 Subject: [PATCH] test: support storage tests with non-standard kubelet root directory Some storage tests deploy DaemonSets which hard-code /var/lib/kubelet as root directory for kubelet registration and pod directory. There was already a parameter which allowed specifying the root directory, just with a very confusing name ("--volume-dir") and matching field name. A --kubelet-root-dir parameters gets added because this may make it easier to find the parameter, with the old name preserved as an alias for the same field for backwards compatibility. --- test/e2e/framework/test_context.go | 5 +++-- test/e2e/node/security_context.go | 4 ++-- test/e2e/storage/utils/deployment.go | 16 ++++++++++++++++ 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index 5c0c5fef15e..b4c4743b6ea 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -77,7 +77,7 @@ type TestContextType struct { KubeConfig string KubeContext string KubeAPIContentType string - KubeVolumeDir string + KubeletRootDir string CertDir string Host string BearerToken string `datapolicy:"token"` @@ -337,7 +337,8 @@ func RegisterClusterFlags(flags *flag.FlagSet) { flags.StringVar(&TestContext.KubeContext, clientcmd.FlagContext, "", "kubeconfig context to use/override. If unset, will use value from 'current-context'") flags.StringVar(&TestContext.KubeAPIContentType, "kube-api-content-type", "application/vnd.kubernetes.protobuf", "ContentType used to communicate with apiserver") - flags.StringVar(&TestContext.KubeVolumeDir, "volume-dir", "/var/lib/kubelet", "Path to the directory containing the kubelet volumes.") + flags.StringVar(&TestContext.KubeletRootDir, "kubelet-root-dir", "/var/lib/kubelet", "The data directory of kubelet. Some tests (for example, CSI storage tests) deploy DaemonSets which need to know this value and cannot query it. Such tests only work in clusters where the data directory is the same on all nodes.") + flags.StringVar(&TestContext.KubeletRootDir, "volume-dir", "/var/lib/kubelet", "An alias for --kubelet-root-dir, kept for backwards compatibility.") flags.StringVar(&TestContext.CertDir, "cert-dir", "", "Path to the directory containing the certs. Default is empty, which doesn't use certs.") flags.StringVar(&TestContext.RepoRoot, "repo-root", "../../", "Root directory of kubernetes repository, for finding test files.") // NOTE: Node E2E tests have this flag defined as well, but true by default. diff --git a/test/e2e/node/security_context.go b/test/e2e/node/security_context.go index 35075495c65..e5bd636049e 100644 --- a/test/e2e/node/security_context.go +++ b/test/e2e/node/security_context.go @@ -234,8 +234,8 @@ func testPodSELinuxLabeling(f *framework.Framework, hostIPC bool, hostPID bool) // Confirm that the file can be accessed from a second // pod using host_path with the same MCS label - volumeHostPath := fmt.Sprintf("%s/pods/%s/volumes/kubernetes.io~empty-dir/%s", framework.TestContext.KubeVolumeDir, foundPod.UID, volumeName) - ginkgo.By(fmt.Sprintf("confirming a container with the same label can read the file under --volume-dir=%s", framework.TestContext.KubeVolumeDir)) + volumeHostPath := fmt.Sprintf("%s/pods/%s/volumes/kubernetes.io~empty-dir/%s", framework.TestContext.KubeletRootDir, foundPod.UID, volumeName) + ginkgo.By(fmt.Sprintf("confirming a container with the same label can read the file under --kubelet-root-dir=%s", framework.TestContext.KubeletRootDir)) pod = scTestPod(hostIPC, hostPID) pod.Spec.NodeName = foundPod.Spec.NodeName volumeMounts := []v1.VolumeMount{ diff --git a/test/e2e/storage/utils/deployment.go b/test/e2e/storage/utils/deployment.go index e86812406cb..a0a964b1ba7 100644 --- a/test/e2e/storage/utils/deployment.go +++ b/test/e2e/storage/utils/deployment.go @@ -24,6 +24,7 @@ import ( v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" "k8s.io/kubernetes/test/e2e/framework" + e2eframework "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" ) @@ -52,6 +53,10 @@ func PatchCSIDeployment(f *framework.Framework, o PatchCSIOptions, object interf rename := o.OldDriverName != "" && o.NewDriverName != "" && o.OldDriverName != o.NewDriverName + substKubeletRootDir := func(s string) string { + return strings.ReplaceAll(s, "/var/lib/kubelet/", e2eframework.TestContext.KubeletRootDir+"/") + } + patchVolumes := func(volumes []v1.Volume) { if !rename { return @@ -65,6 +70,8 @@ func PatchCSIDeployment(f *framework.Framework, o PatchCSIOptions, object interf if file == o.OldDriverName { *p = path.Join(dir, o.NewDriverName) } + // Inject non-standard kubelet path. + *p = substKubeletRootDir(*p) } } } @@ -79,6 +86,15 @@ func PatchCSIDeployment(f *framework.Framework, o PatchCSIOptions, object interf container.Args[e] = strings.Replace(container.Args[e], "/"+o.OldDriverName+"/", "/"+o.NewDriverName+"/", 1) } } + + // Modify --kubelet-registration-path. + for e := range container.Args { + container.Args[e] = substKubeletRootDir(container.Args[e]) + } + for e := range container.VolumeMounts { + container.VolumeMounts[e].MountPath = substKubeletRootDir(container.VolumeMounts[e].MountPath) + } + // Overwrite driver name resp. provider name // by appending a parameter with the right // value.