diff --git a/test/e2e/storage/testsuites/subpath.go b/test/e2e/storage/testsuites/subpath.go index cbee41ee78f..80a1670096a 100644 --- a/test/e2e/storage/testsuites/subpath.go +++ b/test/e2e/storage/testsuites/subpath.go @@ -36,6 +36,7 @@ import ( "time" "github.com/onsi/ginkgo" + "github.com/onsi/gomega" ) var ( @@ -157,24 +158,24 @@ func (s *subPathTestSuite) defineTests(driver TestDriver, pattern testpatterns.T } cleanup := func() { - // if l.pod != nil { - // ginkgo.By("Deleting pod") - // err := e2epod.DeletePodWithWait(f.ClientSet, l.pod) - // framework.ExpectNoError(err, "while deleting pod") - // l.pod = nil - // } + if l.pod != nil { + ginkgo.By("Deleting pod") + err := e2epod.DeletePodWithWait(f.ClientSet, l.pod) + framework.ExpectNoError(err, "while deleting pod") + l.pod = nil + } - // if l.resource != nil { - // l.resource.cleanupResource() - // l.resource = nil - // } + if l.resource != nil { + l.resource.cleanupResource() + l.resource = nil + } - // if l.driverCleanup != nil { - // l.driverCleanup() - // l.driverCleanup = nil - // } + if l.driverCleanup != nil { + l.driverCleanup() + l.driverCleanup = nil + } - // validateMigrationVolumeOpCounts(f.ClientSet, driver.GetDriverInfo().InTreePluginName, l.intreeOps, l.migratedOps) + validateMigrationVolumeOpCounts(f.ClientSet, driver.GetDriverInfo().InTreePluginName, l.intreeOps, l.migratedOps) } ginkgo.It("should support non-existent path", func() { @@ -880,6 +881,10 @@ func testPodContainerRestart(f *framework.Framework, pod *v1.Pod) { func testSubpathReconstruction(f *framework.Framework, pod *v1.Pod, forceDelete bool) { // This is mostly copied from TestVolumeUnmountsFromDeletedPodWithForceOption() + // Disruptive test run serially, we can cache all voluem global mount + // points and verify after the test that we do not leak any global mount point. + mountPoints := utils.FindVolumeGlobalMountPoints(f.ClientSet, pod) + // Change to busybox pod.Spec.Containers[0].Image = volume.GetTestImage(imageutils.GetE2EImage(imageutils.BusyBox)) pod.Spec.Containers[0].Command = volume.GenerateScriptCmd("sleep 100000") @@ -902,7 +907,10 @@ func testSubpathReconstruction(f *framework.Framework, pod *v1.Pod, forceDelete pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(pod.Name, metav1.GetOptions{}) framework.ExpectNoError(err, "while getting pod") - utils.TestVolumeUnmountsFromDeletedPodWithForceOption(f.ClientSet, f, pod, forceDelete, true, true) + utils.TestVolumeUnmountsFromDeletedPodWithForceOption(f.ClientSet, f, pod, forceDelete, true) + + mountPointsAfter := utils.FindVolumeGlobalMountPoints(f.ClientSet, pod) + gomega.Expect(mountPointsAfter).To(gomega.ConsistOf(mountPoints), "Global mount points leaked. Before: %v, After: %v.", mountPoints, mountPointsAfter) } func formatVolume(f *framework.Framework, pod *v1.Pod) { diff --git a/test/e2e/storage/utils/BUILD b/test/e2e/storage/utils/BUILD index b37922062d2..9583069d0ac 100644 --- a/test/e2e/storage/utils/BUILD +++ b/test/e2e/storage/utils/BUILD @@ -1,6 +1,6 @@ package(default_visibility = ["//visibility:public"]) -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") +load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", @@ -13,7 +13,6 @@ go_library( ], importpath = "k8s.io/kubernetes/test/e2e/storage/utils", deps = [ - "//pkg/util/mount:go_default_library", "//staging/src/k8s.io/api/apps/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/rbac/v1:go_default_library", @@ -48,10 +47,3 @@ filegroup( srcs = [":package-srcs"], tags = ["automanaged"], ) - -go_test( - name = "go_default_test", - srcs = ["utils_test.go"], - embed = [":go_default_library"], - deps = ["//vendor/github.com/onsi/gomega:go_default_library"], -) diff --git a/test/e2e/storage/utils/utils.go b/test/e2e/storage/utils/utils.go index fd758d14797..d4e3767df34 100644 --- a/test/e2e/storage/utils/utils.go +++ b/test/e2e/storage/utils/utils.go @@ -20,9 +20,7 @@ import ( "crypto/sha256" "encoding/base64" "fmt" - "io/ioutil" "math/rand" - "os" "path/filepath" "strings" "time" @@ -36,7 +34,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" - "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/test/e2e/framework" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" @@ -237,48 +234,10 @@ func TestKubeletRestartsAndRestoresMap(c clientset.Interface, f *framework.Frame framework.Logf("Volume map detected on pod %s and written data %s is readable post-restart.", clientPod.Name, path) } -// findGlobalVolumeMountPaths finds all global volume mount paths for given pod from the host mount information. -// This function assumes: -// 1) pod volume mount paths exists in /var/lib/kubelet/pods//volumes/ -// 2) global volume mount paths exists in /var/lib/kubelet/plugins/ -func findGlobalVolumeMountPaths(mountInfo string, podUID string) ([]string, error) { - tmpfile, err := ioutil.TempFile("", "mountinfo") - if err != nil { - return nil, err - } - defer os.Remove(tmpfile.Name()) // clean up - err = ioutil.WriteFile(tmpfile.Name(), []byte(mountInfo), 0644) - if err != nil { - return nil, err - } - podVolumeMountBase := fmt.Sprintf("/var/lib/kubelet/pods/%s/volumes/", podUID) - globalVolumeMountBase := "/var/lib/kubelet/plugins" - mis, err := mount.ParseMountInfo(tmpfile.Name()) - if err != nil { - return nil, err - } - globalVolumeMountPaths := []string{} - for _, mi := range mis { - if mount.PathWithinBase(mi.MountPoint, podVolumeMountBase) { - refs, err := mount.SearchMountPoints(mi.MountPoint, tmpfile.Name()) - if err != nil { - return nil, err - } - for _, ref := range refs { - if mount.PathWithinBase(ref, globalVolumeMountBase) { - globalVolumeMountPaths = append(globalVolumeMountPaths, ref) - } - } - } - } - return globalVolumeMountPaths, nil -} - // TestVolumeUnmountsFromDeletedPodWithForceOption tests that a volume unmounts if the client pod was deleted while the kubelet was down. // forceDelete is true indicating whether the pod is forcefully deleted. // checkSubpath is true indicating whether the subpath should be checked. -// checkGlobalMount is true indicating whether the global mount should be checked. -func TestVolumeUnmountsFromDeletedPodWithForceOption(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, forceDelete bool, checkSubpath bool, checkGlobalMount bool) { +func TestVolumeUnmountsFromDeletedPodWithForceOption(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, forceDelete bool, checkSubpath bool) { nodeIP, err := framework.GetHostAddress(c, clientPod) framework.ExpectNoError(err) nodeIP = nodeIP + ":22" @@ -297,24 +256,6 @@ func TestVolumeUnmountsFromDeletedPodWithForceOption(c clientset.Interface, f *f gomega.Expect(result.Code).To(gomega.BeZero(), fmt.Sprintf("Expected grep exit code of 0, got %d", result.Code)) } - var globalVolumeMountPaths []string - if checkGlobalMount { - // Find global mount path and verify it will be unmounted later. - // We don't verify it must exist because: - // 1) not all volume types have global mount path, e.g. local filesystem volume with directory source - // 2) volume types which failed to mount global mount path will fail in other test - ginkgo.By("Find the volume global mount paths") - result, err = e2essh.SSH("cat /proc/self/mountinfo", nodeIP, framework.TestContext.Provider) - framework.ExpectNoError(err, "Encountered SSH error.") - globalVolumeMountPaths, err = findGlobalVolumeMountPaths(result.Stdout, string(clientPod.UID)) - framework.ExpectNoError(err, fmt.Sprintf("Failed to get global volume mount paths: %v", err)) - if len(globalVolumeMountPaths) > 0 { - framework.Logf("Volume global mount paths found at %v", globalVolumeMountPaths) - } else { - framework.Logf("No volume global mount paths found") - } - } - // This command is to make sure kubelet is started after test finishes no matter it fails or not. defer func() { KubeletCommand(KStart, c, clientPod) @@ -358,28 +299,16 @@ func TestVolumeUnmountsFromDeletedPodWithForceOption(c clientset.Interface, f *f gomega.Expect(result.Stdout).To(gomega.BeEmpty(), "Expected grep stdout to be empty (i.e. no subpath mount found).") framework.Logf("Subpath volume unmounted on node %s", clientPod.Spec.NodeName) } - - if checkGlobalMount && len(globalVolumeMountPaths) > 0 { - globalMountPathCmd := fmt.Sprintf("ls %s | grep '.'", strings.Join(globalVolumeMountPaths, " ")) - if isSudoPresent(nodeIP, framework.TestContext.Provider) { - globalMountPathCmd = fmt.Sprintf("sudo sh -c \"%s\"", globalMountPathCmd) - } - ginkgo.By("Expecting the volume global mount path not to be found.") - result, err = e2essh.SSH(globalMountPathCmd, nodeIP, framework.TestContext.Provider) - e2essh.LogResult(result) - framework.ExpectNoError(err, "Encountered SSH error.") - gomega.Expect(result.Stdout).To(gomega.BeEmpty(), "Expected grep stdout to be empty.") - } } // TestVolumeUnmountsFromDeletedPod tests that a volume unmounts if the client pod was deleted while the kubelet was down. func TestVolumeUnmountsFromDeletedPod(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod) { - TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, clientPod, false, false, false) + TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, clientPod, false, false) } // TestVolumeUnmountsFromForceDeletedPod tests that a volume unmounts if the client pod was forcefully deleted while the kubelet was down. func TestVolumeUnmountsFromForceDeletedPod(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod) { - TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, clientPod, true, false, false) + TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, clientPod, true, false) } // TestVolumeUnmapsFromDeletedPodWithForceOption tests that a volume unmaps if the client pod was deleted while the kubelet was down. @@ -719,3 +648,28 @@ func CheckWriteToPath(pod *v1.Pod, volMode v1.PersistentVolumeMode, path string, VerifyExecInPodSucceed(pod, fmt.Sprintf("echo %s | base64 -d | sha256sum", encoded)) VerifyExecInPodSucceed(pod, fmt.Sprintf("echo %s | base64 -d | dd of=%s bs=%d count=1", encoded, pathForVolMode, len)) } + +// findMountPoints returns all mount points on given node under specified directory. +func findMountPoints(nodeIP string, dir string) []string { + result, err := e2essh.SSH(fmt.Sprintf(`find %s -type d -exec mountpoint {} \; | grep 'is a mountpoint$'`, dir), nodeIP, framework.TestContext.Provider) + e2essh.LogResult(result) + framework.ExpectNoError(err, "Encountered SSH error.") + var mountPoints []string + if err != nil { + for _, line := range strings.Split(result.Stdout, "\n") { + if line == "" { + continue + } + mountPoints = append(mountPoints, strings.TrimSuffix(line, " is a mountpoint")) + } + } + return mountPoints +} + +// FindVolumeGlobalMountPoints returns all volume global mount points on the node of given pod. +func FindVolumeGlobalMountPoints(c clientset.Interface, pod *v1.Pod) []string { + nodeIP, err := framework.GetHostAddress(c, pod) + framework.ExpectNoError(err) + nodeIP = nodeIP + ":22" + return findMountPoints(nodeIP, "/var/lib/kubelet/plugins") +} diff --git a/test/e2e/storage/utils/utils_test.go b/test/e2e/storage/utils/utils_test.go deleted file mode 100644 index 34b2ca1a854..00000000000 --- a/test/e2e/storage/utils/utils_test.go +++ /dev/null @@ -1,56 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package utils - -import ( - "testing" - - "github.com/onsi/gomega" -) - -func TestFindGlobalVolumeMountPaths(t *testing.T) { - tests := []struct { - name string - mountInfo string - podUID string - expected []string - }{ - { - name: "pod uses local filesystem pv with block source", - mountInfo: `1045 245 0:385 / /var/lib/kubelet/pods/ff5e9fa2-7111-486d-854c-848bcc6b3819/volumes/kubernetes.io~secret/default-token-djlt2 rw,relatime shared:199 - tmpfs tmpfs rw -1047 245 7:6 / /var/lib/kubelet/plugins/kubernetes.io/local-volume/mounts/local-wdx8b rw,relatime shared:200 - ext4 /dev/loop6 rw,data=ordered -1048 245 7:6 / /var/lib/kubelet/pods/ff5e9fa2-7111-486d-854c-848bcc6b3819/volumes/kubernetes.io~local-volume/local-wdx8b rw,relatime shared:200 - ext4 /dev/loop6 rw,data=ordered -1054 245 7:6 /provisioning-9823 /var/lib/kubelet/pods/ff5e9fa2-7111-486d-854c-848bcc6b3819/volume-subpaths/local-wdx8b/test-container-subpath-local-preprovisionedpv-d72p/0 rw,relatime shared:200 - ext4 /dev/loop6 rw,data=ordered -`, - podUID: "ff5e9fa2-7111-486d-854c-848bcc6b3819", - expected: []string{ - "/var/lib/kubelet/plugins/kubernetes.io/local-volume/mounts/local-wdx8b", - }, - }, - } - - g := gomega.NewWithT(t) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - mountPaths, err := findGlobalVolumeMountPaths(tt.mountInfo, tt.podUID) - if err != nil { - t.Fatal(err) - } - g.Expect(mountPaths).To(gomega.ConsistOf(tt.expected)) - }) - } -}