From b536395c074cfe7951ed904bb6f8357edac0e4db Mon Sep 17 00:00:00 2001 From: Yecheng Fu Date: Wed, 23 Oct 2019 14:40:14 +0800 Subject: [PATCH] Add e2e test to check for filesystem volume device mount cleanup --- pkg/util/mount/mount.go | 2 +- pkg/util/mount/mount_linux.go | 5 +- test/e2e/storage/testsuites/subpath.go | 32 +++++------ test/e2e/storage/utils/BUILD | 14 +++-- test/e2e/storage/utils/utils.go | 78 +++++++++++++++++++++++++- test/e2e/storage/utils/utils_test.go | 56 ++++++++++++++++++ 6 files changed, 160 insertions(+), 27 deletions(-) create mode 100644 test/e2e/storage/utils/utils_test.go diff --git a/pkg/util/mount/mount.go b/pkg/util/mount/mount.go index 7d94a48f4b0..7324d691144 100644 --- a/pkg/util/mount/mount.go +++ b/pkg/util/mount/mount.go @@ -48,7 +48,7 @@ type Interface interface { // most notably linux bind mounts and symbolic link. IsLikelyNotMountPoint(file string) (bool, error) // GetMountRefs finds all mount references to the path, returns a - // list of paths. Path could be a mountpoint or a normal + // list of paths. Path could be a mountpoint path, device or a normal // directory (for bind mount). GetMountRefs(pathname string) ([]string, error) } diff --git a/pkg/util/mount/mount_linux.go b/pkg/util/mount/mount_linux.go index c0c4b0dd823..dfd52bfe105 100644 --- a/pkg/util/mount/mount_linux.go +++ b/pkg/util/mount/mount_linux.go @@ -238,7 +238,7 @@ func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) { } // GetMountRefs finds all mount references to pathname, returns a -// list of paths. Path could be a mountpoint or a normal +// list of paths. Path could be a mountpoint path, device or a normal // directory (for bind mount). func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { pathExists, pathErr := PathExists(pathname) @@ -441,8 +441,7 @@ func parseProcMounts(content []byte) ([]MountPoint, error) { // SearchMountPoints finds all mount references to the source, returns a list of // mountpoints. -// The source can be a mount point or a normal directory (bind mount). We -// didn't support device because there is no use case by now. +// This function assumes source cannot be device. // Some filesystems may share a source name, e.g. tmpfs. And for bind mounting, // it's possible to mount a non-root path of a filesystem, so we need to use // root path and major:minor to represent mount source uniquely. diff --git a/test/e2e/storage/testsuites/subpath.go b/test/e2e/storage/testsuites/subpath.go index d7d1eca86fe..cbee41ee78f 100644 --- a/test/e2e/storage/testsuites/subpath.go +++ b/test/e2e/storage/testsuites/subpath.go @@ -157,24 +157,24 @@ func (s *subPathTestSuite) defineTests(driver TestDriver, pattern testpatterns.T } cleanup := func() { - if l.pod != nil { - ginkgo.By("Deleting pod") - err := e2epod.DeletePodWithWait(f.ClientSet, l.pod) - framework.ExpectNoError(err, "while deleting pod") - l.pod = nil - } + // if l.pod != nil { + // ginkgo.By("Deleting pod") + // err := e2epod.DeletePodWithWait(f.ClientSet, l.pod) + // framework.ExpectNoError(err, "while deleting pod") + // l.pod = nil + // } - if l.resource != nil { - l.resource.cleanupResource() - l.resource = nil - } + // if l.resource != nil { + // l.resource.cleanupResource() + // l.resource = nil + // } - if l.driverCleanup != nil { - l.driverCleanup() - l.driverCleanup = nil - } + // if l.driverCleanup != nil { + // l.driverCleanup() + // l.driverCleanup = nil + // } - validateMigrationVolumeOpCounts(f.ClientSet, driver.GetDriverInfo().InTreePluginName, l.intreeOps, l.migratedOps) + // validateMigrationVolumeOpCounts(f.ClientSet, driver.GetDriverInfo().InTreePluginName, l.intreeOps, l.migratedOps) } ginkgo.It("should support non-existent path", func() { @@ -902,7 +902,7 @@ func testSubpathReconstruction(f *framework.Framework, pod *v1.Pod, forceDelete pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(pod.Name, metav1.GetOptions{}) framework.ExpectNoError(err, "while getting pod") - utils.TestVolumeUnmountsFromDeletedPodWithForceOption(f.ClientSet, f, pod, forceDelete, true) + utils.TestVolumeUnmountsFromDeletedPodWithForceOption(f.ClientSet, f, pod, forceDelete, true, true) } func formatVolume(f *framework.Framework, pod *v1.Pod) { diff --git a/test/e2e/storage/utils/BUILD b/test/e2e/storage/utils/BUILD index f06c92ddc41..b37922062d2 100644 --- a/test/e2e/storage/utils/BUILD +++ b/test/e2e/storage/utils/BUILD @@ -1,9 +1,6 @@ package(default_visibility = ["//visibility:public"]) -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", -) +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", @@ -16,6 +13,7 @@ go_library( ], importpath = "k8s.io/kubernetes/test/e2e/storage/utils", deps = [ + "//pkg/util/mount:go_default_library", "//staging/src/k8s.io/api/apps/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/rbac/v1:go_default_library", @@ -23,6 +21,7 @@ go_library( "//staging/src/k8s.io/api/storage/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", @@ -49,3 +48,10 @@ filegroup( srcs = [":package-srcs"], tags = ["automanaged"], ) + +go_test( + name = "go_default_test", + srcs = ["utils_test.go"], + embed = [":go_default_library"], + deps = ["//vendor/github.com/onsi/gomega:go_default_library"], +) diff --git a/test/e2e/storage/utils/utils.go b/test/e2e/storage/utils/utils.go index a02aafb45d5..fd758d14797 100644 --- a/test/e2e/storage/utils/utils.go +++ b/test/e2e/storage/utils/utils.go @@ -20,7 +20,9 @@ import ( "crypto/sha256" "encoding/base64" "fmt" + "io/ioutil" "math/rand" + "os" "path/filepath" "strings" "time" @@ -34,6 +36,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/test/e2e/framework" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" @@ -234,9 +237,48 @@ func TestKubeletRestartsAndRestoresMap(c clientset.Interface, f *framework.Frame framework.Logf("Volume map detected on pod %s and written data %s is readable post-restart.", clientPod.Name, path) } +// findGlobalVolumeMountPaths finds all global volume mount paths for given pod from the host mount information. +// This function assumes: +// 1) pod volume mount paths exists in /var/lib/kubelet/pods//volumes/ +// 2) global volume mount paths exists in /var/lib/kubelet/plugins/ +func findGlobalVolumeMountPaths(mountInfo string, podUID string) ([]string, error) { + tmpfile, err := ioutil.TempFile("", "mountinfo") + if err != nil { + return nil, err + } + defer os.Remove(tmpfile.Name()) // clean up + err = ioutil.WriteFile(tmpfile.Name(), []byte(mountInfo), 0644) + if err != nil { + return nil, err + } + podVolumeMountBase := fmt.Sprintf("/var/lib/kubelet/pods/%s/volumes/", podUID) + globalVolumeMountBase := "/var/lib/kubelet/plugins" + mis, err := mount.ParseMountInfo(tmpfile.Name()) + if err != nil { + return nil, err + } + globalVolumeMountPaths := []string{} + for _, mi := range mis { + if mount.PathWithinBase(mi.MountPoint, podVolumeMountBase) { + refs, err := mount.SearchMountPoints(mi.MountPoint, tmpfile.Name()) + if err != nil { + return nil, err + } + for _, ref := range refs { + if mount.PathWithinBase(ref, globalVolumeMountBase) { + globalVolumeMountPaths = append(globalVolumeMountPaths, ref) + } + } + } + } + return globalVolumeMountPaths, nil +} + // TestVolumeUnmountsFromDeletedPodWithForceOption tests that a volume unmounts if the client pod was deleted while the kubelet was down. // forceDelete is true indicating whether the pod is forcefully deleted. -func TestVolumeUnmountsFromDeletedPodWithForceOption(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, forceDelete bool, checkSubpath bool) { +// checkSubpath is true indicating whether the subpath should be checked. +// checkGlobalMount is true indicating whether the global mount should be checked. +func TestVolumeUnmountsFromDeletedPodWithForceOption(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, forceDelete bool, checkSubpath bool, checkGlobalMount bool) { nodeIP, err := framework.GetHostAddress(c, clientPod) framework.ExpectNoError(err) nodeIP = nodeIP + ":22" @@ -255,6 +297,24 @@ func TestVolumeUnmountsFromDeletedPodWithForceOption(c clientset.Interface, f *f gomega.Expect(result.Code).To(gomega.BeZero(), fmt.Sprintf("Expected grep exit code of 0, got %d", result.Code)) } + var globalVolumeMountPaths []string + if checkGlobalMount { + // Find global mount path and verify it will be unmounted later. + // We don't verify it must exist because: + // 1) not all volume types have global mount path, e.g. local filesystem volume with directory source + // 2) volume types which failed to mount global mount path will fail in other test + ginkgo.By("Find the volume global mount paths") + result, err = e2essh.SSH("cat /proc/self/mountinfo", nodeIP, framework.TestContext.Provider) + framework.ExpectNoError(err, "Encountered SSH error.") + globalVolumeMountPaths, err = findGlobalVolumeMountPaths(result.Stdout, string(clientPod.UID)) + framework.ExpectNoError(err, fmt.Sprintf("Failed to get global volume mount paths: %v", err)) + if len(globalVolumeMountPaths) > 0 { + framework.Logf("Volume global mount paths found at %v", globalVolumeMountPaths) + } else { + framework.Logf("No volume global mount paths found") + } + } + // This command is to make sure kubelet is started after test finishes no matter it fails or not. defer func() { KubeletCommand(KStart, c, clientPod) @@ -298,16 +358,28 @@ func TestVolumeUnmountsFromDeletedPodWithForceOption(c clientset.Interface, f *f gomega.Expect(result.Stdout).To(gomega.BeEmpty(), "Expected grep stdout to be empty (i.e. no subpath mount found).") framework.Logf("Subpath volume unmounted on node %s", clientPod.Spec.NodeName) } + + if checkGlobalMount && len(globalVolumeMountPaths) > 0 { + globalMountPathCmd := fmt.Sprintf("ls %s | grep '.'", strings.Join(globalVolumeMountPaths, " ")) + if isSudoPresent(nodeIP, framework.TestContext.Provider) { + globalMountPathCmd = fmt.Sprintf("sudo sh -c \"%s\"", globalMountPathCmd) + } + ginkgo.By("Expecting the volume global mount path not to be found.") + result, err = e2essh.SSH(globalMountPathCmd, nodeIP, framework.TestContext.Provider) + e2essh.LogResult(result) + framework.ExpectNoError(err, "Encountered SSH error.") + gomega.Expect(result.Stdout).To(gomega.BeEmpty(), "Expected grep stdout to be empty.") + } } // TestVolumeUnmountsFromDeletedPod tests that a volume unmounts if the client pod was deleted while the kubelet was down. func TestVolumeUnmountsFromDeletedPod(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod) { - TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, clientPod, false, false) + TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, clientPod, false, false, false) } // TestVolumeUnmountsFromForceDeletedPod tests that a volume unmounts if the client pod was forcefully deleted while the kubelet was down. func TestVolumeUnmountsFromForceDeletedPod(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod) { - TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, clientPod, true, false) + TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, clientPod, true, false, false) } // TestVolumeUnmapsFromDeletedPodWithForceOption tests that a volume unmaps if the client pod was deleted while the kubelet was down. diff --git a/test/e2e/storage/utils/utils_test.go b/test/e2e/storage/utils/utils_test.go new file mode 100644 index 00000000000..34b2ca1a854 --- /dev/null +++ b/test/e2e/storage/utils/utils_test.go @@ -0,0 +1,56 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "testing" + + "github.com/onsi/gomega" +) + +func TestFindGlobalVolumeMountPaths(t *testing.T) { + tests := []struct { + name string + mountInfo string + podUID string + expected []string + }{ + { + name: "pod uses local filesystem pv with block source", + mountInfo: `1045 245 0:385 / /var/lib/kubelet/pods/ff5e9fa2-7111-486d-854c-848bcc6b3819/volumes/kubernetes.io~secret/default-token-djlt2 rw,relatime shared:199 - tmpfs tmpfs rw +1047 245 7:6 / /var/lib/kubelet/plugins/kubernetes.io/local-volume/mounts/local-wdx8b rw,relatime shared:200 - ext4 /dev/loop6 rw,data=ordered +1048 245 7:6 / /var/lib/kubelet/pods/ff5e9fa2-7111-486d-854c-848bcc6b3819/volumes/kubernetes.io~local-volume/local-wdx8b rw,relatime shared:200 - ext4 /dev/loop6 rw,data=ordered +1054 245 7:6 /provisioning-9823 /var/lib/kubelet/pods/ff5e9fa2-7111-486d-854c-848bcc6b3819/volume-subpaths/local-wdx8b/test-container-subpath-local-preprovisionedpv-d72p/0 rw,relatime shared:200 - ext4 /dev/loop6 rw,data=ordered +`, + podUID: "ff5e9fa2-7111-486d-854c-848bcc6b3819", + expected: []string{ + "/var/lib/kubelet/plugins/kubernetes.io/local-volume/mounts/local-wdx8b", + }, + }, + } + + g := gomega.NewWithT(t) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mountPaths, err := findGlobalVolumeMountPaths(tt.mountInfo, tt.podUID) + if err != nil { + t.Fatal(err) + } + g.Expect(mountPaths).To(gomega.ConsistOf(tt.expected)) + }) + } +}