github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-21 19:01:49 +00:00
Code Issues Projects Releases Wiki Activity

Add e2e test to check for filesystem volume device mount cleanup

Browse Source
This commit is contained in:
Yecheng Fu 2019-10-23 14:40:14 +08:00
parent 36a54399a6
commit b536395c07
6 changed files with 160 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/util/mount/mount.go
View File

@ -48,7 +48,7 @@ type Interface interface {
// most notably linux bind mounts and symbolic link.
IsLikelyNotMountPoint(file string) (bool, error)
// GetMountRefs finds all mount references to the path, returns a
// list of paths. Path could be a mountpoint or a normal
// list of paths. Path could be a mountpoint path, device or a normal
// directory (for bind mount).
GetMountRefs(pathname string) ([]string, error)
}

5
pkg/util/mount/mount_linux.go
View File

@ -238,7 +238,7 @@ func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) {
}
// GetMountRefs finds all mount references to pathname, returns a
// list of paths. Path could be a mountpoint or a normal
// list of paths. Path could be a mountpoint path, device or a normal
// directory (for bind mount).
func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) {
pathExists, pathErr := PathExists(pathname)
@ -441,8 +441,7 @@ func parseProcMounts(content []byte) ([]MountPoint, error) {
// SearchMountPoints finds all mount references to the source, returns a list of
// mountpoints.
// The source can be a mount point or a normal directory (bind mount). We
// didn't support device because there is no use case by now.
// This function assumes source cannot be device.
// Some filesystems may share a source name, e.g. tmpfs. And for bind mounting,
// it's possible to mount a non-root path of a filesystem, so we need to use
// root path and major:minor to represent mount source uniquely.

32
test/e2e/storage/testsuites/subpath.go
View File

@ -157,24 +157,24 @@ func (s *subPathTestSuite) defineTests(driver TestDriver, pattern testpatterns.T
}
cleanup := func() {
if l.pod != nil {
ginkgo.By("Deleting pod")
err := e2epod.DeletePodWithWait(f.ClientSet, l.pod)
framework.ExpectNoError(err, "while deleting pod")
l.pod = nil
}
// if l.pod != nil {
// ginkgo.By("Deleting pod")
// err := e2epod.DeletePodWithWait(f.ClientSet, l.pod)
// framework.ExpectNoError(err, "while deleting pod")
// l.pod = nil
// }
if l.resource != nil {
l.resource.cleanupResource()
l.resource = nil
}
// if l.resource != nil {
// l.resource.cleanupResource()
// l.resource = nil
// }
if l.driverCleanup != nil {
l.driverCleanup()
l.driverCleanup = nil
}
// if l.driverCleanup != nil {
// l.driverCleanup()
// l.driverCleanup = nil
// }
validateMigrationVolumeOpCounts(f.ClientSet, driver.GetDriverInfo().InTreePluginName, l.intreeOps, l.migratedOps)
// validateMigrationVolumeOpCounts(f.ClientSet, driver.GetDriverInfo().InTreePluginName, l.intreeOps, l.migratedOps)
}
ginkgo.It("should support non-existent path", func() {
@ -902,7 +902,7 @@ func testSubpathReconstruction(f *framework.Framework, pod *v1.Pod, forceDelete
pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(pod.Name, metav1.GetOptions{})
framework.ExpectNoError(err, "while getting pod")
utils.TestVolumeUnmountsFromDeletedPodWithForceOption(f.ClientSet, f, pod, forceDelete, true)
utils.TestVolumeUnmountsFromDeletedPodWithForceOption(f.ClientSet, f, pod, forceDelete, true, true)
}
func formatVolume(f *framework.Framework, pod *v1.Pod) {

14
test/e2e/storage/utils/BUILD
View File

@ -1,9 +1,6 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
@ -16,6 +13,7 @@ go_library(
],
importpath = "k8s.io/kubernetes/test/e2e/storage/utils",
deps = [
"//pkg/util/mount:go_default_library",
"//staging/src/k8s.io/api/apps/v1:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/api/rbac/v1:go_default_library",
@ -23,6 +21,7 @@ go_library(
"//staging/src/k8s.io/api/storage/v1beta1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
@ -49,3 +48,10 @@ filegroup(
srcs = [":package-srcs"],
tags = ["automanaged"],
)
go_test(
name = "go_default_test",
srcs = ["utils_test.go"],
embed = [":go_default_library"],
deps = ["//vendor/github.com/onsi/gomega:go_default_library"],
)

78
test/e2e/storage/utils/utils.go
View File

@ -20,7 +20,9 @@ import (
"crypto/sha256"
"encoding/base64"
"fmt"
"io/ioutil"
"math/rand"
"os"
"path/filepath"
"strings"
"time"
@ -34,6 +36,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/pkg/util/mount"
"k8s.io/kubernetes/test/e2e/framework"
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
@ -234,9 +237,48 @@ func TestKubeletRestartsAndRestoresMap(c clientset.Interface, f *framework.Frame
framework.Logf("Volume map detected on pod %s and written data %s is readable post-restart.", clientPod.Name, path)
}
// findGlobalVolumeMountPaths finds all global volume mount paths for given pod from the host mount information.
// This function assumes:
// 1) pod volume mount paths exists in /var/lib/kubelet/pods/<pod-uid>/volumes/
// 2) global volume mount paths exists in /var/lib/kubelet/plugins/
func findGlobalVolumeMountPaths(mountInfo string, podUID string) ([]string, error) {
tmpfile, err := ioutil.TempFile("", "mountinfo")
if err != nil {
return nil, err
}
defer os.Remove(tmpfile.Name()) // clean up
err = ioutil.WriteFile(tmpfile.Name(), []byte(mountInfo), 0644)
if err != nil {
return nil, err
}
podVolumeMountBase := fmt.Sprintf("/var/lib/kubelet/pods/%s/volumes/", podUID)
globalVolumeMountBase := "/var/lib/kubelet/plugins"
mis, err := mount.ParseMountInfo(tmpfile.Name())
if err != nil {
return nil, err
}
globalVolumeMountPaths := []string{}
for _, mi := range mis {
if mount.PathWithinBase(mi.MountPoint, podVolumeMountBase) {
refs, err := mount.SearchMountPoints(mi.MountPoint, tmpfile.Name())
if err != nil {
return nil, err
}
for _, ref := range refs {
if mount.PathWithinBase(ref, globalVolumeMountBase) {
globalVolumeMountPaths = append(globalVolumeMountPaths, ref)
}
}
}
}
return globalVolumeMountPaths, nil
}
// TestVolumeUnmountsFromDeletedPodWithForceOption tests that a volume unmounts if the client pod was deleted while the kubelet was down.
// forceDelete is true indicating whether the pod is forcefully deleted.
func TestVolumeUnmountsFromDeletedPodWithForceOption(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, forceDelete bool, checkSubpath bool) {
// checkSubpath is true indicating whether the subpath should be checked.
// checkGlobalMount is true indicating whether the global mount should be checked.
func TestVolumeUnmountsFromDeletedPodWithForceOption(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, forceDelete bool, checkSubpath bool, checkGlobalMount bool) {
nodeIP, err := framework.GetHostAddress(c, clientPod)
framework.ExpectNoError(err)
nodeIP = nodeIP + ":22"
@ -255,6 +297,24 @@ func TestVolumeUnmountsFromDeletedPodWithForceOption(c clientset.Interface, f *f
gomega.Expect(result.Code).To(gomega.BeZero(), fmt.Sprintf("Expected grep exit code of 0, got %d", result.Code))
}
var globalVolumeMountPaths []string
if checkGlobalMount {
// Find global mount path and verify it will be unmounted later.
// We don't verify it must exist because:
// 1) not all volume types have global mount path, e.g. local filesystem volume with directory source
// 2) volume types which failed to mount global mount path will fail in other test
ginkgo.By("Find the volume global mount paths")
result, err = e2essh.SSH("cat /proc/self/mountinfo", nodeIP, framework.TestContext.Provider)
framework.ExpectNoError(err, "Encountered SSH error.")
globalVolumeMountPaths, err = findGlobalVolumeMountPaths(result.Stdout, string(clientPod.UID))
framework.ExpectNoError(err, fmt.Sprintf("Failed to get global volume mount paths: %v", err))
if len(globalVolumeMountPaths) > 0 {
framework.Logf("Volume global mount paths found at %v", globalVolumeMountPaths)
} else {
framework.Logf("No volume global mount paths found")
}
}
// This command is to make sure kubelet is started after test finishes no matter it fails or not.
defer func() {
KubeletCommand(KStart, c, clientPod)
@ -298,16 +358,28 @@ func TestVolumeUnmountsFromDeletedPodWithForceOption(c clientset.Interface, f *f
gomega.Expect(result.Stdout).To(gomega.BeEmpty(), "Expected grep stdout to be empty (i.e. no subpath mount found).")
framework.Logf("Subpath volume unmounted on node %s", clientPod.Spec.NodeName)
}
if checkGlobalMount && len(globalVolumeMountPaths) > 0 {
globalMountPathCmd := fmt.Sprintf("ls %s | grep '.'", strings.Join(globalVolumeMountPaths, " "))
if isSudoPresent(nodeIP, framework.TestContext.Provider) {
globalMountPathCmd = fmt.Sprintf("sudo sh -c \"%s\"", globalMountPathCmd)
}
ginkgo.By("Expecting the volume global mount path not to be found.")
result, err = e2essh.SSH(globalMountPathCmd, nodeIP, framework.TestContext.Provider)
e2essh.LogResult(result)
framework.ExpectNoError(err, "Encountered SSH error.")
gomega.Expect(result.Stdout).To(gomega.BeEmpty(), "Expected grep stdout to be empty.")
}
}
// TestVolumeUnmountsFromDeletedPod tests that a volume unmounts if the client pod was deleted while the kubelet was down.
func TestVolumeUnmountsFromDeletedPod(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod) {
TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, clientPod, false, false)
TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, clientPod, false, false, false)
}
// TestVolumeUnmountsFromForceDeletedPod tests that a volume unmounts if the client pod was forcefully deleted while the kubelet was down.
func TestVolumeUnmountsFromForceDeletedPod(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod) {
TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, clientPod, true, false)
TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, clientPod, true, false, false)
}
// TestVolumeUnmapsFromDeletedPodWithForceOption tests that a volume unmaps if the client pod was deleted while the kubelet was down.

56
test/e2e/storage/utils/utils_test.go Normal file
View File

@ -0,0 +1,56 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package utils
import (
"testing"
"github.com/onsi/gomega"
)
func TestFindGlobalVolumeMountPaths(t *testing.T) {
tests := []struct {
name string
mountInfo string
podUID string
expected []string
}{
{
name: "pod uses local filesystem pv with block source",
mountInfo: `1045 245 0:385 / /var/lib/kubelet/pods/ff5e9fa2-7111-486d-854c-848bcc6b3819/volumes/kubernetes.io~secret/default-token-djlt2 rw,relatime shared:199 - tmpfs tmpfs rw
1047 245 7:6 / /var/lib/kubelet/plugins/kubernetes.io/local-volume/mounts/local-wdx8b rw,relatime shared:200 - ext4 /dev/loop6 rw,data=ordered
1048 245 7:6 / /var/lib/kubelet/pods/ff5e9fa2-7111-486d-854c-848bcc6b3819/volumes/kubernetes.io~local-volume/local-wdx8b rw,relatime shared:200 - ext4 /dev/loop6 rw,data=ordered
1054 245 7:6 /provisioning-9823 /var/lib/kubelet/pods/ff5e9fa2-7111-486d-854c-848bcc6b3819/volume-subpaths/local-wdx8b/test-container-subpath-local-preprovisionedpv-d72p/0 rw,relatime shared:200 - ext4 /dev/loop6 rw,data=ordered
`,
podUID: "ff5e9fa2-7111-486d-854c-848bcc6b3819",
expected: []string{
"/var/lib/kubelet/plugins/kubernetes.io/local-volume/mounts/local-wdx8b",
},
},
}
g := gomega.NewWithT(t)
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
mountPaths, err := findGlobalVolumeMountPaths(tt.mountInfo, tt.podUID)
if err != nil {
t.Fatal(err)
}
g.Expect(mountPaths).To(gomega.ConsistOf(tt.expected))
})
}
}