From c60f5e6dbc5cbcccfca180d100e82acf3faad71f Mon Sep 17 00:00:00 2001 From: Tomas Smetana Date: Thu, 9 Dec 2021 10:40:46 +0100 Subject: [PATCH] mount-utils: Detect potential stale file handle This is a fix for the issue #97031. It may happen that a subpath mount point on a networking FS gets deleted on the server but in the case of some file systems (CIFS) it would make the stat syscall end with ENOENT, preventing the mount point to be cleaned up and pods using the mount can't be cleanly deleted. This situation can be detected by calling also access syscall on the same path: if there is a discrepancy between stat and access results, treating the mount as corrupted allows for successful cleanup and unmount. --- .../k8s.io/mount-utils/mount_helper_common.go | 14 ---------- .../k8s.io/mount-utils/mount_helper_unix.go | 28 +++++++++++++++++++ .../mount-utils/mount_helper_windows.go | 14 ++++++++++ 3 files changed, 42 insertions(+), 14 deletions(-) diff --git a/staging/src/k8s.io/mount-utils/mount_helper_common.go b/staging/src/k8s.io/mount-utils/mount_helper_common.go index dd4dc2c8e5a..196ae30ad1a 100644 --- a/staging/src/k8s.io/mount-utils/mount_helper_common.go +++ b/staging/src/k8s.io/mount-utils/mount_helper_common.go @@ -135,17 +135,3 @@ func removePathIfNotMountPoint(mountPath string, mounter Interface, extensiveMou } return notMnt, nil } - -// PathExists returns true if the specified path exists. -// TODO: clean this up to use pkg/util/file/FileExists -func PathExists(path string) (bool, error) { - _, err := os.Stat(path) - if err == nil { - return true, nil - } else if os.IsNotExist(err) { - return false, nil - } else if IsCorruptedMnt(err) { - return true, err - } - return false, err -} diff --git a/staging/src/k8s.io/mount-utils/mount_helper_unix.go b/staging/src/k8s.io/mount-utils/mount_helper_unix.go index 77fb817db08..013c688a2a6 100644 --- a/staging/src/k8s.io/mount-utils/mount_helper_unix.go +++ b/staging/src/k8s.io/mount-utils/mount_helper_unix.go @@ -20,12 +20,15 @@ limitations under the License. package mount import ( + "errors" "fmt" + "io/fs" "os" "strconv" "strings" "syscall" + "k8s.io/klog/v2" utilio "k8s.io/utils/io" ) @@ -51,6 +54,8 @@ func IsCorruptedMnt(err error) bool { underlyingError = pe.Err case *os.SyscallError: underlyingError = pe.Err + case syscall.Errno: + underlyingError = err } return underlyingError == syscall.ENOTCONN || underlyingError == syscall.ESTALE || underlyingError == syscall.EIO || underlyingError == syscall.EACCES || underlyingError == syscall.EHOSTDOWN @@ -157,3 +162,26 @@ func isMountPointMatch(mp MountPoint, dir string) bool { deletedDir := fmt.Sprintf("%s\\040(deleted)", dir) return ((mp.Path == dir) || (mp.Path == deletedDir)) } + +// PathExists returns true if the specified path exists. +// TODO: clean this up to use pkg/util/file/FileExists +func PathExists(path string) (bool, error) { + _, err := os.Stat(path) + if err == nil { + return true, nil + } else if errors.Is(err, fs.ErrNotExist) { + err = syscall.Access(path, syscall.F_OK) + if err == nil { + // The access syscall says the file exists, the stat syscall says it + // doesn't. This was observed on CIFS when the path was removed at + // the server somehow. POSIX calls this a stale file handle, let's fake + // that error and treat the path as existing but corrupted. + klog.Warningf("Potential stale file handle detected: %s", path) + return true, syscall.ESTALE + } + return false, nil + } else if IsCorruptedMnt(err) { + return true, err + } + return false, err +} diff --git a/staging/src/k8s.io/mount-utils/mount_helper_windows.go b/staging/src/k8s.io/mount-utils/mount_helper_windows.go index 7653062f65f..995fd5a0cd7 100644 --- a/staging/src/k8s.io/mount-utils/mount_helper_windows.go +++ b/staging/src/k8s.io/mount-utils/mount_helper_windows.go @@ -95,3 +95,17 @@ func ValidateDiskNumber(disk string) error { func isMountPointMatch(mp MountPoint, dir string) bool { return mp.Path == dir } + +// PathExists returns true if the specified path exists. +// TODO: clean this up to use pkg/util/file/FileExists +func PathExists(path string) (bool, error) { + _, err := os.Stat(path) + if err == nil { + return true, nil + } else if os.IsNotExist(err) { + return false, nil + } else if IsCorruptedMnt(err) { + return true, err + } + return false, err +}