github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-11-13 06:04:02 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #74393 from codenrhoden/refactor-subpath

Browse Source 
Refactor subpath out of pkg/util/mount
This commit is contained in:
Kubernetes Prow Robot
2019-02-27 15:54:36 -08:00
committed by GitHub
parent bf2c9ccec7 2c4d748bed
commit 84dce4d119
48 changed files with 3382 additions and 2822 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/util/mount/BUILD
View File

@@ -80,8 +80,6 @@ go_test(
"//vendor/k8s.io/utils/exec/testing:go_default_library",
] + select({
"@io_bazel_rules_go//go/platform:linux": [
"//vendor/golang.org/x/sys/unix:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library",
"//vendor/k8s.io/utils/nsenter:go_default_library",
],

12
pkg/util/mount/exec_mount.go
View File

@@ -144,18 +144,6 @@ func (m *execMounter) EvalHostSymlinks(pathname string) (string, error) {
return m.wrappedMounter.EvalHostSymlinks(pathname)
}
func (m *execMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return m.wrappedMounter.PrepareSafeSubpath(subPath)
}
func (m *execMounter) CleanSubPaths(podDir string, volumeName string) error {
return m.wrappedMounter.CleanSubPaths(podDir, volumeName)
}
func (m *execMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return m.wrappedMounter.SafeMakeDir(pathname, base, perm)
}
func (m *execMounter) GetMountRefs(pathname string) ([]string, error) {
return m.wrappedMounter.GetMountRefs(pathname)
}

12
pkg/util/mount/exec_mount_unsupported.go
View File

@@ -91,18 +91,6 @@ func (m *execMounter) EvalHostSymlinks(pathname string) (string, error) {
return "", errors.New("not implemented")
}
func (mounter *execMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return subPath.Path, nil, nil
}
func (mounter *execMounter) CleanSubPaths(podDir string, volumeName string) error {
return nil
}
func (mounter *execMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return nil
}
func (mounter *execMounter) GetMountRefs(pathname string) ([]string, error) {
return nil, errors.New("not implemented")
}

11
pkg/util/mount/fake.go
View File

@@ -220,17 +220,6 @@ func (f *FakeMounter) EvalHostSymlinks(pathname string) (string, error) {
return pathname, nil
}
func (f *FakeMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return subPath.Path, nil, nil
}
func (f *FakeMounter) CleanSubPaths(podDir string, volumeName string) error {
return nil
}
func (mounter *FakeMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return nil
}
func (f *FakeMounter) GetMountRefs(pathname string) ([]string, error) {
realpath, err := filepath.EvalSymlinks(pathname)
if err != nil {

29
pkg/util/mount/mount.go
View File

@@ -84,35 +84,12 @@ type Interface interface {
// MakeDir creates a new directory.
// Will operate in the host mount namespace if kubelet is running in a container
MakeDir(pathname string) error
// SafeMakeDir creates subdir within given base. It makes sure that the
// created directory does not escape given base directory mis-using
// symlinks. Note that the function makes sure that it creates the directory
// somewhere under the base, nothing else. E.g. if the directory already
// exists, it may exist outside of the base due to symlinks.
// This method should be used if the directory to create is inside volume
// that's under user control. User must not be able to use symlinks to
// escape the volume to create directories somewhere else.
SafeMakeDir(subdir string, base string, perm os.FileMode) error
// Will operate in the host mount namespace if kubelet is running in a container.
// Error is returned on any other error than "file not found".
ExistsPath(pathname string) (bool, error)
// EvalHostSymlinks returns the path name after evaluating symlinks.
// Will operate in the host mount namespace if kubelet is running in a container.
EvalHostSymlinks(pathname string) (string, error)
// CleanSubPaths removes any bind-mounts created by PrepareSafeSubpath in given
// pod volume directory.
CleanSubPaths(podDir string, volumeName string) error
// PrepareSafeSubpath does everything that's necessary to prepare a subPath
// that's 1) inside given volumePath and 2) immutable after this call.
//
// newHostPath - location of prepared subPath. It should be used instead of
// hostName when running the container.
// cleanupAction - action to run when the container is running or it failed to start.
//
// CleanupAction must be called immediately after the container with given
// subpath starts. On the other hand, Interface.CleanSubPaths must be called
// when the pod finishes.
PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error)
// GetMountRefs finds all mount references to the path, returns a
// list of paths. Path could be a mountpoint path, device or a normal
// directory (for bind mount).
@@ -355,15 +332,15 @@ func PathWithinBase(fullPath, basePath string) bool {
if err != nil {
return false
}
if startsWithBackstep(rel) {
if StartsWithBackstep(rel) {
// Needed to escape the base path
return false
}
return true
}
// startsWithBackstep checks if the given path starts with a backstep segment
func startsWithBackstep(rel string) bool {
// StartsWithBackstep checks if the given path starts with a backstep segment
func StartsWithBackstep(rel string) bool {
// normalize to / and check for ../
return rel == ".." || strings.HasPrefix(filepath.ToSlash(rel), "../")
}

516
pkg/util/mount/mount_linux.go
View File

@@ -21,7 +21,6 @@ package mount
import (
"errors"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path"
@@ -53,14 +52,6 @@ const (
fsckErrorsCorrected = 1
// 'fsck' found errors but exited without correcting them
fsckErrorsUncorrected = 4
// place for subpath mounts
// TODO: pass in directory using kubelet_getters instead
containerSubPathDirectoryName = "volume-subpaths"
// syscall.Openat flags used to traverse directories not following symlinks
nofollowFlags = unix.O_RDONLY | unix.O_NOFOLLOW
// flags for getting file descriptor without following the symlink
openFDFlags = unix.O_NOFOLLOW | unix.O_PATH
)
// Mounter provides the default implementation of mount.Interface
@@ -726,282 +717,6 @@ func getSELinuxSupport(path string, mountInfoFilename string) (bool, error) {
return false, nil
}
func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
newHostPath, err = doBindSubPath(mounter, subPath)
// There is no action when the container starts. Bind-mount will be cleaned
// when container stops by CleanSubPaths.
cleanupAction = nil
return newHostPath, cleanupAction, err
}
// This implementation is shared between Linux and NsEnterMounter
func safeOpenSubPath(mounter Interface, subpath Subpath) (int, error) {
if !PathWithinBase(subpath.Path, subpath.VolumePath) {
return -1, fmt.Errorf("subpath %q not within volume path %q", subpath.Path, subpath.VolumePath)
}
fd, err := doSafeOpen(subpath.Path, subpath.VolumePath)
if err != nil {
return -1, fmt.Errorf("error opening subpath %v: %v", subpath.Path, err)
}
return fd, nil
}
// prepareSubpathTarget creates target for bind-mount of subpath. It returns
// "true" when the target already exists and something is mounted there.
// Given Subpath must have all paths with already resolved symlinks and with
// paths relevant to kubelet (when it runs in a container).
// This function is called also by NsEnterMounter. It works because
// /var/lib/kubelet is mounted from the host into the container with Kubelet as
// /var/lib/kubelet too.
func prepareSubpathTarget(mounter Interface, subpath Subpath) (bool, string, error) {
// Early check for already bind-mounted subpath.
bindPathTarget := getSubpathBindTarget(subpath)
notMount, err := mounter.IsNotMountPoint(bindPathTarget)
if err != nil {
if !os.IsNotExist(err) {
return false, "", fmt.Errorf("error checking path %s for mount: %s", bindPathTarget, err)
}
// Ignore ErrorNotExist: the file/directory will be created below if it does not exist yet.
notMount = true
}
if !notMount {
// It's already mounted
klog.V(5).Infof("Skipping bind-mounting subpath %s: already mounted", bindPathTarget)
return true, bindPathTarget, nil
}
// bindPathTarget is in /var/lib/kubelet and thus reachable without any
// translation even to containerized kubelet.
bindParent := filepath.Dir(bindPathTarget)
err = os.MkdirAll(bindParent, 0750)
if err != nil && !os.IsExist(err) {
return false, "", fmt.Errorf("error creating directory %s: %s", bindParent, err)
}
t, err := os.Lstat(subpath.Path)
if err != nil {
return false, "", fmt.Errorf("lstat %s failed: %s", subpath.Path, err)
}
if t.Mode()&os.ModeDir > 0 {
if err = os.Mkdir(bindPathTarget, 0750); err != nil && !os.IsExist(err) {
return false, "", fmt.Errorf("error creating directory %s: %s", bindPathTarget, err)
}
} else {
// "/bin/touch <bindPathTarget>".
// A file is enough for all possible targets (symlink, device, pipe,
// socket, ...), bind-mounting them into a file correctly changes type
// of the target file.
if err = ioutil.WriteFile(bindPathTarget, []byte{}, 0640); err != nil {
return false, "", fmt.Errorf("error creating file %s: %s", bindPathTarget, err)
}
}
return false, bindPathTarget, nil
}
func getSubpathBindTarget(subpath Subpath) string {
// containerName is DNS label, i.e. safe as a directory name.
return filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName, strconv.Itoa(subpath.VolumeMountIndex))
}
func doBindSubPath(mounter Interface, subpath Subpath) (hostPath string, err error) {
// Linux, kubelet runs on the host:
// - safely open the subpath
// - bind-mount /proc/<pid of kubelet>/fd/<fd> to subpath target
// User can't change /proc/<pid of kubelet>/fd/<fd> to point to a bad place.
// Evaluate all symlinks here once for all subsequent functions.
newVolumePath, err := filepath.EvalSymlinks(subpath.VolumePath)
if err != nil {
return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err)
}
newPath, err := filepath.EvalSymlinks(subpath.Path)
if err != nil {
return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err)
}
klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, newPath, subpath.VolumePath)
subpath.VolumePath = newVolumePath
subpath.Path = newPath
fd, err := safeOpenSubPath(mounter, subpath)
if err != nil {
return "", err
}
defer syscall.Close(fd)
alreadyMounted, bindPathTarget, err := prepareSubpathTarget(mounter, subpath)
if err != nil {
return "", err
}
if alreadyMounted {
return bindPathTarget, nil
}
success := false
defer func() {
// Cleanup subpath on error
if !success {
klog.V(4).Infof("doBindSubPath() failed for %q, cleaning up subpath", bindPathTarget)
if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil {
klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr)
}
}
}()
kubeletPid := os.Getpid()
mountSource := fmt.Sprintf("/proc/%d/fd/%v", kubeletPid, fd)
// Do the bind mount
options := []string{"bind"}
klog.V(5).Infof("bind mounting %q at %q", mountSource, bindPathTarget)
if err = mounter.Mount(mountSource, bindPathTarget, "" /*fstype*/, options); err != nil {
return "", fmt.Errorf("error mounting %s: %s", subpath.Path, err)
}
success = true
klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget)
return bindPathTarget, nil
}
func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error {
return doCleanSubPaths(mounter, podDir, volumeName)
}
// This implementation is shared between Linux and NsEnterMounter
func doCleanSubPaths(mounter Interface, podDir string, volumeName string) error {
// scan /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/*
subPathDir := filepath.Join(podDir, containerSubPathDirectoryName, volumeName)
klog.V(4).Infof("Cleaning up subpath mounts for %s", subPathDir)
containerDirs, err := ioutil.ReadDir(subPathDir)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return fmt.Errorf("error reading %s: %s", subPathDir, err)
}
for _, containerDir := range containerDirs {
if !containerDir.IsDir() {
klog.V(4).Infof("Container file is not a directory: %s", containerDir.Name())
continue
}
klog.V(4).Infof("Cleaning up subpath mounts for container %s", containerDir.Name())
// scan /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/<container name>/*
fullContainerDirPath := filepath.Join(subPathDir, containerDir.Name())
err = filepath.Walk(fullContainerDirPath, func(path string, info os.FileInfo, err error) error {
if path == fullContainerDirPath {
// Skip top level directory
return nil
}
// pass through errors and let doCleanSubPath handle them
if err = doCleanSubPath(mounter, fullContainerDirPath, filepath.Base(path)); err != nil {
return err
}
return nil
})
if err != nil {
return fmt.Errorf("error processing %s: %s", fullContainerDirPath, err)
}
// Whole container has been processed, remove its directory.
if err := os.Remove(fullContainerDirPath); err != nil {
return fmt.Errorf("error deleting %s: %s", fullContainerDirPath, err)
}
klog.V(5).Infof("Removed %s", fullContainerDirPath)
}
// Whole pod volume subpaths have been cleaned up, remove its subpath directory.
if err := os.Remove(subPathDir); err != nil {
return fmt.Errorf("error deleting %s: %s", subPathDir, err)
}
klog.V(5).Infof("Removed %s", subPathDir)
// Remove entire subpath directory if it's the last one
podSubPathDir := filepath.Join(podDir, containerSubPathDirectoryName)
if err := os.Remove(podSubPathDir); err != nil && !os.IsExist(err) {
return fmt.Errorf("error deleting %s: %s", podSubPathDir, err)
}
klog.V(5).Infof("Removed %s", podSubPathDir)
return nil
}
// doCleanSubPath tears down the single subpath bind mount
func doCleanSubPath(mounter Interface, fullContainerDirPath, subPathIndex string) error {
// process /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/<container name>/<subPathName>
klog.V(4).Infof("Cleaning up subpath mounts for subpath %v", subPathIndex)
fullSubPath := filepath.Join(fullContainerDirPath, subPathIndex)
if err := CleanupMountPoint(fullSubPath, mounter, true); err != nil {
return fmt.Errorf("error cleaning subpath mount %s: %s", fullSubPath, err)
}
klog.V(4).Infof("Successfully cleaned subpath directory %s", fullSubPath)
return nil
}
// cleanSubPath will teardown the subpath bind mount and any remove any directories if empty
func cleanSubPath(mounter Interface, subpath Subpath) error {
containerDir := filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName)
// Clean subdir bindmount
if err := doCleanSubPath(mounter, containerDir, strconv.Itoa(subpath.VolumeMountIndex)); err != nil && !os.IsNotExist(err) {
return err
}
// Recusively remove directories if empty
if err := removeEmptyDirs(subpath.PodDir, containerDir); err != nil {
return err
}
return nil
}
// removeEmptyDirs works backwards from endDir to baseDir and removes each directory
// if it is empty. It stops once it encounters a directory that has content
func removeEmptyDirs(baseDir, endDir string) error {
if !PathWithinBase(endDir, baseDir) {
return fmt.Errorf("endDir %q is not within baseDir %q", endDir, baseDir)
}
for curDir := endDir; curDir != baseDir; curDir = filepath.Dir(curDir) {
s, err := os.Stat(curDir)
if err != nil {
if os.IsNotExist(err) {
klog.V(5).Infof("curDir %q doesn't exist, skipping", curDir)
continue
}
return fmt.Errorf("error stat %q: %v", curDir, err)
}
if !s.IsDir() {
return fmt.Errorf("path %q not a directory", curDir)
}
err = os.Remove(curDir)
if os.IsExist(err) {
klog.V(5).Infof("Directory %q not empty, not removing", curDir)
break
} else if err != nil {
return fmt.Errorf("error removing directory %q: %v", curDir, err)
}
klog.V(5).Infof("Removed directory %q", curDir)
}
return nil
}
func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
realBase, err := filepath.EvalSymlinks(base)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
}
realFullPath := filepath.Join(realBase, subdir)
return doSafeMakeDir(realFullPath, realBase, perm)
}
func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) {
if _, err := os.Stat(pathname); os.IsNotExist(err) {
return []string{}, nil
@@ -1049,237 +764,6 @@ func getMode(pathname string) (os.FileMode, error) {
return info.Mode(), nil
}
// This implementation is shared between Linux and NsEnterMounter. Both pathname
// and base must be either already resolved symlinks or thet will be resolved in
// kubelet's mount namespace (in case it runs containerized).
func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
klog.V(4).Infof("Creating directory %q within base %q", pathname, base)
if !PathWithinBase(pathname, base) {
return fmt.Errorf("path %s is outside of allowed base %s", pathname, base)
}
// Quick check if the directory already exists
s, err := os.Stat(pathname)
if err == nil {
// Path exists
if s.IsDir() {
// The directory already exists. It can be outside of the parent,
// but there is no race-proof check.
klog.V(4).Infof("Directory %s already exists", pathname)
return nil
}
return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR}
}
// Find all existing directories
existingPath, toCreate, err := findExistingPrefix(base, pathname)
if err != nil {
return fmt.Errorf("error opening directory %s: %s", pathname, err)
}
// Ensure the existing directory is inside allowed base
fullExistingPath, err := filepath.EvalSymlinks(existingPath)
if err != nil {
return fmt.Errorf("error opening directory %s: %s", existingPath, err)
}
if !PathWithinBase(fullExistingPath, base) {
return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err)
}
klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...))
parentFD, err := doSafeOpen(fullExistingPath, base)
if err != nil {
return fmt.Errorf("cannot open directory %s: %s", existingPath, err)
}
childFD := -1
defer func() {
if parentFD != -1 {
if err = syscall.Close(parentFD); err != nil {
klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err)
}
}
if childFD != -1 {
if err = syscall.Close(childFD); err != nil {
klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", childFD, pathname, err)
}
}
}()
currentPath := fullExistingPath
// create the directories one by one, making sure nobody can change
// created directory into symlink.
for _, dir := range toCreate {
currentPath = filepath.Join(currentPath, dir)
klog.V(4).Infof("Creating %s", dir)
err = syscall.Mkdirat(parentFD, currentPath, uint32(perm))
if err != nil {
return fmt.Errorf("cannot create directory %s: %s", currentPath, err)
}
// Dive into the created directory
childFD, err := syscall.Openat(parentFD, dir, nofollowFlags, 0)
if err != nil {
return fmt.Errorf("cannot open %s: %s", currentPath, err)
}
// We can be sure that childFD is safe to use. It could be changed
// by user after Mkdirat() and before Openat(), however:
// - it could not be changed to symlink - we use nofollowFlags
// - it could be changed to a file (or device, pipe, socket, ...)
// but either subsequent Mkdirat() fails or we mount this file
// to user's container. Security is no violated in both cases
// and user either gets error or the file that it can already access.
if err = syscall.Close(parentFD); err != nil {
klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err)
}
parentFD = childFD
childFD = -1
}
// Everything was created. mkdirat(..., perm) above was affected by current
// umask and we must apply the right permissions to the last directory
// (that's the one that will be available to the container as subpath)
// so user can read/write it. This is the behavior of previous code.
// TODO: chmod all created directories, not just the last one.
// parentFD is the last created directory.
// Translate perm (os.FileMode) to uint32 that fchmod() expects
kernelPerm := uint32(perm & os.ModePerm)
if perm&os.ModeSetgid > 0 {
kernelPerm |= syscall.S_ISGID
}
if perm&os.ModeSetuid > 0 {
kernelPerm |= syscall.S_ISUID
}
if perm&os.ModeSticky > 0 {
kernelPerm |= syscall.S_ISVTX
}
if err = syscall.Fchmod(parentFD, kernelPerm); err != nil {
return fmt.Errorf("chmod %q failed: %s", currentPath, err)
}
return nil
}
// findExistingPrefix finds prefix of pathname that exists. In addition, it
// returns list of remaining directories that don't exist yet.
func findExistingPrefix(base, pathname string) (string, []string, error) {
rel, err := filepath.Rel(base, pathname)
if err != nil {
return base, nil, err
}
dirs := strings.Split(rel, string(filepath.Separator))
// Do OpenAt in a loop to find the first non-existing dir. Resolve symlinks.
// This should be faster than looping through all dirs and calling os.Stat()
// on each of them, as the symlinks are resolved only once with OpenAt().
currentPath := base
fd, err := syscall.Open(currentPath, syscall.O_RDONLY, 0)
if err != nil {
return pathname, nil, fmt.Errorf("error opening %s: %s", currentPath, err)
}
defer func() {
if err = syscall.Close(fd); err != nil {
klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err)
}
}()
for i, dir := range dirs {
// Using O_PATH here will prevent hangs in case user replaces directory with
// fifo
childFD, err := syscall.Openat(fd, dir, unix.O_PATH, 0)
if err != nil {
if os.IsNotExist(err) {
return currentPath, dirs[i:], nil
}
return base, nil, err
}
if err = syscall.Close(fd); err != nil {
klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err)
}
fd = childFD
currentPath = filepath.Join(currentPath, dir)
}
return pathname, []string{}, nil
}
// This implementation is shared between Linux and NsEnterMounter
// Open path and return its fd.
// Symlinks are disallowed (pathname must already resolve symlinks),
// and the path must be within the base directory.
func doSafeOpen(pathname string, base string) (int, error) {
pathname = filepath.Clean(pathname)
base = filepath.Clean(base)
// Calculate segments to follow
subpath, err := filepath.Rel(base, pathname)
if err != nil {
return -1, err
}
segments := strings.Split(subpath, string(filepath.Separator))
// Assumption: base is the only directory that we have under control.
// Base dir is not allowed to be a symlink.
parentFD, err := syscall.Open(base, nofollowFlags, 0)
if err != nil {
return -1, fmt.Errorf("cannot open directory %s: %s", base, err)
}
defer func() {
if parentFD != -1 {
if err = syscall.Close(parentFD); err != nil {
klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", parentFD, pathname, err)
}
}
}()
childFD := -1
defer func() {
if childFD != -1 {
if err = syscall.Close(childFD); err != nil {
klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", childFD, pathname, err)
}
}
}()
currentPath := base
// Follow the segments one by one using openat() to make
// sure the user cannot change already existing directories into symlinks.
for _, seg := range segments {
currentPath = filepath.Join(currentPath, seg)
if !PathWithinBase(currentPath, base) {
return -1, fmt.Errorf("path %s is outside of allowed base %s", currentPath, base)
}
klog.V(5).Infof("Opening path %s", currentPath)
childFD, err = syscall.Openat(parentFD, seg, openFDFlags, 0)
if err != nil {
return -1, fmt.Errorf("cannot open %s: %s", currentPath, err)
}
var deviceStat unix.Stat_t
err := unix.Fstat(childFD, &deviceStat)
if err != nil {
return -1, fmt.Errorf("Error running fstat on %s with %v", currentPath, err)
}
fileFmt := deviceStat.Mode & syscall.S_IFMT
if fileFmt == syscall.S_IFLNK {
return -1, fmt.Errorf("Unexpected symlink found %s", currentPath)
}
// Close parentFD
if err = syscall.Close(parentFD); err != nil {
return -1, fmt.Errorf("closing fd for %q failed: %v", filepath.Dir(currentPath), err)
}
// Set child to new parent
parentFD = childFD
childFD = -1
}
// We made it to the end, return this fd, don't close it
finalFD := parentFD
parentFD = -1
return finalFD, nil
}
// searchMountPoints finds all mount references to the source, returns a list of
// mountpoints.
// This function assumes source cannot be device.

1136
pkg/util/mount/mount_linux_test.go
View File

File diff suppressed because it is too large Load Diff

12
pkg/util/mount/mount_unsupported.go
View File

@@ -110,18 +110,6 @@ func (mounter *Mounter) EvalHostSymlinks(pathname string) (string, error) {
return "", unsupportedErr
}
func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return subPath.Path, nil, unsupportedErr
}
func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error {
return unsupportedErr
}
func (mounter *Mounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return unsupportedErr
}
func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) {
return nil, errors.New("not implemented")
}

241
pkg/util/mount/mount_windows.go
View File

@@ -26,7 +26,6 @@ import (
"path/filepath"
"strconv"
"strings"
"syscall"
"k8s.io/klog"
@@ -279,123 +278,6 @@ func (mounter *Mounter) EvalHostSymlinks(pathname string) (string, error) {
return filepath.EvalSymlinks(pathname)
}
// check whether hostPath is within volume path
// this func will lock all intermediate subpath directories, need to close handle outside of this func after container started
func lockAndCheckSubPath(volumePath, hostPath string) ([]uintptr, error) {
if len(volumePath) == 0 || len(hostPath) == 0 {
return []uintptr{}, nil
}
finalSubPath, err := filepath.EvalSymlinks(hostPath)
if err != nil {
return []uintptr{}, fmt.Errorf("cannot read link %s: %s", hostPath, err)
}
finalVolumePath, err := filepath.EvalSymlinks(volumePath)
if err != nil {
return []uintptr{}, fmt.Errorf("cannot read link %s: %s", volumePath, err)
}
return lockAndCheckSubPathWithoutSymlink(finalVolumePath, finalSubPath)
}
// lock all intermediate subPath directories and check they are all within volumePath
// volumePath & subPath should not contain any symlink, otherwise it will return error
func lockAndCheckSubPathWithoutSymlink(volumePath, subPath string) ([]uintptr, error) {
if len(volumePath) == 0 || len(subPath) == 0 {
return []uintptr{}, nil
}
// get relative path to volumePath
relSubPath, err := filepath.Rel(volumePath, subPath)
if err != nil {
return []uintptr{}, fmt.Errorf("Rel(%s, %s) error: %v", volumePath, subPath, err)
}
if startsWithBackstep(relSubPath) {
return []uintptr{}, fmt.Errorf("SubPath %q not within volume path %q", subPath, volumePath)
}
if relSubPath == "." {
// volumePath and subPath are equal
return []uintptr{}, nil
}
fileHandles := []uintptr{}
var errorResult error
currentFullPath := volumePath
dirs := strings.Split(relSubPath, string(os.PathSeparator))
for _, dir := range dirs {
// lock intermediate subPath directory first
currentFullPath = filepath.Join(currentFullPath, dir)
handle, err := lockPath(currentFullPath)
if err != nil {
errorResult = fmt.Errorf("cannot lock path %s: %s", currentFullPath, err)
break
}
fileHandles = append(fileHandles, handle)
// make sure intermediate subPath directory does not contain symlink any more
stat, err := os.Lstat(currentFullPath)
if err != nil {
errorResult = fmt.Errorf("Lstat(%q) error: %v", currentFullPath, err)
break
}
if stat.Mode()&os.ModeSymlink != 0 {
errorResult = fmt.Errorf("subpath %q is an unexpected symlink after EvalSymlinks", currentFullPath)
break
}
if !PathWithinBase(currentFullPath, volumePath) {
errorResult = fmt.Errorf("SubPath %q not within volume path %q", currentFullPath, volumePath)
break
}
}
return fileHandles, errorResult
}
// unlockPath unlock directories
func unlockPath(fileHandles []uintptr) {
if fileHandles != nil {
for _, handle := range fileHandles {
syscall.CloseHandle(syscall.Handle(handle))
}
}
}
// lockPath locks a directory or symlink, return handle, exec "syscall.CloseHandle(handle)" to unlock the path
func lockPath(path string) (uintptr, error) {
if len(path) == 0 {
return uintptr(syscall.InvalidHandle), syscall.ERROR_FILE_NOT_FOUND
}
pathp, err := syscall.UTF16PtrFromString(path)
if err != nil {
return uintptr(syscall.InvalidHandle), err
}
access := uint32(syscall.GENERIC_READ)
sharemode := uint32(syscall.FILE_SHARE_READ)
createmode := uint32(syscall.OPEN_EXISTING)
flags := uint32(syscall.FILE_FLAG_BACKUP_SEMANTICS | syscall.FILE_FLAG_OPEN_REPARSE_POINT)
fd, err := syscall.CreateFile(pathp, access, sharemode, nil, createmode, flags, 0)
return uintptr(fd), err
}
// Lock all directories in subPath and check they're not symlinks.
func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
handles, err := lockAndCheckSubPath(subPath.VolumePath, subPath.Path)
// Unlock the directories when the container starts
cleanupAction = func() {
unlockPath(handles)
}
return subPath.Path, cleanupAction, err
}
// No bind-mounts for subpaths are necessary on Windows
func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error {
return nil
}
func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error {
// Try to mount the disk
klog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, source, target)
@@ -522,126 +404,3 @@ func (mounter *Mounter) GetMode(pathname string) (os.FileMode, error) {
}
return info.Mode(), nil
}
// SafeMakeDir makes sure that the created directory does not escape given base directory mis-using symlinks.
func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
realBase, err := filepath.EvalSymlinks(base)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
}
realFullPath := filepath.Join(realBase, subdir)
return doSafeMakeDir(realFullPath, realBase, perm)
}
func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
klog.V(4).Infof("Creating directory %q within base %q", pathname, base)
if !PathWithinBase(pathname, base) {
return fmt.Errorf("path %s is outside of allowed base %s", pathname, base)
}
// Quick check if the directory already exists
s, err := os.Stat(pathname)
if err == nil {
// Path exists
if s.IsDir() {
// The directory already exists. It can be outside of the parent,
// but there is no race-proof check.
klog.V(4).Infof("Directory %s already exists", pathname)
return nil
}
return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR}
}
// Find all existing directories
existingPath, toCreate, err := findExistingPrefix(base, pathname)
if err != nil {
return fmt.Errorf("error opening directory %s: %s", pathname, err)
}
if len(toCreate) == 0 {
return nil
}
// Ensure the existing directory is inside allowed base
fullExistingPath, err := filepath.EvalSymlinks(existingPath)
if err != nil {
return fmt.Errorf("error opening existing directory %s: %s", existingPath, err)
}
fullBasePath, err := filepath.EvalSymlinks(base)
if err != nil {
return fmt.Errorf("cannot read link %s: %s", base, err)
}
if !PathWithinBase(fullExistingPath, fullBasePath) {
return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err)
}
// lock all intermediate directories from fullBasePath to fullExistingPath (top to bottom)
fileHandles, err := lockAndCheckSubPathWithoutSymlink(fullBasePath, fullExistingPath)
defer unlockPath(fileHandles)
if err != nil {
return err
}
klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...))
currentPath := fullExistingPath
// create the directories one by one, making sure nobody can change
// created directory into symlink by lock that directory immediately
for _, dir := range toCreate {
currentPath = filepath.Join(currentPath, dir)
klog.V(4).Infof("Creating %s", dir)
if err := os.Mkdir(currentPath, perm); err != nil {
return fmt.Errorf("cannot create directory %s: %s", currentPath, err)
}
handle, err := lockPath(currentPath)
if err != nil {
return fmt.Errorf("cannot lock path %s: %s", currentPath, err)
}
defer syscall.CloseHandle(syscall.Handle(handle))
// make sure newly created directory does not contain symlink after lock
stat, err := os.Lstat(currentPath)
if err != nil {
return fmt.Errorf("Lstat(%q) error: %v", currentPath, err)
}
if stat.Mode()&os.ModeSymlink != 0 {
return fmt.Errorf("subpath %q is an unexpected symlink after Mkdir", currentPath)
}
}
return nil
}
// findExistingPrefix finds prefix of pathname that exists. In addition, it
// returns list of remaining directories that don't exist yet.
func findExistingPrefix(base, pathname string) (string, []string, error) {
rel, err := filepath.Rel(base, pathname)
if err != nil {
return base, nil, err
}
if startsWithBackstep(rel) {
return base, nil, fmt.Errorf("pathname(%s) is not within base(%s)", pathname, base)
}
if rel == "." {
// base and pathname are equal
return pathname, []string{}, nil
}
dirs := strings.Split(rel, string(filepath.Separator))
parent := base
currentPath := base
for i, dir := range dirs {
parent = currentPath
currentPath = filepath.Join(parent, dir)
if _, err := os.Lstat(currentPath); err != nil {
if os.IsNotExist(err) {
return parent, dirs[i:], nil
}
return base, nil, err
}
}
return pathname, []string{}, nil
}

403
pkg/util/mount/mount_windows_test.go
View File

@@ -134,409 +134,6 @@ func TestGetMountRefs(t *testing.T) {
}
}
func TestDoSafeMakeDir(t *testing.T) {
base, err := ioutil.TempDir("", "TestDoSafeMakeDir")
if err != nil {
t.Fatalf(err.Error())
}
defer os.RemoveAll(base)
testingVolumePath := filepath.Join(base, "testingVolumePath")
os.MkdirAll(testingVolumePath, 0755)
defer os.RemoveAll(testingVolumePath)
tests := []struct {
volumePath string
subPath string
expectError bool
symlinkTarget string
}{
{
volumePath: testingVolumePath,
subPath: ``,
expectError: true,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `x`),
expectError: false,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a\b\c\d`),
expectError: false,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `symlink`),
expectError: false,
symlinkTarget: base,
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `symlink\c\d`),
expectError: true,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `symlink\y926`),
expectError: true,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a\b\symlink`),
expectError: false,
symlinkTarget: base,
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a\x\symlink`),
expectError: false,
symlinkTarget: filepath.Join(testingVolumePath, `a`),
},
}
for _, test := range tests {
if len(test.volumePath) > 0 && len(test.subPath) > 0 && len(test.symlinkTarget) > 0 {
// make all parent sub directories
if parent := filepath.Dir(test.subPath); parent != "." {
os.MkdirAll(parent, 0755)
}
// make last element as symlink
linkPath := test.subPath
if _, err := os.Stat(linkPath); err != nil && os.IsNotExist(err) {
if err := makeLink(linkPath, test.symlinkTarget); err != nil {
t.Fatalf("unexpected error: %v", fmt.Errorf("mklink link(%q) target(%q) error: %q", linkPath, test.symlinkTarget, err))
}
}
}
err := doSafeMakeDir(test.subPath, test.volumePath, os.FileMode(0755))
if test.expectError {
assert.NotNil(t, err, "Expect error during doSafeMakeDir(%s, %s)", test.subPath, test.volumePath)
continue
}
assert.Nil(t, err, "Expect no error during doSafeMakeDir(%s, %s)", test.subPath, test.volumePath)
if _, err := os.Stat(test.subPath); os.IsNotExist(err) {
t.Errorf("subPath should exists after doSafeMakeDir(%s, %s)", test.subPath, test.volumePath)
}
}
}
func TestLockAndCheckSubPath(t *testing.T) {
base, err := ioutil.TempDir("", "TestLockAndCheckSubPath")
if err != nil {
t.Fatalf(err.Error())
}
defer os.RemoveAll(base)
testingVolumePath := filepath.Join(base, "testingVolumePath")
tests := []struct {
volumePath string
subPath string
expectedHandleCount int
expectError bool
symlinkTarget string
}{
{
volumePath: `c:\`,
subPath: ``,
expectedHandleCount: 0,
expectError: false,
symlinkTarget: "",
},
{
volumePath: ``,
subPath: `a`,
expectedHandleCount: 0,
expectError: false,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a`),
expectedHandleCount: 1,
expectError: false,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a\b\c\d`),
expectedHandleCount: 4,
expectError: false,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `symlink`),
expectedHandleCount: 0,
expectError: true,
symlinkTarget: base,
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a\b\c\symlink`),
expectedHandleCount: 0,
expectError: true,
symlinkTarget: base,
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a\b\c\d\symlink`),
expectedHandleCount: 2,
expectError: false,
symlinkTarget: filepath.Join(testingVolumePath, `a\b`),
},
}
for _, test := range tests {
if len(test.volumePath) > 0 && len(test.subPath) > 0 {
os.MkdirAll(test.volumePath, 0755)
if len(test.symlinkTarget) == 0 {
// make all intermediate sub directories
os.MkdirAll(test.subPath, 0755)
} else {
// make all parent sub directories
if parent := filepath.Dir(test.subPath); parent != "." {
os.MkdirAll(parent, 0755)
}
// make last element as symlink
linkPath := test.subPath
if _, err := os.Stat(linkPath); err != nil && os.IsNotExist(err) {
if err := makeLink(linkPath, test.symlinkTarget); err != nil {
t.Fatalf("unexpected error: %v", fmt.Errorf("mklink link(%q) target(%q) error: %q", linkPath, test.symlinkTarget, err))
}
}
}
}
fileHandles, err := lockAndCheckSubPath(test.volumePath, test.subPath)
unlockPath(fileHandles)
assert.Equal(t, test.expectedHandleCount, len(fileHandles))
if test.expectError {
assert.NotNil(t, err, "Expect error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath)
continue
}
assert.Nil(t, err, "Expect no error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath)
}
// remove dir will happen after closing all file handles
assert.Nil(t, os.RemoveAll(testingVolumePath), "Expect no error during remove dir %s", testingVolumePath)
}
func TestLockAndCheckSubPathWithoutSymlink(t *testing.T) {
base, err := ioutil.TempDir("", "TestLockAndCheckSubPathWithoutSymlink")
if err != nil {
t.Fatalf(err.Error())
}
defer os.RemoveAll(base)
testingVolumePath := filepath.Join(base, "testingVolumePath")
tests := []struct {
volumePath string
subPath string
expectedHandleCount int
expectError bool
symlinkTarget string
}{
{
volumePath: `c:\`,
subPath: ``,
expectedHandleCount: 0,
expectError: false,
symlinkTarget: "",
},
{
volumePath: ``,
subPath: `a`,
expectedHandleCount: 0,
expectError: false,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a`),
expectedHandleCount: 1,
expectError: false,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a\b\c\d`),
expectedHandleCount: 4,
expectError: false,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `symlink`),
expectedHandleCount: 1,
expectError: true,
symlinkTarget: base,
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a\b\c\symlink`),
expectedHandleCount: 4,
expectError: true,
symlinkTarget: base,
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a\b\c\d\symlink`),
expectedHandleCount: 5,
expectError: true,
symlinkTarget: filepath.Join(testingVolumePath, `a\b`),
},
}
for _, test := range tests {
if len(test.volumePath) > 0 && len(test.subPath) > 0 {
os.MkdirAll(test.volumePath, 0755)
if len(test.symlinkTarget) == 0 {
// make all intermediate sub directories
os.MkdirAll(test.subPath, 0755)
} else {
// make all parent sub directories
if parent := filepath.Dir(test.subPath); parent != "." {
os.MkdirAll(parent, 0755)
}
// make last element as symlink
linkPath := test.subPath
if _, err := os.Stat(linkPath); err != nil && os.IsNotExist(err) {
if err := makeLink(linkPath, test.symlinkTarget); err != nil {
t.Fatalf("unexpected error: %v", fmt.Errorf("mklink link(%q) target(%q) error: %q", linkPath, test.symlinkTarget, err))
}
}
}
}
fileHandles, err := lockAndCheckSubPathWithoutSymlink(test.volumePath, test.subPath)
unlockPath(fileHandles)
assert.Equal(t, test.expectedHandleCount, len(fileHandles))
if test.expectError {
assert.NotNil(t, err, "Expect error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath)
continue
}
assert.Nil(t, err, "Expect no error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath)
}
// remove dir will happen after closing all file handles
assert.Nil(t, os.RemoveAll(testingVolumePath), "Expect no error during remove dir %s", testingVolumePath)
}
func TestFindExistingPrefix(t *testing.T) {
base, err := ioutil.TempDir("", "TestFindExistingPrefix")
if err != nil {
t.Fatalf(err.Error())
}
defer os.RemoveAll(base)
testingVolumePath := filepath.Join(base, "testingVolumePath")
tests := []struct {
base string
pathname string
expectError bool
expectedExistingPath string
expectedToCreateDirs []string
createSubPathBeforeTest bool
}{
{
base: `c:\tmp\a`,
pathname: `c:\tmp\b`,
expectError: true,
expectedExistingPath: "",
expectedToCreateDirs: []string{},
createSubPathBeforeTest: false,
},
{
base: ``,
pathname: `c:\tmp\b`,
expectError: true,
expectedExistingPath: "",
expectedToCreateDirs: []string{},
createSubPathBeforeTest: false,
},
{
base: `c:\tmp\a`,
pathname: `d:\tmp\b`,
expectError: true,
expectedExistingPath: "",
expectedToCreateDirs: []string{},
createSubPathBeforeTest: false,
},
{
base: testingVolumePath,
pathname: testingVolumePath,
expectError: false,
expectedExistingPath: testingVolumePath,
expectedToCreateDirs: []string{},
createSubPathBeforeTest: false,
},
{
base: testingVolumePath,
pathname: filepath.Join(testingVolumePath, `a\b`),
expectError: false,
expectedExistingPath: filepath.Join(testingVolumePath, `a\b`),
expectedToCreateDirs: []string{},
createSubPathBeforeTest: true,
},
{
base: testingVolumePath,
pathname: filepath.Join(testingVolumePath, `a\b\c\`),
expectError: false,
expectedExistingPath: filepath.Join(testingVolumePath, `a\b`),
expectedToCreateDirs: []string{`c`},
createSubPathBeforeTest: false,
},
{
base: testingVolumePath,
pathname: filepath.Join(testingVolumePath, `a\b\c\d`),
expectError: false,
expectedExistingPath: filepath.Join(testingVolumePath, `a\b`),
expectedToCreateDirs: []string{`c`, `d`},
createSubPathBeforeTest: false,
},
}
for _, test := range tests {
if test.createSubPathBeforeTest {
os.MkdirAll(test.pathname, 0755)
}
existingPath, toCreate, err := findExistingPrefix(test.base, test.pathname)
if test.expectError {
assert.NotNil(t, err, "Expect error during findExistingPrefix(%s, %s)", test.base, test.pathname)
continue
}
assert.Nil(t, err, "Expect no error during findExistingPrefix(%s, %s)", test.base, test.pathname)
assert.Equal(t, test.expectedExistingPath, existingPath, "Expect result not equal with findExistingPrefix(%s, %s) return: %q, expected: %q",
test.base, test.pathname, existingPath, test.expectedExistingPath)
assert.Equal(t, test.expectedToCreateDirs, toCreate, "Expect result not equal with findExistingPrefix(%s, %s) return: %q, expected: %q",
test.base, test.pathname, toCreate, test.expectedToCreateDirs)
}
// remove dir will happen after closing all file handles
assert.Nil(t, os.RemoveAll(testingVolumePath), "Expect no error during remove dir %s", testingVolumePath)
}
func TestPathWithinBase(t *testing.T) {
tests := []struct {
fullPath string

136
pkg/util/mount/nsenter_mount.go
View File

@@ -23,9 +23,7 @@ import (
"os"
"path/filepath"
"strings"
"syscall"
"golang.org/x/sys/unix"
"k8s.io/klog"
"k8s.io/utils/nsenter"
utilpath "k8s.io/utils/path"
@@ -298,51 +296,6 @@ func (mounter *NsenterMounter) EvalHostSymlinks(pathname string) (string, error)
return mounter.ne.EvalSymlinks(pathname, true)
}
func (mounter *NsenterMounter) CleanSubPaths(podDir string, volumeName string) error {
return doCleanSubPaths(mounter, podDir, volumeName)
}
func (mounter *NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
// Bind-mount the subpath to avoid using symlinks in subpaths.
newHostPath, err = doNsEnterBindSubPath(mounter, subPath)
// There is no action when the container starts. Bind-mount will be cleaned
// when container stops by CleanSubPaths.
cleanupAction = nil
return newHostPath, cleanupAction, err
}
func (mounter *NsenterMounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
fullSubdirPath := filepath.Join(base, subdir)
evaluatedSubdirPath, err := mounter.ne.EvalSymlinks(fullSubdirPath, false /* mustExist */)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", fullSubdirPath, err)
}
evaluatedSubdirPath = filepath.Clean(evaluatedSubdirPath)
evaluatedBase, err := mounter.ne.EvalSymlinks(base, true /* mustExist */)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
}
evaluatedBase = filepath.Clean(evaluatedBase)
rootDir := filepath.Clean(mounter.rootDir)
if PathWithinBase(evaluatedBase, rootDir) {
// Base is in /var/lib/kubelet. This directory is shared between the
// container with kubelet and the host. We don't need to add '/rootfs'.
// This is useful when /rootfs is mounted as read-only - we can still
// create subpaths for paths in /var/lib/kubelet.
return doSafeMakeDir(evaluatedSubdirPath, evaluatedBase, perm)
}
// Base is somewhere on the host's filesystem. Add /rootfs and try to make
// the directory there.
// This requires /rootfs to be writable.
kubeletSubdirPath := mounter.ne.KubeletPath(evaluatedSubdirPath)
kubeletBase := mounter.ne.KubeletPath(evaluatedBase)
return doSafeMakeDir(kubeletSubdirPath, kubeletBase, perm)
}
func (mounter *NsenterMounter) GetMountRefs(pathname string) ([]string, error) {
exists, err := mounter.ExistsPath(pathname)
if err != nil {
@@ -358,95 +311,6 @@ func (mounter *NsenterMounter) GetMountRefs(pathname string) ([]string, error) {
return searchMountPoints(hostpath, hostProcMountinfoPath)
}
func doNsEnterBindSubPath(mounter *NsenterMounter, subpath Subpath) (hostPath string, err error) {
// Linux, kubelet runs in a container:
// - safely open the subpath
// - bind-mount the subpath to target (this can be unsafe)
// - check that we mounted the right thing by comparing device ID and inode
// of the subpath (via safely opened fd) and the target (that's under our
// control)
// Evaluate all symlinks here once for all subsequent functions.
evaluatedHostVolumePath, err := mounter.ne.EvalSymlinks(subpath.VolumePath, true /*mustExist*/)
if err != nil {
return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err)
}
evaluatedHostSubpath, err := mounter.ne.EvalSymlinks(subpath.Path, true /*mustExist*/)
if err != nil {
return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err)
}
klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, evaluatedHostSubpath, subpath.VolumePath)
subpath.VolumePath = mounter.ne.KubeletPath(evaluatedHostVolumePath)
subpath.Path = mounter.ne.KubeletPath(evaluatedHostSubpath)
// Check the subpath is correct and open it
fd, err := safeOpenSubPath(mounter, subpath)
if err != nil {
return "", err
}
defer syscall.Close(fd)
alreadyMounted, bindPathTarget, err := prepareSubpathTarget(mounter, subpath)
if err != nil {
return "", err
}
if alreadyMounted {
return bindPathTarget, nil
}
success := false
defer func() {
// Cleanup subpath on error
if !success {
klog.V(4).Infof("doNsEnterBindSubPath() failed for %q, cleaning up subpath", bindPathTarget)
if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil {
klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr)
}
}
}()
// Leap of faith: optimistically expect that nobody has modified previously
// expanded evalSubPath with evil symlinks and bind-mount it.
// Mount is done on the host! don't use kubelet path!
klog.V(5).Infof("bind mounting %q at %q", evaluatedHostSubpath, bindPathTarget)
if err = mounter.Mount(evaluatedHostSubpath, bindPathTarget, "" /*fstype*/, []string{"bind"}); err != nil {
return "", fmt.Errorf("error mounting %s: %s", evaluatedHostSubpath, err)
}
// Check that the bind-mount target is the same inode and device as the
// source that we keept open, i.e. we mounted the right thing.
err = checkDeviceInode(fd, bindPathTarget)
if err != nil {
return "", fmt.Errorf("error checking bind mount for subpath %s: %s", subpath.VolumePath, err)
}
success = true
klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget)
return bindPathTarget, nil
}
// checkDeviceInode checks that opened file and path represent the same file.
func checkDeviceInode(fd int, path string) error {
var srcStat, dstStat unix.Stat_t
err := unix.Fstat(fd, &srcStat)
if err != nil {
return fmt.Errorf("error running fstat on subpath FD: %v", err)
}
err = unix.Stat(path, &dstStat)
if err != nil {
return fmt.Errorf("error running fstat on %s: %v", path, err)
}
if srcStat.Dev != dstStat.Dev {
return fmt.Errorf("different device number")
}
if srcStat.Ino != dstStat.Ino {
return fmt.Errorf("different inode")
}
return nil
}
func (mounter *NsenterMounter) GetFSGroup(pathname string) (int64, error) {
hostPath, err := mounter.ne.EvalSymlinks(pathname, true /* mustExist */)
if err != nil {

294
pkg/util/mount/nsenter_mount_test.go
View File

@@ -23,10 +23,8 @@ import (
"os"
"os/user"
"path/filepath"
"strings"
"testing"
"golang.org/x/sys/unix"
"k8s.io/utils/nsenter"
)
@@ -76,80 +74,6 @@ func TestParseFindMnt(t *testing.T) {
}
}
func TestCheckDeviceInode(t *testing.T) {
testDir, err := ioutil.TempDir("", "nsenter-mounter-device-")
if err != nil {
t.Fatalf("Cannot create temporary directory: %s", err)
}
defer os.RemoveAll(testDir)
tests := []struct {
name string
srcPath string
dstPath string
expectError string
}{
{
name: "the same file",
srcPath: filepath.Join(testDir, "1"),
dstPath: filepath.Join(testDir, "1"),
expectError: "",
},
{
name: "different file on the same FS",
srcPath: filepath.Join(testDir, "2.1"),
dstPath: filepath.Join(testDir, "2.2"),
expectError: "different inode",
},
{
name: "different file on different device",
srcPath: filepath.Join(testDir, "3"),
// /proc is always on a different "device" than /tmp (or $TEMP)
dstPath: "/proc/self/status",
expectError: "different device",
},
}
for _, test := range tests {
if err := ioutil.WriteFile(test.srcPath, []byte{}, 0644); err != nil {
t.Errorf("Test %q: cannot create srcPath %s: %s", test.name, test.srcPath, err)
continue
}
// Don't create dst if it exists
if _, err := os.Stat(test.dstPath); os.IsNotExist(err) {
if err := ioutil.WriteFile(test.dstPath, []byte{}, 0644); err != nil {
t.Errorf("Test %q: cannot create dstPath %s: %s", test.name, test.dstPath, err)
continue
}
} else if err != nil {
t.Errorf("Test %q: cannot check existence of dstPath %s: %s", test.name, test.dstPath, err)
continue
}
fd, err := unix.Open(test.srcPath, unix.O_CREAT, 0644)
if err != nil {
t.Errorf("Test %q: cannot open srcPath %s: %s", test.name, test.srcPath, err)
continue
}
err = checkDeviceInode(fd, test.dstPath)
if test.expectError == "" && err != nil {
t.Errorf("Test %q: expected no error, got %s", test.name, err)
}
if test.expectError != "" {
if err == nil {
t.Errorf("Test %q: expected error, got none", test.name)
} else {
if !strings.Contains(err.Error(), test.expectError) {
t.Errorf("Test %q: expected error %q, got %q", test.name, test.expectError, err)
}
}
}
}
}
func newFakeNsenterMounter(tmpdir string, t *testing.T) (mounter *NsenterMounter, rootfsPath string, varlibPath string, err error) {
rootfsPath = filepath.Join(tmpdir, "rootfs")
if err := os.Mkdir(rootfsPath, 0755); err != nil {
@@ -504,221 +428,3 @@ func writeRootfsFile(rootfs, path string, mode os.FileMode) (string, error) {
}
return fullPath, nil
}
func TestNsenterSafeMakeDir(t *testing.T) {
tests := []struct {
name string
prepare func(base, rootfs, varlib string) (expectedDir string, err error)
subdir string
expectError bool
// If true, "base" directory for SafeMakeDir will be /var/lib/kubelet
baseIsVarLib bool
}{
{
name: "simple directory",
// evaluated in base
subdir: "some/subdirectory/structure",
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// expected to be created in /roots/
expectedDir = filepath.Join(rootfs, base, "some/subdirectory/structure")
return expectedDir, nil
},
},
{
name: "simple existing directory",
// evaluated in base
subdir: "some/subdirectory/structure",
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// On the host: directory exists
hostPath := filepath.Join(base, "some/subdirectory/structure")
if err := os.MkdirAll(hostPath, 0755); err != nil {
return "", err
}
// In rootfs: directory exists
kubeletPath := filepath.Join(rootfs, hostPath)
if err := os.MkdirAll(kubeletPath, 0755); err != nil {
return "", err
}
// expected to be created in /roots/
expectedDir = kubeletPath
return expectedDir, nil
},
},
{
name: "absolute symlink into safe place",
// evaluated in base
subdir: "some/subdirectory/structure",
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// On the host: /base/other/subdirectory exists, /base/some is link to /base/other
hostPath := filepath.Join(base, "other/subdirectory")
if err := os.MkdirAll(hostPath, 0755); err != nil {
return "", err
}
somePath := filepath.Join(base, "some")
otherPath := filepath.Join(base, "other")
if err := os.Symlink(otherPath, somePath); err != nil {
return "", err
}
// In rootfs: /base/other/subdirectory exists
kubeletPath := filepath.Join(rootfs, hostPath)
if err := os.MkdirAll(kubeletPath, 0755); err != nil {
return "", err
}
// expected 'structure' to be created
expectedDir = filepath.Join(rootfs, hostPath, "structure")
return expectedDir, nil
},
},
{
name: "relative symlink into safe place",
// evaluated in base
subdir: "some/subdirectory/structure",
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// On the host: /base/other/subdirectory exists, /base/some is link to other
hostPath := filepath.Join(base, "other/subdirectory")
if err := os.MkdirAll(hostPath, 0755); err != nil {
return "", err
}
somePath := filepath.Join(base, "some")
if err := os.Symlink("other", somePath); err != nil {
return "", err
}
// In rootfs: /base/other/subdirectory exists
kubeletPath := filepath.Join(rootfs, hostPath)
if err := os.MkdirAll(kubeletPath, 0755); err != nil {
return "", err
}
// expected 'structure' to be created
expectedDir = filepath.Join(rootfs, hostPath, "structure")
return expectedDir, nil
},
},
{
name: "symlink into unsafe place",
// evaluated in base
subdir: "some/subdirectory/structure",
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// On the host: /base/some is link to /bin/other
somePath := filepath.Join(base, "some")
if err := os.Symlink("/bin", somePath); err != nil {
return "", err
}
return "", nil
},
expectError: true,
},
{
name: "simple directory in /var/lib/kubelet",
// evaluated in varlib
subdir: "some/subdirectory/structure",
baseIsVarLib: true,
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// expected to be created in /base/var/lib/kubelet, not in /rootfs!
expectedDir = filepath.Join(varlib, "some/subdirectory/structure")
return expectedDir, nil
},
},
{
name: "safe symlink in /var/lib/kubelet",
// evaluated in varlib
subdir: "some/subdirectory/structure",
baseIsVarLib: true,
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// On the host: /varlib/kubelet/other/subdirectory exists, /varlib/some is link to other
hostPath := filepath.Join(varlib, "other/subdirectory")
if err := os.MkdirAll(hostPath, 0755); err != nil {
return "", err
}
somePath := filepath.Join(varlib, "some")
if err := os.Symlink("other", somePath); err != nil {
return "", err
}
// expected to be created in /base/var/lib/kubelet, not in /rootfs!
expectedDir = filepath.Join(varlib, "other/subdirectory/structure")
return expectedDir, nil
},
},
{
name: "unsafe symlink in /var/lib/kubelet",
// evaluated in varlib
subdir: "some/subdirectory/structure",
baseIsVarLib: true,
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// On the host: /varlib/some is link to /bin
somePath := filepath.Join(varlib, "some")
if err := os.Symlink("/bin", somePath); err != nil {
return "", err
}
return "", nil
},
expectError: true,
},
}
for _, test := range tests {
tmpdir, err := ioutil.TempDir("", "nsenter-get-mode-")
if err != nil {
t.Error(err)
continue
}
defer os.RemoveAll(tmpdir)
mounter, rootfs, varlib, err := newFakeNsenterMounter(tmpdir, t)
if err != nil {
t.Error(err)
continue
}
// Prepare base directory for the test
testBase := filepath.Join(tmpdir, "base")
if err := os.Mkdir(testBase, 0755); err != nil {
t.Error(err)
continue
}
// Prepare base directory also in /rootfs
rootfsBase := filepath.Join(rootfs, testBase)
if err := os.MkdirAll(rootfsBase, 0755); err != nil {
t.Error(err)
continue
}
expectedDir := ""
if test.prepare != nil {
expectedDir, err = test.prepare(testBase, rootfs, varlib)
if err != nil {
t.Error(err)
continue
}
}
if test.baseIsVarLib {
// use /var/lib/kubelet as the test base so we can test creating
// subdirs there directly in /var/lib/kubenet and not in
// /rootfs/var/lib/kubelet
testBase = varlib
}
err = mounter.SafeMakeDir(test.subdir, testBase, 0755)
if err != nil && !test.expectError {
t.Errorf("Test %q: unexpected error: %s", test.name, err)
}
if test.expectError {
if err == nil {
t.Errorf("Test %q: expected error, got none", test.name)
} else {
if !strings.Contains(err.Error(), "is outside of allowed base") {
t.Errorf("Test %q: expected error to contain \"is outside of allowed base\", got this one instead: %s", test.name, err)
}
}
}
if expectedDir != "" {
_, err := os.Stat(expectedDir)
if err != nil {
t.Errorf("Test %q: expected %q to exist, got error: %s", test.name, expectedDir, err)
}
}
}
}

12
pkg/util/mount/nsenter_mount_unsupported.go
View File

@@ -93,18 +93,6 @@ func (*NsenterMounter) EvalHostSymlinks(pathname string) (string, error) {
return "", errors.New("not implemented")
}
func (*NsenterMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return nil
}
func (*NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return subPath.Path, nil, nil
}
func (*NsenterMounter) CleanSubPaths(podDir string, volumeName string) error {
return nil
}
func (*NsenterMounter) GetMountRefs(pathname string) ([]string, error) {
return nil, errors.New("not implemented")
}