diff --git a/hack/.staticcheck_failures b/hack/.staticcheck_failures index 1062e18e1c5..17de5e39cf4 100644 --- a/hack/.staticcheck_failures +++ b/hack/.staticcheck_failures @@ -86,7 +86,6 @@ pkg/volume/storageos pkg/volume/util pkg/volume/util/fsquota pkg/volume/util/fsquota/common -pkg/volume/util/nsenter pkg/volume/util/operationexecutor pkg/volume/util/subpath pkg/volume/vsphere_volume diff --git a/pkg/volume/util/BUILD b/pkg/volume/util/BUILD index e51e73f6231..7e6dcf1252f 100644 --- a/pkg/volume/util/BUILD +++ b/pkg/volume/util/BUILD @@ -91,7 +91,6 @@ filegroup( "//pkg/volume/util/fs:all-srcs", "//pkg/volume/util/fsquota:all-srcs", "//pkg/volume/util/nestedpendingoperations:all-srcs", - "//pkg/volume/util/nsenter:all-srcs", "//pkg/volume/util/operationexecutor:all-srcs", "//pkg/volume/util/recyclerclient:all-srcs", "//pkg/volume/util/subpath:all-srcs", diff --git a/pkg/volume/util/nsenter/BUILD b/pkg/volume/util/nsenter/BUILD deleted file mode 100644 index e4f27121bc1..00000000000 --- a/pkg/volume/util/nsenter/BUILD +++ /dev/null @@ -1,87 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = [ - "nsenter_mount.go", - "nsenter_mount_unsupported.go", - ], - importpath = "k8s.io/kubernetes/pkg/volume/util/nsenter", - visibility = ["//visibility:public"], - deps = select({ - "@io_bazel_rules_go//go/platform:android": [ - "//pkg/util/mount:go_default_library", - "//vendor/k8s.io/utils/nsenter:go_default_library", - ], - "@io_bazel_rules_go//go/platform:darwin": [ - "//pkg/util/mount:go_default_library", - "//vendor/k8s.io/utils/nsenter:go_default_library", - ], - "@io_bazel_rules_go//go/platform:dragonfly": [ - "//pkg/util/mount:go_default_library", - "//vendor/k8s.io/utils/nsenter:go_default_library", - ], - "@io_bazel_rules_go//go/platform:freebsd": [ - "//pkg/util/mount:go_default_library", - "//vendor/k8s.io/utils/nsenter:go_default_library", - ], - "@io_bazel_rules_go//go/platform:linux": [ - "//pkg/util/mount:go_default_library", - "//vendor/k8s.io/klog:go_default_library", - "//vendor/k8s.io/utils/nsenter:go_default_library", - "//vendor/k8s.io/utils/path:go_default_library", - ], - "@io_bazel_rules_go//go/platform:nacl": [ - "//pkg/util/mount:go_default_library", - "//vendor/k8s.io/utils/nsenter:go_default_library", - ], - "@io_bazel_rules_go//go/platform:netbsd": [ - "//pkg/util/mount:go_default_library", - "//vendor/k8s.io/utils/nsenter:go_default_library", - ], - "@io_bazel_rules_go//go/platform:openbsd": [ - "//pkg/util/mount:go_default_library", - "//vendor/k8s.io/utils/nsenter:go_default_library", - ], - "@io_bazel_rules_go//go/platform:plan9": [ - "//pkg/util/mount:go_default_library", - "//vendor/k8s.io/utils/nsenter:go_default_library", - ], - "@io_bazel_rules_go//go/platform:solaris": [ - "//pkg/util/mount:go_default_library", - "//vendor/k8s.io/utils/nsenter:go_default_library", - ], - "@io_bazel_rules_go//go/platform:windows": [ - "//pkg/util/mount:go_default_library", - "//vendor/k8s.io/utils/nsenter:go_default_library", - ], - "//conditions:default": [], - }), -) - -go_test( - name = "go_default_test", - srcs = ["nsenter_mount_test.go"], - embed = [":go_default_library"], - deps = select({ - "@io_bazel_rules_go//go/platform:linux": [ - "//pkg/util/mount:go_default_library", - "//vendor/k8s.io/utils/nsenter:go_default_library", - ], - "//conditions:default": [], - }), -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/pkg/volume/util/nsenter/nsenter_mount.go b/pkg/volume/util/nsenter/nsenter_mount.go deleted file mode 100644 index 93b4c5978c0..00000000000 --- a/pkg/volume/util/nsenter/nsenter_mount.go +++ /dev/null @@ -1,360 +0,0 @@ -// +build linux - -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package nsenter - -import ( - "fmt" - "os" - "path/filepath" - "strings" - - "k8s.io/klog" - "k8s.io/kubernetes/pkg/util/mount" - "k8s.io/utils/nsenter" - utilpath "k8s.io/utils/path" -) - -const ( - // hostProcMountsPath is the default mount path for rootfs - hostProcMountsPath = "/rootfs/proc/1/mounts" - // hostProcMountinfoPath is the default mount info path for rootfs - hostProcMountinfoPath = "/rootfs/proc/1/mountinfo" -) - -// Mounter implements mount.Interface -// Currently, all docker containers receive their own mount namespaces. -// Mounter works by executing nsenter to run commands in -// the host's mount namespace. -type Mounter struct { - ne *nsenter.Nsenter - // rootDir is location of /var/lib/kubelet directory. - rootDir string -} - -// NewMounter creates a new mounter for kubelet that runs as a container. -func NewMounter(rootDir string, ne *nsenter.Nsenter) *Mounter { - return &Mounter{ - rootDir: rootDir, - ne: ne, - } -} - -// Mounter implements mount.Interface -var _ = mount.Interface(&Mounter{}) - -// Mount runs mount(8) in the host's root mount namespace. Aside from this -// aspect, Mount has the same semantics as the mounter returned by mount.New() -func (n *Mounter) Mount(source string, target string, fstype string, options []string) error { - bind, bindOpts, bindRemountOpts := mount.IsBind(options) - - if bind { - err := n.doNsenterMount(source, target, fstype, bindOpts) - if err != nil { - return err - } - return n.doNsenterMount(source, target, fstype, bindRemountOpts) - } - - return n.doNsenterMount(source, target, fstype, options) -} - -// doNsenterMount nsenters the host's mount namespace and performs the -// requested mount. -func (n *Mounter) doNsenterMount(source, target, fstype string, options []string) error { - klog.V(5).Infof("nsenter mount %s %s %s %v", source, target, fstype, options) - cmd, args := n.makeNsenterArgs(source, target, fstype, options) - outputBytes, err := n.ne.Exec(cmd, args).CombinedOutput() - if len(outputBytes) != 0 { - klog.V(5).Infof("Output of mounting %s to %s: %v", source, target, string(outputBytes)) - } - return err -} - -// makeNsenterArgs makes a list of argument to nsenter in order to do the -// requested mount. -func (n *Mounter) makeNsenterArgs(source, target, fstype string, options []string) (string, []string) { - mountCmd := n.ne.AbsHostPath("mount") - mountArgs := mount.MakeMountArgs(source, target, fstype, options) - - if systemdRunPath, hasSystemd := n.ne.SupportsSystemd(); hasSystemd { - // Complete command line: - // nsenter --mount=/rootfs/proc/1/ns/mnt -- /bin/systemd-run --description=... --scope -- /bin/mount -t - // Expected flow is: - // * nsenter breaks out of container's mount namespace and executes - // host's systemd-run. - // * systemd-run creates a transient scope (=~ cgroup) and executes its - // argument (/bin/mount) there. - // * mount does its job, forks a fuse daemon if necessary and finishes. - // (systemd-run --scope finishes at this point, returning mount's exit - // code and stdout/stderr - thats one of --scope benefits). - // * systemd keeps the fuse daemon running in the scope (i.e. in its own - // cgroup) until the fuse daemon dies (another --scope benefit). - // Kubelet container can be restarted and the fuse daemon survives. - // * When the daemon dies (e.g. during unmount) systemd removes the - // scope automatically. - mountCmd, mountArgs = mount.AddSystemdScope(systemdRunPath, target, mountCmd, mountArgs) - } else { - // Fall back to simple mount when the host has no systemd. - // Complete command line: - // nsenter --mount=/rootfs/proc/1/ns/mnt -- /bin/mount -t - // Expected flow is: - // * nsenter breaks out of container's mount namespace and executes host's /bin/mount. - // * mount does its job, forks a fuse daemon if necessary and finishes. - // * Any fuse daemon runs in cgroup of kubelet docker container, - // restart of kubelet container will kill it! - - // No code here, mountCmd and mountArgs use /bin/mount - } - - return mountCmd, mountArgs -} - -// Unmount runs umount(8) in the host's mount namespace. -func (n *Mounter) Unmount(target string) error { - args := []string{target} - // No need to execute systemd-run here, it's enough that unmount is executed - // in the host's mount namespace. It will finish appropriate fuse daemon(s) - // running in any scope. - klog.V(5).Infof("nsenter unmount args: %v", args) - outputBytes, err := n.ne.Exec("umount", args).CombinedOutput() - if len(outputBytes) != 0 { - klog.V(5).Infof("Output of unmounting %s: %v", target, string(outputBytes)) - } - return err -} - -// List returns a list of all mounted filesystems in the host's mount namespace. -func (*Mounter) List() ([]mount.MountPoint, error) { - return mount.ListProcMounts(hostProcMountsPath) -} - -// IsMountPointMatch tests if dir and mp are the same path -func (*Mounter) IsMountPointMatch(mp mount.MountPoint, dir string) bool { - deletedDir := fmt.Sprintf("%s\\040(deleted)", dir) - return (mp.Path == dir) || (mp.Path == deletedDir) -} - -// IsLikelyNotMountPoint determines whether a path is a mountpoint by calling findmnt -// in the host's root mount namespace. -func (n *Mounter) IsLikelyNotMountPoint(file string) (bool, error) { - file, err := filepath.Abs(file) - if err != nil { - return true, err - } - - // Check the directory exists - if _, err = os.Stat(file); os.IsNotExist(err) { - klog.V(5).Infof("findmnt: directory %s does not exist", file) - return true, err - } - - // Resolve any symlinks in file, kernel would do the same and use the resolved path in /proc/mounts - hu := NewHostUtil(n.ne, n.rootDir) - resolvedFile, err := hu.EvalHostSymlinks(file) - if err != nil { - return true, err - } - - // Add --first-only option: since we are testing for the absence of a mountpoint, it is sufficient to get only - // the first of multiple possible mountpoints using --first-only. - // Also add fstype output to make sure that the output of target file will give the full path - // TODO: Need more refactoring for this function. Track the solution with issue #26996 - args := []string{"-o", "target,fstype", "--noheadings", "--first-only", "--target", resolvedFile} - klog.V(5).Infof("nsenter findmnt args: %v", args) - out, err := n.ne.Exec("findmnt", args).CombinedOutput() - if err != nil { - klog.V(2).Infof("Failed findmnt command for path %s: %s %v", resolvedFile, out, err) - // Different operating systems behave differently for paths which are not mount points. - // On older versions (e.g. 2.20.1) we'd get error, on newer ones (e.g. 2.26.2) we'd get "/". - // It's safer to assume that it's not a mount point. - return true, nil - } - mountTarget, err := parseFindMnt(string(out)) - if err != nil { - return false, err - } - - klog.V(5).Infof("IsLikelyNotMountPoint findmnt output for path %s: %v:", resolvedFile, mountTarget) - - if mountTarget == resolvedFile { - klog.V(5).Infof("IsLikelyNotMountPoint: %s is a mount point", resolvedFile) - return false, nil - } - klog.V(5).Infof("IsLikelyNotMountPoint: %s is not a mount point", resolvedFile) - return true, nil -} - -// parse output of "findmnt -o target,fstype" and return just the target -func parseFindMnt(out string) (string, error) { - // cut trailing newline - out = strings.TrimSuffix(out, "\n") - // cut everything after the last space - it's the filesystem type - i := strings.LastIndex(out, " ") - if i == -1 { - return "", fmt.Errorf("error parsing findmnt output, expected at least one space: %q", out) - } - return out[:i], nil -} - -// GetMountRefs finds all mount references to the path, returns a -// list of paths. Path could be a mountpoint path, device or a normal -// directory (for bind mount). -func (n *Mounter) GetMountRefs(pathname string) ([]string, error) { - pathExists, pathErr := mount.PathExists(pathname) - if !pathExists || mount.IsCorruptedMnt(pathErr) { - return []string{}, nil - } else if pathErr != nil { - return nil, fmt.Errorf("Error checking path %s: %v", pathname, pathErr) - } - hostpath, err := n.ne.EvalSymlinks(pathname, true /* mustExist */) - if err != nil { - return nil, err - } - return mount.SearchMountPoints(hostpath, hostProcMountinfoPath) -} - -type hostUtil struct { - ne *nsenter.Nsenter - rootDir string -} - -// hostUtil implements mount.HostUtils -var _ = mount.HostUtils(&hostUtil{}) - -// NewHostUtil returns a new mount.HostUtils implementation that works -// for kubelet running in a container -func NewHostUtil(ne *nsenter.Nsenter, rootDir string) mount.HostUtils { - return &hostUtil{ne: ne, rootDir: rootDir} -} - -// DeviceOpened checks if block device in use by calling Open with O_EXCL flag. -// Returns true if open returns errno EBUSY, and false if errno is nil. -// Returns an error if errno is any error other than EBUSY. -// Returns with error if pathname is not a device. -func (hu *hostUtil) DeviceOpened(pathname string) (bool, error) { - return mount.ExclusiveOpenFailsOnDevice(pathname) -} - -// PathIsDevice uses FileInfo returned from os.Stat to check if path refers -// to a device. -func (hu *hostUtil) PathIsDevice(pathname string) (bool, error) { - pathType, err := hu.GetFileType(pathname) - isDevice := pathType == mount.FileTypeCharDev || pathType == mount.FileTypeBlockDev - return isDevice, err -} - -//GetDeviceNameFromMount given a mount point, find the volume id from checking /proc/mounts -func (hu *hostUtil) GetDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) { - return mount.GetDeviceNameFromMountLinux(mounter, mountPath, pluginMountDir) -} - -// MakeRShared checks if path is shared and bind-mounts it as rshared if needed. -func (hu *hostUtil) MakeRShared(path string) error { - return mount.DoMakeRShared(path, hostProcMountinfoPath) -} - -// GetFileType checks for file/directory/socket/block/character devices. -func (hu *hostUtil) GetFileType(pathname string) (mount.FileType, error) { - var pathType mount.FileType - outputBytes, err := hu.ne.Exec("stat", []string{"-L", "--printf=%F", pathname}).CombinedOutput() - if err != nil { - if strings.Contains(string(outputBytes), "No such file") { - err = fmt.Errorf("%s does not exist", pathname) - } else { - err = fmt.Errorf("stat %s error: %v", pathname, string(outputBytes)) - } - return pathType, err - } - - switch string(outputBytes) { - case "socket": - return mount.FileTypeSocket, nil - case "character special file": - return mount.FileTypeCharDev, nil - case "block special file": - return mount.FileTypeBlockDev, nil - case "directory": - return mount.FileTypeDirectory, nil - case "regular file", "regular empty file": - return mount.FileTypeFile, nil - } - - return pathType, fmt.Errorf("only recognise file, directory, socket, block device and character device") -} - -// MakeDir creates a new directory. -func (hu *hostUtil) MakeDir(pathname string) error { - args := []string{"-p", pathname} - if _, err := hu.ne.Exec("mkdir", args).CombinedOutput(); err != nil { - return err - } - return nil -} - -// MakeFile creates an empty file. -func (hu *hostUtil) MakeFile(pathname string) error { - args := []string{pathname} - if _, err := hu.ne.Exec("touch", args).CombinedOutput(); err != nil { - return err - } - return nil -} - -// PathExists checks if pathname exists. -// Error is returned on any other error than "file not found". -func (hu *hostUtil) PathExists(pathname string) (bool, error) { - // Resolve the symlinks but allow the target not to exist. EvalSymlinks - // would return an generic error when the target does not exist. - hostPath, err := hu.ne.EvalSymlinks(pathname, false /* mustExist */) - if err != nil { - return false, err - } - kubeletpath := hu.ne.KubeletPath(hostPath) - return utilpath.Exists(utilpath.CheckFollowSymlink, kubeletpath) -} - -// EvalHostSymlinks returns the path name after evaluating symlinks. -func (hu *hostUtil) EvalHostSymlinks(pathname string) (string, error) { - return hu.ne.EvalSymlinks(pathname, true) -} - -// GetOwner returns the integer ID for the user and group of the given path -func (hu *hostUtil) GetOwner(pathname string) (int64, int64, error) { - hostPath, err := hu.ne.EvalSymlinks(pathname, true /* mustExist */) - if err != nil { - return -1, -1, err - } - kubeletpath := hu.ne.KubeletPath(hostPath) - return mount.GetOwnerLinux(kubeletpath) -} - -// GetSELinuxSupport tests if pathname is on a mount that supports SELinux. -func (hu *hostUtil) GetSELinuxSupport(pathname string) (bool, error) { - return mount.GetSELinux(pathname, hostProcMountsPath) -} - -// GetMode returns permissions of pathname. -func (hu *hostUtil) GetMode(pathname string) (os.FileMode, error) { - hostPath, err := hu.ne.EvalSymlinks(pathname, true /* mustExist */) - if err != nil { - return 0, err - } - kubeletpath := hu.ne.KubeletPath(hostPath) - return mount.GetModeLinux(kubeletpath) -} diff --git a/pkg/volume/util/nsenter/nsenter_mount_test.go b/pkg/volume/util/nsenter/nsenter_mount_test.go deleted file mode 100644 index 2df7c6d1cb2..00000000000 --- a/pkg/volume/util/nsenter/nsenter_mount_test.go +++ /dev/null @@ -1,434 +0,0 @@ -// +build linux - -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package nsenter - -import ( - "io/ioutil" - "os" - "os/user" - "path/filepath" - "testing" - - "k8s.io/kubernetes/pkg/util/mount" - "k8s.io/utils/nsenter" -) - -func TestParseFindMnt(t *testing.T) { - tests := []struct { - input string - target string - expectError bool - }{ - { - // standard mount name, e.g. for AWS - "/var/lib/kubelet/plugins/kubernetes.io/aws-ebs/mounts/aws/us-east-1d/vol-020f82b0759f72389 ext4\n", - "/var/lib/kubelet/plugins/kubernetes.io/aws-ebs/mounts/aws/us-east-1d/vol-020f82b0759f72389", - false, - }, - { - // mount name with space, e.g. vSphere - "/var/lib/kubelet/plugins/kubernetes.io/vsphere-volume/mounts/[datastore1] kubevols/kubernetes-dynamic-pvc-4aacaa9b-6ba5-11e7-8f64-0050569f1b82.vmdk ext2\n", - "/var/lib/kubelet/plugins/kubernetes.io/vsphere-volume/mounts/[datastore1] kubevols/kubernetes-dynamic-pvc-4aacaa9b-6ba5-11e7-8f64-0050569f1b82.vmdk", - false, - }, - { - // hypotetic mount with several spaces - "/var/lib/kubelet/plugins/kubernetes.io/vsphere-volume/mounts/[ d a t a s t o r e 1 ] kubevols/kubernetes-dynamic-pvc-4aacaa9b-6ba5-11e7-8f64-0050569f1b82.vmdk ext2\n", - "/var/lib/kubelet/plugins/kubernetes.io/vsphere-volume/mounts/[ d a t a s t o r e 1 ] kubevols/kubernetes-dynamic-pvc-4aacaa9b-6ba5-11e7-8f64-0050569f1b82.vmdk", - false, - }, - { - // invalid output - no filesystem type - "/var/lib/kubelet/plugins/kubernetes.io/vsphere-volume/mounts/blabla", - "", - true, - }, - } - - for i, test := range tests { - target, err := parseFindMnt(test.input) - if test.expectError && err == nil { - t.Errorf("test %d expected error, got nil", i) - } - if !test.expectError && err != nil { - t.Errorf("test %d returned error: %s", i, err) - } - if target != test.target { - t.Errorf("test %d expected %q, got %q", i, test.target, target) - } - } -} - -func newFakeNsenterHostUtil(tmpdir string, t *testing.T) (mount.HostUtils, string, string, error) { - rootfsPath := filepath.Join(tmpdir, "rootfs") - - if err := os.Mkdir(rootfsPath, 0755); err != nil { - return nil, "", "", err - } - ne, err := nsenter.NewFakeNsenter(rootfsPath) - if err != nil { - return nil, "", "", err - } - - varlibPath := filepath.Join(tmpdir, "var/lib/kubelet") - if err := os.MkdirAll(varlibPath, 0755); err != nil { - return nil, "", "", err - } - - hu := NewHostUtil(ne, varlibPath) - - return hu, rootfsPath, varlibPath, nil -} - -func TestNsenterExistsFile(t *testing.T) { - var isRoot bool - usr, err := user.Current() - if err == nil { - isRoot = usr.Username == "root" - } else { - switch err.(type) { - case user.UnknownUserIdError: - // Root should be always known, this is some random UID - isRoot = false - default: - t.Fatal(err) - } - } - - tests := []struct { - name string - prepare func(base, rootfs string) (string, error) - expectedOutput bool - expectError bool - }{ - { - name: "simple existing file", - prepare: func(base, rootfs string) (string, error) { - // On the host: /base/file - path := filepath.Join(base, "file") - if err := ioutil.WriteFile(path, []byte{}, 0644); err != nil { - return "", err - } - // In kubelet: /rootfs/base/file - if _, err := writeRootfsFile(rootfs, path, 0644); err != nil { - return "", err - } - return path, nil - }, - expectedOutput: true, - }, - { - name: "simple non-existing file", - prepare: func(base, rootfs string) (string, error) { - path := filepath.Join(base, "file") - return path, nil - }, - expectedOutput: false, - }, - { - name: "simple non-accessible file", - prepare: func(base, rootfs string) (string, error) { - // On the host: - // create /base/dir/file, then make the dir inaccessible - dir := filepath.Join(base, "dir") - if err := os.MkdirAll(dir, 0755); err != nil { - return "", err - } - path := filepath.Join(dir, "file") - if err := ioutil.WriteFile(path, []byte{}, 0); err != nil { - return "", err - } - if err := os.Chmod(dir, 0644); err != nil { - return "", err - } - - // In kubelet: do the same with /rootfs/base/dir/file - rootfsPath, err := writeRootfsFile(rootfs, path, 0777) - if err != nil { - return "", err - } - rootfsDir := filepath.Dir(rootfsPath) - if err := os.Chmod(rootfsDir, 0644); err != nil { - return "", err - } - - return path, nil - }, - expectedOutput: isRoot, // PathExists success when running as root - expectError: !isRoot, // PathExists must fail when running as not-root - }, - { - name: "relative symlink to existing file", - prepare: func(base, rootfs string) (string, error) { - // On the host: /base/link -> file - file := filepath.Join(base, "file") - if err := ioutil.WriteFile(file, []byte{}, 0); err != nil { - return "", err - } - path := filepath.Join(base, "link") - if err := os.Symlink("file", path); err != nil { - return "", err - } - // In kubelet: /rootfs/base/file - if _, err := writeRootfsFile(rootfs, file, 0644); err != nil { - return "", err - } - return path, nil - }, - expectedOutput: true, - }, - { - name: "absolute symlink to existing file", - prepare: func(base, rootfs string) (string, error) { - // On the host: /base/link -> /base/file - file := filepath.Join(base, "file") - if err := ioutil.WriteFile(file, []byte{}, 0); err != nil { - return "", err - } - path := filepath.Join(base, "link") - if err := os.Symlink(file, path); err != nil { - return "", err - } - // In kubelet: /rootfs/base/file - if _, err := writeRootfsFile(rootfs, file, 0644); err != nil { - return "", err - } - - return path, nil - }, - expectedOutput: true, - }, - { - name: "relative symlink to non-existing file", - prepare: func(base, rootfs string) (string, error) { - path := filepath.Join(base, "link") - if err := os.Symlink("file", path); err != nil { - return "", err - } - return path, nil - }, - expectedOutput: false, - }, - { - name: "absolute symlink to non-existing file", - prepare: func(base, rootfs string) (string, error) { - file := filepath.Join(base, "file") - path := filepath.Join(base, "link") - if err := os.Symlink(file, path); err != nil { - return "", err - } - return path, nil - }, - expectedOutput: false, - }, - { - name: "symlink loop", - prepare: func(base, rootfs string) (string, error) { - path := filepath.Join(base, "link") - if err := os.Symlink(path, path); err != nil { - return "", err - } - return path, nil - }, - expectedOutput: false, - // TODO: realpath -m is not able to detect symlink loop. Should we care? - expectError: false, - }, - } - - for _, test := range tests { - tmpdir, err := ioutil.TempDir("", "nsenter-exists-file") - if err != nil { - t.Error(err) - continue - } - defer os.RemoveAll(tmpdir) - - testBase := filepath.Join(tmpdir, "base") - if err := os.Mkdir(testBase, 0755); err != nil { - t.Error(err) - continue - } - - hu, rootfs, _, err := newFakeNsenterHostUtil(tmpdir, t) - if err != nil { - t.Error(err) - continue - } - - path, err := test.prepare(testBase, rootfs) - if err != nil { - t.Error(err) - continue - } - - out, err := hu.PathExists(path) - if err != nil && !test.expectError { - t.Errorf("Test %q: unexpected error: %s", test.name, err) - } - if err == nil && test.expectError { - t.Errorf("Test %q: expected error, got none", test.name) - } - - if out != test.expectedOutput { - t.Errorf("Test %q: expected return value %v, got %v", test.name, test.expectedOutput, out) - } - } -} - -func TestNsenterGetMode(t *testing.T) { - tests := []struct { - name string - prepare func(base, rootfs string) (string, error) - expectedMode os.FileMode - expectError bool - }{ - { - name: "simple file", - prepare: func(base, rootfs string) (string, error) { - // On the host: /base/file - path := filepath.Join(base, "file") - if err := ioutil.WriteFile(path, []byte{}, 0644); err != nil { - return "", err - } - - // Prepare a different file as /rootfs/base/file (="the host - // visible from container") to check that NsEnterMounter calls - // stat on this file and not on /base/file. - // Visible from kubelet: /rootfs/base/file - if _, err := writeRootfsFile(rootfs, path, 0777); err != nil { - return "", err - } - - return path, nil - }, - expectedMode: 0777, - }, - { - name: "non-existing file", - prepare: func(base, rootfs string) (string, error) { - path := filepath.Join(base, "file") - return path, nil - }, - expectedMode: 0, - expectError: true, - }, - { - name: "absolute symlink to existing file", - prepare: func(base, rootfs string) (string, error) { - // On the host: /base/link -> /base/file - file := filepath.Join(base, "file") - if err := ioutil.WriteFile(file, []byte{}, 0644); err != nil { - return "", err - } - path := filepath.Join(base, "link") - if err := os.Symlink(file, path); err != nil { - return "", err - } - - // Visible from kubelet: - // /rootfs/base/file - if _, err := writeRootfsFile(rootfs, file, 0747); err != nil { - return "", err - } - - return path, nil - }, - expectedMode: 0747, - }, - { - name: "relative symlink to existing file", - prepare: func(base, rootfs string) (string, error) { - // On the host: /base/link -> file - file := filepath.Join(base, "file") - if err := ioutil.WriteFile(file, []byte{}, 0741); err != nil { - return "", err - } - path := filepath.Join(base, "link") - if err := os.Symlink("file", path); err != nil { - return "", err - } - - // Visible from kubelet: - // /rootfs/base/file - if _, err := writeRootfsFile(rootfs, file, 0647); err != nil { - return "", err - } - - return path, nil - }, - expectedMode: 0647, - }, - } - - for _, test := range tests { - tmpdir, err := ioutil.TempDir("", "nsenter-get-mode-") - if err != nil { - t.Error(err) - continue - } - defer os.RemoveAll(tmpdir) - - testBase := filepath.Join(tmpdir, "base") - if err := os.Mkdir(testBase, 0755); err != nil { - t.Error(err) - continue - } - - hu, rootfs, _, err := newFakeNsenterHostUtil(tmpdir, t) - if err != nil { - t.Error(err) - continue - } - - path, err := test.prepare(testBase, rootfs) - if err != nil { - t.Error(err) - continue - } - - mode, err := hu.GetMode(path) - if err != nil && !test.expectError { - t.Errorf("Test %q: unexpected error: %s", test.name, err) - } - if err == nil && test.expectError { - t.Errorf("Test %q: expected error, got none", test.name) - } - - if mode != test.expectedMode { - t.Errorf("Test %q: expected return value %v, got %v", test.name, test.expectedMode, mode) - } - } -} - -func writeRootfsFile(rootfs, path string, mode os.FileMode) (string, error) { - fullPath := filepath.Join(rootfs, path) - dir := filepath.Dir(fullPath) - if err := os.MkdirAll(dir, 0755); err != nil { - return "", err - } - if err := ioutil.WriteFile(fullPath, []byte{}, mode); err != nil { - return "", err - } - // Use chmod, io.WriteFile is affected by umask - if err := os.Chmod(fullPath, mode); err != nil { - return "", err - } - return fullPath, nil -} diff --git a/pkg/volume/util/nsenter/nsenter_mount_unsupported.go b/pkg/volume/util/nsenter/nsenter_mount_unsupported.go deleted file mode 100644 index abe0a9d8dc7..00000000000 --- a/pkg/volume/util/nsenter/nsenter_mount_unsupported.go +++ /dev/null @@ -1,151 +0,0 @@ -// +build !linux - -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package nsenter - -import ( - "errors" - "os" - - "k8s.io/utils/nsenter" - - "k8s.io/kubernetes/pkg/util/mount" -) - -// Mounter provides the mount.Interface implementation for unsupported -// platforms. -type Mounter struct{} - -// NewMounter returns a new Mounter for the current system -func NewMounter(rootDir string, ne *nsenter.Nsenter) *Mounter { - return &Mounter{} -} - -var _ = mount.Interface(&Mounter{}) - -// Mount mounts the source to the target. It is a noop for unsupported systems -func (*Mounter) Mount(source string, target string, fstype string, options []string) error { - return nil -} - -// Unmount unmounts the target path from the system. it is a noop for unsupported -// systems -func (*Mounter) Unmount(target string) error { - return nil -} - -// List returns a list of all mounted filesystems. It is a noop for unsupported systems -func (*Mounter) List() ([]mount.MountPoint, error) { - return []mount.MountPoint{}, nil -} - -// IsMountPointMatch tests if dir and mp are the same path -func (*Mounter) IsMountPointMatch(mp mount.MountPoint, dir string) bool { - return (mp.Path == dir) -} - -// IsLikelyNotMountPoint determines if a directory is not a mountpoint. -// It is a noop on unsupported systems -func (*Mounter) IsLikelyNotMountPoint(file string) (bool, error) { - return true, nil -} - -// GetMountRefs finds all mount references to the path, returns a -// list of paths. Always returns an error on unsupported platforms -func (*Mounter) GetMountRefs(pathname string) ([]string, error) { - return nil, errors.New("not implemented") -} - -type hostUtil struct { -} - -// hostUtil implements mount.HostUtils -var _ = mount.HostUtils(&hostUtil{}) - -// NewHostUtil returns a new implementation of mount.HostUtils for unsupported -// platforms -func NewHostUtil(ne *nsenter.Nsenter, rootDir string) mount.HostUtils { - return &hostUtil{} -} - -// DeviceOpened checks if block device in use. I tis a noop for unsupported systems -func (*hostUtil) DeviceOpened(pathname string) (bool, error) { - return false, nil -} - -// PathIsDevice checks if pathname refers to a device. It is a noop for unsupported -// systems -func (*hostUtil) PathIsDevice(pathname string) (bool, error) { - return true, nil -} - -// GetDeviceNameFromMount finds the device name from its global mount point using the -// given mountpath and plugin location. It is a noop of unsupported platforms -func (*hostUtil) GetDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) { - return "", nil -} - -// MakeRShared checks if path is shared and bind-mounts it as rshared if needed. -// It is a noop on unsupported platforms -func (*hostUtil) MakeRShared(path string) error { - return nil -} - -// GetFileType checks for file/directory/socket/block/character devices. -// Always returns an error and "fake" filetype on unsupported platforms -func (*hostUtil) GetFileType(_ string) (mount.FileType, error) { - return mount.FileType("fake"), errors.New("not implemented") -} - -// MakeDir creates a new directory. Noop on unsupported platforms -func (*hostUtil) MakeDir(pathname string) error { - return nil -} - -// MakeFile creats an empty file. Noop on unsupported platforms -func (*hostUtil) MakeFile(pathname string) error { - return nil -} - -// PathExists checks if pathname exists. Always returns an error on unsupported -// platforms -func (*hostUtil) PathExists(pathname string) (bool, error) { - return true, errors.New("not implemented") -} - -// EvalHostSymlinks returns the path name after evaluating symlinks. Always -// returns an error on unsupported platforms -func (*hostUtil) EvalHostSymlinks(pathname string) (string, error) { - return "", errors.New("not implemented") -} - -// GetOwner returns the integer ID for the user and group of the given path -func (*hostUtil) GetOwner(pathname string) (int64, int64, error) { - return -1, -1, errors.New("not implemented") -} - -// GetSELinuxSupport tests if pathname is on a mount that supports SELinux. -// Always returns an error on unsupported platforms -func (*hostUtil) GetSELinuxSupport(pathname string) (bool, error) { - return false, errors.New("not implemented") -} - -// GetMode returns permissions of pathname. Always returns an error on unsupported platforms -func (*hostUtil) GetMode(pathname string) (os.FileMode, error) { - return 0, errors.New("not implemented") -} diff --git a/pkg/volume/util/subpath/BUILD b/pkg/volume/util/subpath/BUILD index a5b54c47dea..64e11ad23d1 100644 --- a/pkg/volume/util/subpath/BUILD +++ b/pkg/volume/util/subpath/BUILD @@ -5,7 +5,6 @@ go_library( srcs = [ "subpath.go", "subpath_linux.go", - "subpath_nsenter.go", "subpath_unsupported.go", "subpath_windows.go", ], @@ -32,7 +31,6 @@ go_library( "//pkg/util/mount:go_default_library", "//vendor/golang.org/x/sys/unix:go_default_library", "//vendor/k8s.io/klog:go_default_library", - "//vendor/k8s.io/utils/nsenter:go_default_library", ], "@io_bazel_rules_go//go/platform:nacl": [ "//pkg/util/mount:go_default_library", @@ -67,17 +65,13 @@ go_test( name = "go_default_test", srcs = [ "subpath_linux_test.go", - "subpath_nsenter_test.go", "subpath_windows_test.go", ], embed = [":go_default_library"], deps = select({ "@io_bazel_rules_go//go/platform:linux": [ "//pkg/util/mount:go_default_library", - "//pkg/volume/util/nsenter:go_default_library", - "//vendor/golang.org/x/sys/unix:go_default_library", "//vendor/k8s.io/klog:go_default_library", - "//vendor/k8s.io/utils/nsenter:go_default_library", ], "@io_bazel_rules_go//go/platform:windows": [ "//vendor/github.com/stretchr/testify/assert:go_default_library", diff --git a/pkg/volume/util/subpath/subpath_nsenter.go b/pkg/volume/util/subpath/subpath_nsenter.go deleted file mode 100644 index b24379adbbf..00000000000 --- a/pkg/volume/util/subpath/subpath_nsenter.go +++ /dev/null @@ -1,186 +0,0 @@ -// +build linux - -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package subpath - -import ( - "fmt" - "os" - "path/filepath" - "syscall" - - "golang.org/x/sys/unix" - - "k8s.io/klog" - "k8s.io/utils/nsenter" - - "k8s.io/kubernetes/pkg/util/mount" -) - -type subpathNSE struct { - mounter mount.Interface - ne *nsenter.Nsenter - rootDir string -} - -// Compile time-check for all implementers of subpath interface -var _ Interface = &subpathNSE{} - -// NewNSEnter returns a subpath.Interface that is to be used with the NsenterMounter -// It is only valid on Linux systems -func NewNSEnter(mounter mount.Interface, ne *nsenter.Nsenter, rootDir string) Interface { - return &subpathNSE{ - mounter: mounter, - ne: ne, - rootDir: rootDir, - } -} - -func (sp *subpathNSE) CleanSubPaths(podDir string, volumeName string) error { - return doCleanSubPaths(sp.mounter, podDir, volumeName) -} - -func (sp *subpathNSE) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { - // Bind-mount the subpath to avoid using symlinks in subpaths. - newHostPath, err = sp.doNsEnterBindSubPath(subPath) - - // There is no action when the container starts. Bind-mount will be cleaned - // when container stops by CleanSubPaths. - cleanupAction = nil - return newHostPath, cleanupAction, err -} - -func (sp *subpathNSE) SafeMakeDir(subdir string, base string, perm os.FileMode) error { - fullSubdirPath := filepath.Join(base, subdir) - evaluatedSubdirPath, err := sp.ne.EvalSymlinks(fullSubdirPath, false /* mustExist */) - if err != nil { - return fmt.Errorf("error resolving symlinks in %s: %s", fullSubdirPath, err) - } - evaluatedSubdirPath = filepath.Clean(evaluatedSubdirPath) - - evaluatedBase, err := sp.ne.EvalSymlinks(base, true /* mustExist */) - if err != nil { - return fmt.Errorf("error resolving symlinks in %s: %s", base, err) - } - evaluatedBase = filepath.Clean(evaluatedBase) - - rootDir := filepath.Clean(sp.rootDir) - if mount.PathWithinBase(evaluatedBase, rootDir) { - // Base is in /var/lib/kubelet. This directory is shared between the - // container with kubelet and the host. We don't need to add '/rootfs'. - // This is useful when /rootfs is mounted as read-only - we can still - // create subpaths for paths in /var/lib/kubelet. - return doSafeMakeDir(evaluatedSubdirPath, evaluatedBase, perm) - } - - // Base is somewhere on the host's filesystem. Add /rootfs and try to make - // the directory there. - // This requires /rootfs to be writable. - kubeletSubdirPath := sp.ne.KubeletPath(evaluatedSubdirPath) - kubeletBase := sp.ne.KubeletPath(evaluatedBase) - return doSafeMakeDir(kubeletSubdirPath, kubeletBase, perm) -} - -func (sp *subpathNSE) doNsEnterBindSubPath(subpath Subpath) (hostPath string, err error) { - // Linux, kubelet runs in a container: - // - safely open the subpath - // - bind-mount the subpath to target (this can be unsafe) - // - check that we mounted the right thing by comparing device ID and inode - // of the subpath (via safely opened fd) and the target (that's under our - // control) - - // Evaluate all symlinks here once for all subsequent functions. - evaluatedHostVolumePath, err := sp.ne.EvalSymlinks(subpath.VolumePath, true /*mustExist*/) - if err != nil { - return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err) - } - evaluatedHostSubpath, err := sp.ne.EvalSymlinks(subpath.Path, true /*mustExist*/) - if err != nil { - return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err) - } - klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, evaluatedHostSubpath, subpath.VolumePath) - subpath.VolumePath = sp.ne.KubeletPath(evaluatedHostVolumePath) - subpath.Path = sp.ne.KubeletPath(evaluatedHostSubpath) - - // Check the subpath is correct and open it - fd, err := safeOpenSubPath(sp.mounter, subpath) - if err != nil { - return "", err - } - defer syscall.Close(fd) - - alreadyMounted, bindPathTarget, err := prepareSubpathTarget(sp.mounter, subpath) - if err != nil { - return "", err - } - if alreadyMounted { - return bindPathTarget, nil - } - - success := false - defer func() { - // Cleanup subpath on error - if !success { - klog.V(4).Infof("doNsEnterBindSubPath() failed for %q, cleaning up subpath", bindPathTarget) - if cleanErr := cleanSubPath(sp.mounter, subpath); cleanErr != nil { - klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr) - } - } - }() - - // Leap of faith: optimistically expect that nobody has modified previously - // expanded evalSubPath with evil symlinks and bind-mount it. - // Mount is done on the host! don't use kubelet path! - klog.V(5).Infof("bind mounting %q at %q", evaluatedHostSubpath, bindPathTarget) - if err = sp.mounter.Mount(evaluatedHostSubpath, bindPathTarget, "" /*fstype*/, []string{"bind"}); err != nil { - return "", fmt.Errorf("error mounting %s: %s", evaluatedHostSubpath, err) - } - - // Check that the bind-mount target is the same inode and device as the - // source that we keept open, i.e. we mounted the right thing. - err = checkDeviceInode(fd, bindPathTarget) - if err != nil { - return "", fmt.Errorf("error checking bind mount for subpath %s: %s", subpath.VolumePath, err) - } - - success = true - klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget) - return bindPathTarget, nil -} - -// checkDeviceInode checks that opened file and path represent the same file. -func checkDeviceInode(fd int, path string) error { - var srcStat, dstStat unix.Stat_t - err := unix.Fstat(fd, &srcStat) - if err != nil { - return fmt.Errorf("error running fstat on subpath FD: %v", err) - } - - err = unix.Stat(path, &dstStat) - if err != nil { - return fmt.Errorf("error running fstat on %s: %v", path, err) - } - - if srcStat.Dev != dstStat.Dev { - return fmt.Errorf("different device number") - } - if srcStat.Ino != dstStat.Ino { - return fmt.Errorf("different inode") - } - return nil -} diff --git a/pkg/volume/util/subpath/subpath_nsenter_test.go b/pkg/volume/util/subpath/subpath_nsenter_test.go deleted file mode 100644 index b962f3354db..00000000000 --- a/pkg/volume/util/subpath/subpath_nsenter_test.go +++ /dev/null @@ -1,346 +0,0 @@ -// +build linux - -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package subpath - -import ( - "io/ioutil" - "os" - "path/filepath" - "strings" - "testing" - - "golang.org/x/sys/unix" - - "k8s.io/utils/nsenter" - - nsutil "k8s.io/kubernetes/pkg/volume/util/nsenter" -) - -func TestCheckDeviceInode(t *testing.T) { - testDir, err := ioutil.TempDir("", "nsenter-mounter-device-") - if err != nil { - t.Fatalf("Cannot create temporary directory: %s", err) - } - defer os.RemoveAll(testDir) - - tests := []struct { - name string - srcPath string - dstPath string - expectError string - }{ - { - name: "the same file", - srcPath: filepath.Join(testDir, "1"), - dstPath: filepath.Join(testDir, "1"), - expectError: "", - }, - { - name: "different file on the same FS", - srcPath: filepath.Join(testDir, "2.1"), - dstPath: filepath.Join(testDir, "2.2"), - expectError: "different inode", - }, - { - name: "different file on different device", - srcPath: filepath.Join(testDir, "3"), - // /proc is always on a different "device" than /tmp (or $TEMP) - dstPath: "/proc/self/status", - expectError: "different device", - }, - } - - for _, test := range tests { - if err := ioutil.WriteFile(test.srcPath, []byte{}, 0644); err != nil { - t.Errorf("Test %q: cannot create srcPath %s: %s", test.name, test.srcPath, err) - continue - } - - // Don't create dst if it exists - if _, err := os.Stat(test.dstPath); os.IsNotExist(err) { - if err := ioutil.WriteFile(test.dstPath, []byte{}, 0644); err != nil { - t.Errorf("Test %q: cannot create dstPath %s: %s", test.name, test.dstPath, err) - continue - } - } else if err != nil { - t.Errorf("Test %q: cannot check existence of dstPath %s: %s", test.name, test.dstPath, err) - continue - } - - fd, err := unix.Open(test.srcPath, unix.O_CREAT|unix.O_CLOEXEC, 0644) - if err != nil { - t.Errorf("Test %q: cannot open srcPath %s: %s", test.name, test.srcPath, err) - continue - } - - err = checkDeviceInode(fd, test.dstPath) - - if test.expectError == "" && err != nil { - t.Errorf("Test %q: expected no error, got %s", test.name, err) - } - if test.expectError != "" { - if err == nil { - t.Errorf("Test %q: expected error, got none", test.name) - } else { - if !strings.Contains(err.Error(), test.expectError) { - t.Errorf("Test %q: expected error %q, got %q", test.name, test.expectError, err) - } - } - } - } -} - -func newFakeNsenterMounter(tmpdir string, t *testing.T) (*nsutil.Mounter, string, string, *nsenter.Nsenter, error) { - rootfsPath := filepath.Join(tmpdir, "rootfs") - if err := os.Mkdir(rootfsPath, 0755); err != nil { - return nil, "", "", nil, err - } - ne, err := nsenter.NewFakeNsenter(rootfsPath) - if err != nil { - return nil, "", "", nil, err - } - - varlibPath := filepath.Join(tmpdir, "/var/lib/kubelet") - if err := os.MkdirAll(varlibPath, 0755); err != nil { - return nil, "", "", nil, err - } - - return nsutil.NewMounter(varlibPath, ne), rootfsPath, varlibPath, ne, nil -} - -func TestNsenterSafeMakeDir(t *testing.T) { - tests := []struct { - name string - prepare func(base, rootfs, varlib string) (expectedDir string, err error) - subdir string - expectError bool - // If true, "base" directory for SafeMakeDir will be /var/lib/kubelet - baseIsVarLib bool - }{ - { - name: "simple directory", - // evaluated in base - subdir: "some/subdirectory/structure", - prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { - // expected to be created in /roots/ - expectedDir = filepath.Join(rootfs, base, "some/subdirectory/structure") - return expectedDir, nil - }, - }, - { - name: "simple existing directory", - // evaluated in base - subdir: "some/subdirectory/structure", - prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { - // On the host: directory exists - hostPath := filepath.Join(base, "some/subdirectory/structure") - if err := os.MkdirAll(hostPath, 0755); err != nil { - return "", err - } - // In rootfs: directory exists - kubeletPath := filepath.Join(rootfs, hostPath) - if err := os.MkdirAll(kubeletPath, 0755); err != nil { - return "", err - } - // expected to be created in /roots/ - expectedDir = kubeletPath - return expectedDir, nil - }, - }, - { - name: "absolute symlink into safe place", - // evaluated in base - subdir: "some/subdirectory/structure", - prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { - // On the host: /base/other/subdirectory exists, /base/some is link to /base/other - hostPath := filepath.Join(base, "other/subdirectory") - if err := os.MkdirAll(hostPath, 0755); err != nil { - return "", err - } - somePath := filepath.Join(base, "some") - otherPath := filepath.Join(base, "other") - if err := os.Symlink(otherPath, somePath); err != nil { - return "", err - } - - // In rootfs: /base/other/subdirectory exists - kubeletPath := filepath.Join(rootfs, hostPath) - if err := os.MkdirAll(kubeletPath, 0755); err != nil { - return "", err - } - // expected 'structure' to be created - expectedDir = filepath.Join(rootfs, hostPath, "structure") - return expectedDir, nil - }, - }, - { - name: "relative symlink into safe place", - // evaluated in base - subdir: "some/subdirectory/structure", - prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { - // On the host: /base/other/subdirectory exists, /base/some is link to other - hostPath := filepath.Join(base, "other/subdirectory") - if err := os.MkdirAll(hostPath, 0755); err != nil { - return "", err - } - somePath := filepath.Join(base, "some") - if err := os.Symlink("other", somePath); err != nil { - return "", err - } - - // In rootfs: /base/other/subdirectory exists - kubeletPath := filepath.Join(rootfs, hostPath) - if err := os.MkdirAll(kubeletPath, 0755); err != nil { - return "", err - } - // expected 'structure' to be created - expectedDir = filepath.Join(rootfs, hostPath, "structure") - return expectedDir, nil - }, - }, - { - name: "symlink into unsafe place", - // evaluated in base - subdir: "some/subdirectory/structure", - prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { - // On the host: /base/some is link to /bin/other - somePath := filepath.Join(base, "some") - if err := os.Symlink("/bin", somePath); err != nil { - return "", err - } - return "", nil - }, - expectError: true, - }, - { - name: "simple directory in /var/lib/kubelet", - // evaluated in varlib - subdir: "some/subdirectory/structure", - baseIsVarLib: true, - prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { - // expected to be created in /base/var/lib/kubelet, not in /rootfs! - expectedDir = filepath.Join(varlib, "some/subdirectory/structure") - return expectedDir, nil - }, - }, - { - name: "safe symlink in /var/lib/kubelet", - // evaluated in varlib - subdir: "some/subdirectory/structure", - baseIsVarLib: true, - prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { - // On the host: /varlib/kubelet/other/subdirectory exists, /varlib/some is link to other - hostPath := filepath.Join(varlib, "other/subdirectory") - if err := os.MkdirAll(hostPath, 0755); err != nil { - return "", err - } - somePath := filepath.Join(varlib, "some") - if err := os.Symlink("other", somePath); err != nil { - return "", err - } - - // expected to be created in /base/var/lib/kubelet, not in /rootfs! - expectedDir = filepath.Join(varlib, "other/subdirectory/structure") - return expectedDir, nil - }, - }, - { - name: "unsafe symlink in /var/lib/kubelet", - // evaluated in varlib - subdir: "some/subdirectory/structure", - baseIsVarLib: true, - prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { - // On the host: /varlib/some is link to /bin - somePath := filepath.Join(varlib, "some") - if err := os.Symlink("/bin", somePath); err != nil { - return "", err - } - - return "", nil - }, - expectError: true, - }, - } - for _, test := range tests { - tmpdir, err := ioutil.TempDir("", "nsenter-get-safedir-") - if err != nil { - t.Error(err) - continue - } - defer os.RemoveAll(tmpdir) - - mounter, rootfs, varlib, ne, err := newFakeNsenterMounter(tmpdir, t) - if err != nil { - t.Error(err) - continue - } - - fsp := NewNSEnter(mounter, ne, varlib) - - // Prepare base directory for the test - testBase := filepath.Join(tmpdir, "base") - if err := os.Mkdir(testBase, 0755); err != nil { - t.Error(err) - continue - } - // Prepare base directory also in /rootfs - rootfsBase := filepath.Join(rootfs, testBase) - if err := os.MkdirAll(rootfsBase, 0755); err != nil { - t.Error(err) - continue - } - - expectedDir := "" - if test.prepare != nil { - expectedDir, err = test.prepare(testBase, rootfs, varlib) - if err != nil { - t.Error(err) - continue - } - } - - if test.baseIsVarLib { - // use /var/lib/kubelet as the test base so we can test creating - // subdirs there directly in /var/lib/kubenet and not in - // /rootfs/var/lib/kubelet - testBase = varlib - } - - err = fsp.SafeMakeDir(test.subdir, testBase, 0755) - if err != nil && !test.expectError { - t.Errorf("Test %q: unexpected error: %s", test.name, err) - } - if test.expectError { - if err == nil { - t.Errorf("Test %q: expected error, got none", test.name) - } else { - if !strings.Contains(err.Error(), "is outside of allowed base") { - t.Errorf("Test %q: expected error to contain \"is outside of allowed base\", got this one instead: %s", test.name, err) - } - } - } - - if expectedDir != "" { - _, err := os.Stat(expectedDir) - if err != nil { - t.Errorf("Test %q: expected %q to exist, got error: %s", test.name, expectedDir, err) - } - } - } -}