github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-25 12:43:23 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #63143 from jsafrane/containerized-subpath

Browse Source 
Automatic merge from submit-queue (batch tested with PRs 63348, 63839, 63143, 64447, 64567). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Containerized subpath

**What this PR does / why we need it**:
Containerized kubelet needs a different implementation of `PrepareSafeSubpath` than kubelet running directly on the host.

On the host we safely open the subpath and then bind-mount `/proc/<pidof kubelet>/fd/<descriptor of opened subpath>`.

With kubelet running in a container, `/proc/xxx/fd/yy` on the host contains path that works only inside the container, i.e. `/rootfs/path/to/subpath` and thus any bind-mount on the host fails.

Solution:
- safely open the subpath and gets its device ID and inode number
- blindly bind-mount the subpath to `/var/lib/kubelet/pods/<uid>/volume-subpaths/<name of container>/<id of mount>`. This is potentially unsafe, because user can change the subpath source to a link to a bad place (say `/run/docker.sock`) just before the bind-mount.
- get device ID and inode number of the destination. Typical users can't modify this file, as it lies on /var/lib/kubelet on the host.
- compare these device IDs and inode numbers.

**Which issue(s) this PR fixes**
Fixes #61456

**Special notes for your reviewer**:

The PR contains some refactoring of `doBindSubPath` to extract the common code. New `doNsEnterBindSubPath` is added for the nsenter related parts.

**Release note**:

```release-note
NONE
```
This commit is contained in:
Kubernetes Submit Queue 2018-06-01 12:12:19 -07:00 committed by GitHub
commit d2495b8329
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 1509 additions and 321 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/kubelet/app/BUILD
View File

@ -110,6 +110,7 @@ go_library(
"//pkg/util/io:go_default_library", "//pkg/util/io:go_default_library",
"//pkg/util/mount:go_default_library", "//pkg/util/mount:go_default_library",
"//pkg/util/node:go_default_library", "//pkg/util/node:go_default_library",
"//pkg/util/nsenter:go_default_library",
"//pkg/util/oom:go_default_library", "//pkg/util/oom:go_default_library",
"//pkg/util/rlimit:go_default_library", "//pkg/util/rlimit:go_default_library",
"//pkg/version:go_default_library", "//pkg/version:go_default_library",
@ -170,6 +171,7 @@ go_library(
"//vendor/k8s.io/client-go/tools/record:go_default_library", "//vendor/k8s.io/client-go/tools/record:go_default_library",
"//vendor/k8s.io/client-go/util/cert:go_default_library", "//vendor/k8s.io/client-go/util/cert:go_default_library",
"//vendor/k8s.io/client-go/util/certificate:go_default_library", "//vendor/k8s.io/client-go/util/certificate:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library",
] + select({ ] + select({
"@io_bazel_rules_go//go/platform:linux": [ "@io_bazel_rules_go//go/platform:linux": [
"//vendor/golang.org/x/exp/inotify:go_default_library", "//vendor/golang.org/x/exp/inotify:go_default_library",

7
cmd/kubelet/app/server.go
View File

@ -91,10 +91,12 @@ import (
kubeio "k8s.io/kubernetes/pkg/util/io" kubeio "k8s.io/kubernetes/pkg/util/io"
"k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/util/mount"
nodeutil "k8s.io/kubernetes/pkg/util/node" nodeutil "k8s.io/kubernetes/pkg/util/node"
"k8s.io/kubernetes/pkg/util/nsenter"
"k8s.io/kubernetes/pkg/util/oom" "k8s.io/kubernetes/pkg/util/oom"
"k8s.io/kubernetes/pkg/util/rlimit" "k8s.io/kubernetes/pkg/util/rlimit"
"k8s.io/kubernetes/pkg/version" "k8s.io/kubernetes/pkg/version"
"k8s.io/kubernetes/pkg/version/verflag" "k8s.io/kubernetes/pkg/version/verflag"
"k8s.io/utils/exec"
) )
const ( const (
@ -361,11 +363,12 @@ func UnsecuredDependencies(s *options.KubeletServer) (*kubelet.Dependencies, err
var writer kubeio.Writer = &kubeio.StdWriter{} var writer kubeio.Writer = &kubeio.StdWriter{}
if s.Containerized { if s.Containerized {
glog.V(2).Info("Running kubelet in containerized mode") glog.V(2).Info("Running kubelet in containerized mode")
mounter, err = mount.NewNsenterMounter() ne, err := nsenter.NewNsenter(nsenter.DefaultHostRootFsPath, exec.New())
if err != nil { if err != nil {
return nil, err return nil, err
} }
writer = &kubeio.NsenterWriter{} mounter = mount.NewNsenterMounter(s.RootDirectory, ne)
writer = kubeio.NewNsenterWriter(ne)
} }
var dockerClientConfig *dockershim.ClientConfig var dockerClientConfig *dockershim.ClientConfig

8
pkg/kubelet/cm/container_manager_linux_test.go
View File

@ -92,8 +92,8 @@ func (mi *fakeMountInterface) MakeFile(pathname string) error {
return nil return nil
} }
func (mi *fakeMountInterface) ExistsPath(pathname string) bool { func (mi *fakeMountInterface) ExistsPath(pathname string) (bool, error) {
return true return true, errors.New("not implemented")
} }
func (mi *fakeMountInterface) PrepareSafeSubpath(subPath mount.Subpath) (newHostPath string, cleanupAction func(), err error) { func (mi *fakeMountInterface) PrepareSafeSubpath(subPath mount.Subpath) (newHostPath string, cleanupAction func(), err error) {
@ -120,6 +120,10 @@ func (mi *fakeMountInterface) GetSELinuxSupport(pathname string) (bool, error) {
return false, errors.New("not implemented") return false, errors.New("not implemented")
} }
func (mi *fakeMountInterface) GetMode(pathname string) (os.FileMode, error) {
return 0, errors.New("not implemented")
}
func fakeContainerMgrMountInt() mount.Interface { func fakeContainerMgrMountInt() mount.Interface {
return &fakeMountInterface{ return &fakeMountInterface{
[]mount.MountPoint{ []mount.MountPoint{

23
pkg/kubelet/kubelet_pods.go
View File

@ -58,7 +58,6 @@ import (
"k8s.io/kubernetes/pkg/kubelet/status" "k8s.io/kubernetes/pkg/kubelet/status"
kubetypes "k8s.io/kubernetes/pkg/kubelet/types" kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
"k8s.io/kubernetes/pkg/kubelet/util/format" "k8s.io/kubernetes/pkg/kubelet/util/format"
utilfile "k8s.io/kubernetes/pkg/util/file"
mountutil "k8s.io/kubernetes/pkg/util/mount" mountutil "k8s.io/kubernetes/pkg/util/mount"
volumeutil "k8s.io/kubernetes/pkg/volume/util" volumeutil "k8s.io/kubernetes/pkg/volume/util"
"k8s.io/kubernetes/pkg/volume/util/volumepathhandler" "k8s.io/kubernetes/pkg/volume/util/volumepathhandler"
@ -179,19 +178,10 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h
return nil, cleanupAction, fmt.Errorf("unable to provision SubPath `%s`: %v", mount.SubPath, err) return nil, cleanupAction, fmt.Errorf("unable to provision SubPath `%s`: %v", mount.SubPath, err)
} }
fileinfo, err := os.Lstat(hostPath) volumePath := hostPath
if err != nil {
return nil, cleanupAction, err
}
perm := fileinfo.Mode()
volumePath, err := filepath.EvalSymlinks(hostPath)
if err != nil {
return nil, cleanupAction, err
}
hostPath = filepath.Join(volumePath, mount.SubPath) hostPath = filepath.Join(volumePath, mount.SubPath)
if subPathExists, err := utilfile.FileOrSymlinkExists(hostPath); err != nil { if subPathExists, err := mounter.ExistsPath(hostPath); err != nil {
glog.Errorf("Could not determine if subPath %s exists; will not attempt to change its permissions", hostPath) glog.Errorf("Could not determine if subPath %s exists; will not attempt to change its permissions", hostPath)
} else if !subPathExists { } else if !subPathExists {
// Create the sub path now because if it's auto-created later when referenced, it may have an // Create the sub path now because if it's auto-created later when referenced, it may have an
@ -199,10 +189,15 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h
// when the pod specifies an fsGroup, and if the directory is not created here, Docker will // when the pod specifies an fsGroup, and if the directory is not created here, Docker will
// later auto-create it with the incorrect mode 0750 // later auto-create it with the incorrect mode 0750
// Make extra care not to escape the volume! // Make extra care not to escape the volume!
if err := mounter.SafeMakeDir(hostPath, volumePath, perm); err != nil { perm, err := mounter.GetMode(volumePath)
glog.Errorf("failed to mkdir %q: %v", hostPath, err) if err != nil {
return nil, cleanupAction, err return nil, cleanupAction, err
} }
if err := mounter.SafeMakeDir(mount.SubPath, volumePath, perm); err != nil {
// Don't pass detailed error back to the user because it could give information about host filesystem
glog.Errorf("failed to create subPath directory for volumeMount %q of container %q: %v", mount.Name, container.Name, err)
return nil, cleanupAction, fmt.Errorf("failed to create subPath directory for volumeMount %q of container %q", mount.Name, container.Name)
}
} }
hostPath, cleanupAction, err = mounter.PrepareSafeSubpath(mountutil.Subpath{ hostPath, cleanupAction, err = mounter.PrepareSafeSubpath(mountutil.Subpath{
VolumeMountIndex: i, VolumeMountIndex: i,

20
pkg/util/io/writer.go
View File

@ -50,18 +50,24 @@ func (writer *StdWriter) WriteFile(filename string, data []byte, perm os.FileMod
// it will not see the mounted device in its own namespace. To work around this // it will not see the mounted device in its own namespace. To work around this
// limitation one has to first enter hosts namespace (by using 'nsenter') and // limitation one has to first enter hosts namespace (by using 'nsenter') and
// only then write data. // only then write data.
type NsenterWriter struct{} type NsenterWriter struct {
ne *nsenter.Nsenter
}
// NewNsenterWriter creates a new Writer that allows writing data to file using
// nsenter command.
func NewNsenterWriter(ne *nsenter.Nsenter) *NsenterWriter {
return &NsenterWriter{
ne: ne,
}
}
// WriteFile calls 'nsenter cat - > <the file>' and 'nsenter chmod' to create a // WriteFile calls 'nsenter cat - > <the file>' and 'nsenter chmod' to create a
// file on the host. // file on the host.
func (writer *NsenterWriter) WriteFile(filename string, data []byte, perm os.FileMode) error { func (writer *NsenterWriter) WriteFile(filename string, data []byte, perm os.FileMode) error {
ne, err := nsenter.NewNsenter()
if err != nil {
return err
}
echoArgs := []string{"-c", fmt.Sprintf("cat > %s", filename)} echoArgs := []string{"-c", fmt.Sprintf("cat > %s", filename)}
glog.V(5).Infof("nsenter: write data to file %s by nsenter", filename) glog.V(5).Infof("nsenter: write data to file %s by nsenter", filename)
command := ne.Exec("sh", echoArgs) command := writer.ne.Exec("sh", echoArgs)
command.SetStdin(bytes.NewBuffer(data)) command.SetStdin(bytes.NewBuffer(data))
outputBytes, err := command.CombinedOutput() outputBytes, err := command.CombinedOutput()
if err != nil { if err != nil {
@ -71,7 +77,7 @@ func (writer *NsenterWriter) WriteFile(filename string, data []byte, perm os.Fil
chmodArgs := []string{fmt.Sprintf("%o", perm), filename} chmodArgs := []string{fmt.Sprintf("%o", perm), filename}
glog.V(5).Infof("nsenter: change permissions of file %s to %s", filename, chmodArgs[0]) glog.V(5).Infof("nsenter: change permissions of file %s to %s", filename, chmodArgs[0])
outputBytes, err = ne.Exec("chmod", chmodArgs).CombinedOutput() outputBytes, err = writer.ne.Exec("chmod", chmodArgs).CombinedOutput()
if err != nil { if err != nil {
glog.Errorf("Output from chmod command: %v", string(outputBytes)) glog.Errorf("Output from chmod command: %v", string(outputBytes))
return err return err

34
pkg/util/mount/BUILD
View File

@ -71,12 +71,44 @@ go_library(
"//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library",
] + select({ ] + select({
"@io_bazel_rules_go//go/platform:android": [
"//pkg/util/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:darwin": [
"//pkg/util/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:dragonfly": [
"//pkg/util/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:freebsd": [
"//pkg/util/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:linux": [ "@io_bazel_rules_go//go/platform:linux": [
"//pkg/util/file:go_default_library",
"//pkg/util/io:go_default_library", "//pkg/util/io:go_default_library",
"//pkg/util/nsenter:go_default_library", "//pkg/util/nsenter:go_default_library",
"//vendor/golang.org/x/sys/unix:go_default_library", "//vendor/golang.org/x/sys/unix:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
], ],
"@io_bazel_rules_go//go/platform:nacl": [
"//pkg/util/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:netbsd": [
"//pkg/util/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:openbsd": [
"//pkg/util/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:plan9": [
"//pkg/util/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:solaris": [
"//pkg/util/nsenter:go_default_library",
],
"@io_bazel_rules_go//go/platform:windows": [
"//pkg/util/file:go_default_library",
"//pkg/util/nsenter:go_default_library",
],
"//conditions:default": [], "//conditions:default": [],
}), }),
) )
@ -101,7 +133,9 @@ go_test(
"//vendor/k8s.io/utils/exec/testing:go_default_library", "//vendor/k8s.io/utils/exec/testing:go_default_library",
] + select({ ] + select({
"@io_bazel_rules_go//go/platform:linux": [ "@io_bazel_rules_go//go/platform:linux": [
"//pkg/util/nsenter:go_default_library",
"//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/golang/glog:go_default_library",
"//vendor/golang.org/x/sys/unix:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library",
], ],
"@io_bazel_rules_go//go/platform:windows": [ "@io_bazel_rules_go//go/platform:windows": [

6
pkg/util/mount/exec_mount.go
View File

@ -136,7 +136,7 @@ func (m *execMounter) MakeDir(pathname string) error {
return m.wrappedMounter.MakeDir(pathname) return m.wrappedMounter.MakeDir(pathname)
} }
func (m *execMounter) ExistsPath(pathname string) bool { func (m *execMounter) ExistsPath(pathname string) (bool, error) {
return m.wrappedMounter.ExistsPath(pathname) return m.wrappedMounter.ExistsPath(pathname)
} }
@ -163,3 +163,7 @@ func (m *execMounter) GetFSGroup(pathname string) (int64, error) {
func (m *execMounter) GetSELinuxSupport(pathname string) (bool, error) { func (m *execMounter) GetSELinuxSupport(pathname string) (bool, error) {
return m.wrappedMounter.GetSELinuxSupport(pathname) return m.wrappedMounter.GetSELinuxSupport(pathname)
} }
func (m *execMounter) GetMode(pathname string) (os.FileMode, error) {
return m.wrappedMounter.GetMode(pathname)
}

8
pkg/util/mount/exec_mount_test.go
View File

@ -147,8 +147,8 @@ func (fm *fakeMounter) MakeFile(pathname string) error {
func (fm *fakeMounter) MakeDir(pathname string) error { func (fm *fakeMounter) MakeDir(pathname string) error {
return nil return nil
} }
func (fm *fakeMounter) ExistsPath(pathname string) bool { func (fm *fakeMounter) ExistsPath(pathname string) (bool, error) {
return false return false, errors.New("not implemented")
} }
func (fm *fakeMounter) GetFileType(pathname string) (FileType, error) { func (fm *fakeMounter) GetFileType(pathname string) (FileType, error) {
return FileTypeFile, nil return FileTypeFile, nil
@ -176,3 +176,7 @@ func (fm *fakeMounter) GetFSGroup(pathname string) (int64, error) {
func (fm *fakeMounter) GetSELinuxSupport(pathname string) (bool, error) { func (fm *fakeMounter) GetSELinuxSupport(pathname string) (bool, error) {
return false, errors.New("not implemented") return false, errors.New("not implemented")
} }
func (fm *fakeMounter) GetMode(pathname string) (os.FileMode, error) {
return 0, errors.New("not implemented")
}

8
pkg/util/mount/exec_mount_unsupported.go
View File

@ -83,8 +83,8 @@ func (mounter *execMounter) MakeFile(pathname string) error {
return nil return nil
} }
func (mounter *execMounter) ExistsPath(pathname string) bool { func (mounter *execMounter) ExistsPath(pathname string) (bool, error) {
return true return true, errors.New("not implemented")
} }
func (mounter *execMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { func (mounter *execMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
@ -110,3 +110,7 @@ func (mounter *execMounter) GetFSGroup(pathname string) (int64, error) {
func (mounter *execMounter) GetSELinuxSupport(pathname string) (bool, error) { func (mounter *execMounter) GetSELinuxSupport(pathname string) (bool, error) {
return false, errors.New("not implemented") return false, errors.New("not implemented")
} }
func (mounter *execMounter) GetMode(pathname string) (os.FileMode, error) {
return 0, errors.New("not implemented")
}

8
pkg/util/mount/fake.go
View File

@ -201,8 +201,8 @@ func (f *FakeMounter) MakeFile(pathname string) error {
return nil return nil
} }
func (f *FakeMounter) ExistsPath(pathname string) bool { func (f *FakeMounter) ExistsPath(pathname string) (bool, error) {
return false return false, errors.New("not implemented")
} }
func (f *FakeMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { func (f *FakeMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
@ -232,3 +232,7 @@ func (f *FakeMounter) GetFSGroup(pathname string) (int64, error) {
func (f *FakeMounter) GetSELinuxSupport(pathname string) (bool, error) { func (f *FakeMounter) GetSELinuxSupport(pathname string) (bool, error) {
return false, errors.New("GetSELinuxSupport not implemented") return false, errors.New("GetSELinuxSupport not implemented")
} }
func (f *FakeMounter) GetMode(pathname string) (os.FileMode, error) {
return 0, errors.New("not implemented")
}

24
pkg/util/mount/mount.go
View File

@ -84,16 +84,18 @@ type Interface interface {
// MakeDir creates a new directory. // MakeDir creates a new directory.
// Will operate in the host mount namespace if kubelet is running in a container // Will operate in the host mount namespace if kubelet is running in a container
MakeDir(pathname string) error MakeDir(pathname string) error
// SafeMakeDir makes sure that the created directory does not escape given // SafeMakeDir creates subdir within given base. It makes sure that the
// base directory mis-using symlinks. The directory is created in the same // created directory does not escape given base directory mis-using
// mount namespace as where kubelet is running. Note that the function makes // symlinks. Note that the function makes sure that it creates the directory
// sure that it creates the directory somewhere under the base, nothing // somewhere under the base, nothing else. E.g. if the directory already
// else. E.g. if the directory already exists, it may exists outside of the // exists, it may exist outside of the base due to symlinks.
// base due to symlinks. // This method should be used if the directory to create is inside volume
SafeMakeDir(pathname string, base string, perm os.FileMode) error // that's under user control. User must not be able to use symlinks to
// ExistsPath checks whether the path exists. // escape the volume to create directories somewhere else.
// Will operate in the host mount namespace if kubelet is running in a container SafeMakeDir(subdir string, base string, perm os.FileMode) error
ExistsPath(pathname string) bool // Will operate in the host mount namespace if kubelet is running in a container.
// Error is returned on any other error than "file not found".
ExistsPath(pathname string) (bool, error)
// CleanSubPaths removes any bind-mounts created by PrepareSafeSubpath in given // CleanSubPaths removes any bind-mounts created by PrepareSafeSubpath in given
// pod volume directory. // pod volume directory.
CleanSubPaths(podDir string, volumeName string) error CleanSubPaths(podDir string, volumeName string) error
@ -117,6 +119,8 @@ type Interface interface {
// GetSELinuxSupport returns true if given path is on a mount that supports // GetSELinuxSupport returns true if given path is on a mount that supports
// SELinux. // SELinux.
GetSELinuxSupport(pathname string) (bool, error) GetSELinuxSupport(pathname string) (bool, error)
// GetMode returns permissions of the path.
GetMode(pathname string) (os.FileMode, error)
} }
type Subpath struct { type Subpath struct {

198
pkg/util/mount/mount_linux.go
View File

@ -33,6 +33,7 @@ import (
"github.com/golang/glog" "github.com/golang/glog"
"golang.org/x/sys/unix" "golang.org/x/sys/unix"
"k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/sets"
utilfile "k8s.io/kubernetes/pkg/util/file"
utilio "k8s.io/kubernetes/pkg/util/io" utilio "k8s.io/kubernetes/pkg/util/io"
utilexec "k8s.io/utils/exec" utilexec "k8s.io/utils/exec"
) )
@ -449,12 +450,8 @@ func (mounter *Mounter) MakeFile(pathname string) error {
return nil return nil
} }
func (mounter *Mounter) ExistsPath(pathname string) bool { func (mounter *Mounter) ExistsPath(pathname string) (bool, error) {
_, err := os.Stat(pathname) return utilfile.FileExists(pathname)
if err != nil {
return false
}
return true
} }
// formatAndMount uses unix utils to format and mount the given disk // formatAndMount uses unix utils to format and mount the given disk
@ -760,7 +757,8 @@ func getSELinuxSupport(path string, mountInfoFilename string) (bool, error) {
} }
func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
newHostPath, err = doBindSubPath(mounter, subPath, os.Getpid()) newHostPath, err = doBindSubPath(mounter, subPath)
// There is no action when the container starts. Bind-mount will be cleaned // There is no action when the container starts. Bind-mount will be cleaned
// when container stops by CleanSubPaths. // when container stops by CleanSubPaths.
cleanupAction = nil cleanupAction = nil
@ -768,30 +766,107 @@ func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string,
} }
// This implementation is shared between Linux and NsEnterMounter // This implementation is shared between Linux and NsEnterMounter
// kubeletPid is PID of kubelet in the PID namespace where bind-mount is done, func safeOpenSubPath(mounter Interface, subpath Subpath) (int, error) {
// i.e. pid on the *host* if kubelet runs in a container. if !pathWithinBase(subpath.Path, subpath.VolumePath) {
func doBindSubPath(mounter Interface, subpath Subpath, kubeletPid int) (hostPath string, err error) { return -1, fmt.Errorf("subpath %q not within volume path %q", subpath.Path, subpath.VolumePath)
// Check early for symlink. This is just a pre-check to avoid bind-mount }
// before the final check. fd, err := doSafeOpen(subpath.Path, subpath.VolumePath)
evalSubPath, err := filepath.EvalSymlinks(subpath.Path)
if err != nil { if err != nil {
return "", fmt.Errorf("evalSymlinks %q failed: %v", subpath.Path, err) return -1, fmt.Errorf("error opening subpath %v: %v", subpath.Path, err)
} }
glog.V(5).Infof("doBindSubPath %q, full subpath %q for volumepath %q", subpath.Path, evalSubPath, subpath.VolumePath) return fd, nil
}
evalSubPath = filepath.Clean(evalSubPath) // prepareSubpathTarget creates target for bind-mount of subpath. It returns
if !pathWithinBase(evalSubPath, subpath.VolumePath) { // "true" when the target already exists and something is mounted there.
return "", fmt.Errorf("subpath %q not within volume path %q", evalSubPath, subpath.VolumePath) // Given Subpath must have all paths with already resolved symlinks and with
// paths relevant to kubelet (when it runs in a container).
// This function is called also by NsEnterMounter. It works because
// /var/lib/kubelet is mounted from the host into the container with Kubelet as
// /var/lib/kubelet too.
func prepareSubpathTarget(mounter Interface, subpath Subpath) (bool, string, error) {
// Early check for already bind-mounted subpath.
bindPathTarget := getSubpathBindTarget(subpath)
notMount, err := IsNotMountPoint(mounter, bindPathTarget)
if err != nil {
if !os.IsNotExist(err) {
return false, "", fmt.Errorf("error checking path %s for mount: %s", bindPathTarget, err)
}
// Ignore ErrorNotExist: the file/directory will be created below if it does not exist yet.
notMount = true
}
if !notMount {
// It's already mounted
glog.V(5).Infof("Skipping bind-mounting subpath %s: already mounted", bindPathTarget)
return true, bindPathTarget, nil
} }
// Prepare directory for bind mounts // bindPathTarget is in /var/lib/kubelet and thus reachable without any
// containerName is DNS label, i.e. safe as a directory name. // translation even to containerized kubelet.
bindDir := filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName) bindParent := filepath.Dir(bindPathTarget)
err = os.MkdirAll(bindDir, 0750) err = os.MkdirAll(bindParent, 0750)
if err != nil && !os.IsExist(err) { if err != nil && !os.IsExist(err) {
return "", fmt.Errorf("error creating directory %s: %s", bindDir, err) return false, "", fmt.Errorf("error creating directory %s: %s", bindParent, err)
}
t, err := os.Lstat(subpath.Path)
if err != nil {
return false, "", fmt.Errorf("lstat %s failed: %s", subpath.Path, err)
}
if t.Mode()&os.ModeDir > 0 {
if err = os.Mkdir(bindPathTarget, 0750); err != nil && !os.IsExist(err) {
return false, "", fmt.Errorf("error creating directory %s: %s", bindPathTarget, err)
}
} else {
// "/bin/touch <bindPathTarget>".
// A file is enough for all possible targets (symlink, device, pipe,
// socket, ...), bind-mounting them into a file correctly changes type
// of the target file.
if err = ioutil.WriteFile(bindPathTarget, []byte{}, 0640); err != nil {
return false, "", fmt.Errorf("error creating file %s: %s", bindPathTarget, err)
}
}
return false, bindPathTarget, nil
}
func getSubpathBindTarget(subpath Subpath) string {
// containerName is DNS label, i.e. safe as a directory name.
return filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName, strconv.Itoa(subpath.VolumeMountIndex))
}
func doBindSubPath(mounter Interface, subpath Subpath) (hostPath string, err error) {
// Linux, kubelet runs on the host:
// - safely open the subpath
// - bind-mount /proc/<pid of kubelet>/fd/<fd> to subpath target
// User can't change /proc/<pid of kubelet>/fd/<fd> to point to a bad place.
// Evaluate all symlinks here once for all subsequent functions.
newVolumePath, err := filepath.EvalSymlinks(subpath.VolumePath)
if err != nil {
return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err)
}
newPath, err := filepath.EvalSymlinks(subpath.Path)
if err != nil {
return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err)
}
glog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, newPath, subpath.VolumePath)
subpath.VolumePath = newVolumePath
subpath.Path = newPath
fd, err := safeOpenSubPath(mounter, subpath)
if err != nil {
return "", err
}
defer syscall.Close(fd)
alreadyMounted, bindPathTarget, err := prepareSubpathTarget(mounter, subpath)
if err != nil {
return "", err
}
if alreadyMounted {
return bindPathTarget, nil
} }
bindPathTarget := filepath.Join(bindDir, strconv.Itoa(subpath.VolumeMountIndex))
success := false success := false
defer func() { defer func() {
@ -804,49 +879,7 @@ func doBindSubPath(mounter Interface, subpath Subpath, kubeletPid int) (hostPath
} }
}() }()
// Check it's not already bind-mounted kubeletPid := os.Getpid()
notMount, err := IsNotMountPoint(mounter, bindPathTarget)
if err != nil {
if !os.IsNotExist(err) {
return "", fmt.Errorf("error checking path %s for mount: %s", bindPathTarget, err)
}
// Ignore ErrorNotExist: the file/directory will be created below if it does not exist yet.
notMount = true
}
if !notMount {
// It's already mounted
glog.V(5).Infof("Skipping bind-mounting subpath %s: already mounted", bindPathTarget)
success = true
return bindPathTarget, nil
}
// Create target of the bind mount. A directory for directories, empty file
// for everything else.
t, err := os.Lstat(subpath.Path)
if err != nil {
return "", fmt.Errorf("lstat %s failed: %s", subpath.Path, err)
}
if t.Mode()&os.ModeDir > 0 {
if err = os.Mkdir(bindPathTarget, 0750); err != nil && !os.IsExist(err) {
return "", fmt.Errorf("error creating directory %s: %s", bindPathTarget, err)
}
} else {
// "/bin/touch <bindDir>".
// A file is enough for all possible targets (symlink, device, pipe,
// socket, ...), bind-mounting them into a file correctly changes type
// of the target file.
if err = ioutil.WriteFile(bindPathTarget, []byte{}, 0640); err != nil {
return "", fmt.Errorf("error creating file %s: %s", bindPathTarget, err)
}
}
// Safe open subpath and get the fd
fd, err := doSafeOpen(evalSubPath, subpath.VolumePath)
if err != nil {
return "", fmt.Errorf("error opening subpath %v: %v", evalSubPath, err)
}
defer syscall.Close(fd)
mountSource := fmt.Sprintf("/proc/%d/fd/%v", kubeletPid, fd) mountSource := fmt.Sprintf("/proc/%d/fd/%v", kubeletPid, fd)
// Do the bind mount // Do the bind mount
@ -859,8 +892,8 @@ func doBindSubPath(mounter Interface, subpath Subpath, kubeletPid int) (hostPath
if err = mounter.Mount(mountSource, bindPathTarget, "" /*fstype*/, options); err != nil { if err = mounter.Mount(mountSource, bindPathTarget, "" /*fstype*/, options); err != nil {
return "", fmt.Errorf("error mounting %s: %s", subpath.Path, err) return "", fmt.Errorf("error mounting %s: %s", subpath.Path, err)
} }
success = true success = true
glog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget) glog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget)
return bindPathTarget, nil return bindPathTarget, nil
} }
@ -995,8 +1028,15 @@ func removeEmptyDirs(baseDir, endDir string) error {
return nil return nil
} }
func (mounter *Mounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error { func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
return doSafeMakeDir(pathname, base, perm) realBase, err := filepath.EvalSymlinks(base)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
}
realFullPath := filepath.Join(realBase, subdir)
return doSafeMakeDir(realFullPath, realBase, perm)
} }
func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) {
@ -1019,6 +1059,10 @@ func (mounter *Mounter) GetFSGroup(pathname string) (int64, error) {
return getFSGroup(realpath) return getFSGroup(realpath)
} }
func (mounter *Mounter) GetMode(pathname string) (os.FileMode, error) {
return getMode(pathname)
}
// This implementation is shared between Linux and NsEnterMounter // This implementation is shared between Linux and NsEnterMounter
func getFSGroup(pathname string) (int64, error) { func getFSGroup(pathname string) (int64, error) {
info, err := os.Stat(pathname) info, err := os.Stat(pathname)
@ -1029,6 +1073,17 @@ func getFSGroup(pathname string) (int64, error) {
} }
// This implementation is shared between Linux and NsEnterMounter // This implementation is shared between Linux and NsEnterMounter
func getMode(pathname string) (os.FileMode, error) {
info, err := os.Stat(pathname)
if err != nil {
return 0, err
}
return info.Mode(), nil
}
// This implementation is shared between Linux and NsEnterMounter. Both pathname
// and base must be either already resolved symlinks or thet will be resolved in
// kubelet's mount namespace (in case it runs containerized).
func doSafeMakeDir(pathname string, base string, perm os.FileMode) error { func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
glog.V(4).Infof("Creating directory %q within base %q", pathname, base) glog.V(4).Infof("Creating directory %q within base %q", pathname, base)
@ -1182,6 +1237,9 @@ func findExistingPrefix(base, pathname string) (string, []string, error) {
// Symlinks are disallowed (pathname must already resolve symlinks), // Symlinks are disallowed (pathname must already resolve symlinks),
// and the path must be within the base directory. // and the path must be within the base directory.
func doSafeOpen(pathname string, base string) (int, error) { func doSafeOpen(pathname string, base string) (int, error) {
pathname = filepath.Clean(pathname)
base = filepath.Clean(base)
// Calculate segments to follow // Calculate segments to follow
subpath, err := filepath.Rel(base, pathname) subpath, err := filepath.Rel(base, pathname)
if err != nil { if err != nil {

10
pkg/util/mount/mount_linux_test.go
View File

@ -1193,10 +1193,6 @@ func TestBindSubPath(t *testing.T) {
return nil, "", "", err return nil, "", "", err
} }
if err := os.MkdirAll(subpathMount, defaultPerm); err != nil {
return nil, "", "", err
}
socketFile, socketCreateError := createSocketFile(volpath) socketFile, socketCreateError := createSocketFile(volpath)
return mounts, volpath, socketFile, socketCreateError return mounts, volpath, socketFile, socketCreateError
@ -1212,10 +1208,6 @@ func TestBindSubPath(t *testing.T) {
return nil, "", "", err return nil, "", "", err
} }
if err := os.MkdirAll(subpathMount, defaultPerm); err != nil {
return nil, "", "", err
}
testFifo := filepath.Join(volpath, "mount_test.fifo") testFifo := filepath.Join(volpath, "mount_test.fifo")
err := syscall.Mkfifo(testFifo, 0) err := syscall.Mkfifo(testFifo, 0)
return mounts, volpath, testFifo, err return mounts, volpath, testFifo, err
@ -1299,7 +1291,7 @@ func TestBindSubPath(t *testing.T) {
} }
_, subpathMount := getTestPaths(base) _, subpathMount := getTestPaths(base)
bindPathTarget, err := doBindSubPath(fm, subpath, 1) bindPathTarget, err := doBindSubPath(fm, subpath)
if test.expectError { if test.expectError {
if err == nil { if err == nil {
t.Errorf("test %q failed: expected error, got success", test.name) t.Errorf("test %q failed: expected error, got success", test.name)

11
pkg/util/mount/mount_unsupported.go
View File

@ -21,8 +21,6 @@ package mount
import ( import (
"errors" "errors"
"os" "os"
"github.com/golang/glog"
) )
type Mounter struct { type Mounter struct {
@ -110,9 +108,8 @@ func (mounter *Mounter) MakeFile(pathname string) error {
return unsupportedErr return unsupportedErr
} }
func (mounter *Mounter) ExistsPath(pathname string) bool { func (mounter *Mounter) ExistsPath(pathname string) (bool, error) {
glog.Errorf("%s", unsupportedErr) return true, errors.New("not implemented")
return true
} }
func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
@ -138,3 +135,7 @@ func (mounter *Mounter) GetFSGroup(pathname string) (int64, error) {
func (mounter *Mounter) GetSELinuxSupport(pathname string) (bool, error) { func (mounter *Mounter) GetSELinuxSupport(pathname string) (bool, error) {
return false, errors.New("not implemented") return false, errors.New("not implemented")
} }
func (mounter *Mounter) GetMode(pathname string) (os.FileMode, error) {
return 0, errors.New("not implemented")
}

36
pkg/util/mount/mount_windows.go
View File

@ -29,6 +29,8 @@ import (
"syscall" "syscall"
"github.com/golang/glog" "github.com/golang/glog"
utilfile "k8s.io/kubernetes/pkg/util/file"
) )
// Mounter provides the default implementation of mount.Interface // Mounter provides the default implementation of mount.Interface
@ -147,9 +149,13 @@ func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) {
if stat.Mode()&os.ModeSymlink != 0 { if stat.Mode()&os.ModeSymlink != 0 {
target, err := os.Readlink(file) target, err := os.Readlink(file)
if err != nil { if err != nil {
return true, fmt.Errorf("Readlink error: %v", err) return true, fmt.Errorf("readlink error: %v", err)
} }
return !mounter.ExistsPath(target), nil exists, err := mounter.ExistsPath(target)
if err != nil {
return true, err
}
return !exists, nil
} }
return true, nil return true, nil
@ -232,12 +238,8 @@ func (mounter *Mounter) MakeFile(pathname string) error {
} }
// ExistsPath checks whether the path exists // ExistsPath checks whether the path exists
func (mounter *Mounter) ExistsPath(pathname string) bool { func (mounter *Mounter) ExistsPath(pathname string) (bool, error) {
_, err := os.Stat(pathname) return utilfile.FileExists(pathname)
if err != nil {
return false
}
return true
} }
// check whether hostPath is within volume path // check whether hostPath is within volume path
@ -461,9 +463,23 @@ func (mounter *Mounter) GetSELinuxSupport(pathname string) (bool, error) {
return false, nil return false, nil
} }
func (mounter *Mounter) GetMode(pathname string) (os.FileMode, error) {
info, err := os.Stat(pathname)
if err != nil {
return 0, err
}
return info.Mode(), nil
}
// SafeMakeDir makes sure that the created directory does not escape given base directory mis-using symlinks. // SafeMakeDir makes sure that the created directory does not escape given base directory mis-using symlinks.
func (mounter *Mounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error { func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
return doSafeMakeDir(pathname, base, perm) realBase, err := filepath.EvalSymlinks(base)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
}
realFullPath := filepath.Join(realBase, subdir)
return doSafeMakeDir(realFullPath, realBase, perm)
} }
func doSafeMakeDir(pathname string, base string, perm os.FileMode) error { func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {

199
pkg/util/mount/nsenter_mount.go
View File

@ -22,12 +22,12 @@ import (
"fmt" "fmt"
"os" "os"
"path/filepath" "path/filepath"
"regexp"
"strconv"
"strings" "strings"
"syscall"
"github.com/golang/glog" "github.com/golang/glog"
utilio "k8s.io/kubernetes/pkg/util/io" "golang.org/x/sys/unix"
utilfile "k8s.io/kubernetes/pkg/util/file"
"k8s.io/kubernetes/pkg/util/nsenter" "k8s.io/kubernetes/pkg/util/nsenter"
) )
@ -36,13 +36,6 @@ const (
hostProcMountsPath = "/rootfs/proc/1/mounts" hostProcMountsPath = "/rootfs/proc/1/mounts"
// hostProcMountinfoPath is the default mount info path for rootfs // hostProcMountinfoPath is the default mount info path for rootfs
hostProcMountinfoPath = "/rootfs/proc/1/mountinfo" hostProcMountinfoPath = "/rootfs/proc/1/mountinfo"
// hostProcSelfStatusPath is the default path to /proc/self/status on the host
hostProcSelfStatusPath = "/rootfs/proc/self/status"
)
var (
// pidRegExp matches "Pid: <pid>" in /proc/self/status
pidRegExp = regexp.MustCompile(`\nPid:\t([0-9]*)\n`)
) )
// Currently, all docker containers receive their own mount namespaces. // Currently, all docker containers receive their own mount namespaces.
@ -50,14 +43,16 @@ var (
// the host's mount namespace. // the host's mount namespace.
type NsenterMounter struct { type NsenterMounter struct {
ne *nsenter.Nsenter ne *nsenter.Nsenter
// rootDir is location of /var/lib/kubelet directory.
rootDir string
} }
func NewNsenterMounter() (*NsenterMounter, error) { // NewNsenterMounter creates a new mounter for kubelet that runs as a container.
ne, err := nsenter.NewNsenter() func NewNsenterMounter(rootDir string, ne *nsenter.Nsenter) *NsenterMounter {
if err != nil { return &NsenterMounter{
return nil, err rootDir: rootDir,
ne: ne,
} }
return &NsenterMounter{ne: ne}, nil
} }
// NsenterMounter implements mount.Interface // NsenterMounter implements mount.Interface
@ -281,42 +276,24 @@ func (mounter *NsenterMounter) MakeFile(pathname string) error {
return nil return nil
} }
func (mounter *NsenterMounter) ExistsPath(pathname string) bool { func (mounter *NsenterMounter) ExistsPath(pathname string) (bool, error) {
args := []string{pathname} // Resolve the symlinks but allow the target not to exist. EvalSymlinks
_, err := mounter.ne.Exec("ls", args).CombinedOutput() // would return an generic error when the target does not exist.
if err == nil { hostPath, err := mounter.ne.EvalSymlinks(pathname, false /* mustExist */)
return true if err != nil {
return false, err
} }
return false kubeletpath := mounter.ne.KubeletPath(hostPath)
return utilfile.FileExists(kubeletpath)
} }
func (mounter *NsenterMounter) CleanSubPaths(podDir string, volumeName string) error { func (mounter *NsenterMounter) CleanSubPaths(podDir string, volumeName string) error {
return doCleanSubPaths(mounter, podDir, volumeName) return doCleanSubPaths(mounter, podDir, volumeName)
} }
// getPidOnHost returns kubelet's pid in the host pid namespace
func (mounter *NsenterMounter) getPidOnHost(procStatusPath string) (int, error) {
// Get the PID from /rootfs/proc/self/status
statusBytes, err := utilio.ConsistentRead(procStatusPath, maxListTries)
if err != nil {
return 0, fmt.Errorf("error reading %s: %s", procStatusPath, err)
}
matches := pidRegExp.FindSubmatch(statusBytes)
if len(matches) < 2 {
return 0, fmt.Errorf("cannot parse %s: no Pid:", procStatusPath)
}
return strconv.Atoi(string(matches[1]))
}
func (mounter *NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { func (mounter *NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
hostPid, err := mounter.getPidOnHost(hostProcSelfStatusPath)
if err != nil {
return "", nil, err
}
glog.V(4).Infof("Kubelet's PID on the host is %d", hostPid)
// Bind-mount the subpath to avoid using symlinks in subpaths. // Bind-mount the subpath to avoid using symlinks in subpaths.
newHostPath, err = doBindSubPath(mounter, subPath, hostPid) newHostPath, err = doNsEnterBindSubPath(mounter, subPath)
// There is no action when the container starts. Bind-mount will be cleaned // There is no action when the container starts. Bind-mount will be cleaned
// when container stops by CleanSubPaths. // when container stops by CleanSubPaths.
@ -324,26 +301,152 @@ func (mounter *NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath
return newHostPath, cleanupAction, err return newHostPath, cleanupAction, err
} }
func (mounter *NsenterMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error { func (mounter *NsenterMounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
return doSafeMakeDir(pathname, base, perm) fullSubdirPath := filepath.Join(base, subdir)
evaluatedSubdirPath, err := mounter.ne.EvalSymlinks(fullSubdirPath, false /* mustExist */)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", fullSubdirPath, err)
}
evaluatedSubdirPath = filepath.Clean(evaluatedSubdirPath)
evaluatedBase, err := mounter.ne.EvalSymlinks(base, true /* mustExist */)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
}
evaluatedBase = filepath.Clean(evaluatedBase)
rootDir := filepath.Clean(mounter.rootDir)
if pathWithinBase(evaluatedBase, rootDir) {
// Base is in /var/lib/kubelet. This directory is shared between the
// container with kubelet and the host. We don't need to add '/rootfs'.
// This is useful when /rootfs is mounted as read-only - we can still
// create subpaths for paths in /var/lib/kubelet.
return doSafeMakeDir(evaluatedSubdirPath, evaluatedBase, perm)
}
// Base is somewhere on the host's filesystem. Add /rootfs and try to make
// the directory there.
// This requires /rootfs to be writable.
kubeletSubdirPath := mounter.ne.KubeletPath(evaluatedSubdirPath)
kubeletBase := mounter.ne.KubeletPath(evaluatedBase)
return doSafeMakeDir(kubeletSubdirPath, kubeletBase, perm)
} }
func (mounter *NsenterMounter) GetMountRefs(pathname string) ([]string, error) { func (mounter *NsenterMounter) GetMountRefs(pathname string) ([]string, error) {
hostpath, err := mounter.ne.EvalSymlinks(pathname) hostpath, err := mounter.ne.EvalSymlinks(pathname, true /* mustExist */)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return searchMountPoints(hostpath, hostProcMountinfoPath) return searchMountPoints(hostpath, hostProcMountinfoPath)
} }
func (mounter *NsenterMounter) GetFSGroup(pathname string) (int64, error) { func doNsEnterBindSubPath(mounter *NsenterMounter, subpath Subpath) (hostPath string, err error) {
kubeletpath, err := mounter.ne.KubeletPath(pathname) // Linux, kubelet runs in a container:
// - safely open the subpath
// - bind-mount the subpath to target (this can be unsafe)
// - check that we mounted the right thing by comparing device ID and inode
// of the subpath (via safely opened fd) and the target (that's under our
// control)
// Evaluate all symlinks here once for all subsequent functions.
evaluatedHostVolumePath, err := mounter.ne.EvalSymlinks(subpath.VolumePath, true /*mustExist*/)
if err != nil { if err != nil {
return 0, err return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err)
} }
evaluatedHostSubpath, err := mounter.ne.EvalSymlinks(subpath.Path, true /*mustExist*/)
if err != nil {
return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err)
}
glog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, evaluatedHostSubpath, subpath.VolumePath)
subpath.VolumePath = mounter.ne.KubeletPath(evaluatedHostVolumePath)
subpath.Path = mounter.ne.KubeletPath(evaluatedHostSubpath)
// Check the subpath is correct and open it
fd, err := safeOpenSubPath(mounter, subpath)
if err != nil {
return "", err
}
defer syscall.Close(fd)
alreadyMounted, bindPathTarget, err := prepareSubpathTarget(mounter, subpath)
if err != nil {
return "", err
}
if alreadyMounted {
return bindPathTarget, nil
}
success := false
defer func() {
// Cleanup subpath on error
if !success {
glog.V(4).Infof("doNsEnterBindSubPath() failed for %q, cleaning up subpath", bindPathTarget)
if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil {
glog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr)
}
}
}()
// Leap of faith: optimistically expect that nobody has modified previously
// expanded evalSubPath with evil symlinks and bind-mount it.
// Mount is done on the host! don't use kubelet path!
glog.V(5).Infof("bind mounting %q at %q", evaluatedHostSubpath, bindPathTarget)
if err = mounter.Mount(evaluatedHostSubpath, bindPathTarget, "" /*fstype*/, []string{"bind"}); err != nil {
return "", fmt.Errorf("error mounting %s: %s", evaluatedHostSubpath, err)
}
// Check that the bind-mount target is the same inode and device as the
// source that we keept open, i.e. we mounted the right thing.
err = checkDeviceInode(fd, bindPathTarget)
if err != nil {
return "", fmt.Errorf("error checking bind mount for subpath %s: %s", subpath.VolumePath, err)
}
success = true
glog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget)
return bindPathTarget, nil
}
// checkDeviceInode checks that opened file and path represent the same file.
func checkDeviceInode(fd int, path string) error {
var srcStat, dstStat unix.Stat_t
err := unix.Fstat(fd, &srcStat)
if err != nil {
return fmt.Errorf("error running fstat on subpath FD: %v", err)
}
err = unix.Stat(path, &dstStat)
if err != nil {
return fmt.Errorf("error running fstat on %s: %v", path, err)
}
if srcStat.Dev != dstStat.Dev {
return fmt.Errorf("different device number")
}
if srcStat.Ino != dstStat.Ino {
return fmt.Errorf("different inode")
}
return nil
}
func (mounter *NsenterMounter) GetFSGroup(pathname string) (int64, error) {
hostPath, err := mounter.ne.EvalSymlinks(pathname, true /* mustExist */)
if err != nil {
return -1, err
}
kubeletpath := mounter.ne.KubeletPath(hostPath)
return getFSGroup(kubeletpath) return getFSGroup(kubeletpath)
} }
func (mounter *NsenterMounter) GetSELinuxSupport(pathname string) (bool, error) { func (mounter *NsenterMounter) GetSELinuxSupport(pathname string) (bool, error) {
return getSELinuxSupport(pathname, hostProcMountsPath) return getSELinuxSupport(pathname, hostProcMountsPath)
} }
func (mounter *NsenterMounter) GetMode(pathname string) (os.FileMode, error) {
hostPath, err := mounter.ne.EvalSymlinks(pathname, true /* mustExist */)
if err != nil {
return 0, err
}
kubeletpath := mounter.ne.KubeletPath(hostPath)
return getMode(kubeletpath)
}

708
pkg/util/mount/nsenter_mount_test.go
View File

@ -21,9 +21,12 @@ package mount
import ( import (
"io/ioutil" "io/ioutil"
"os" "os"
"path" "path/filepath"
"strconv" "strings"
"testing" "testing"
"golang.org/x/sys/unix"
"k8s.io/kubernetes/pkg/util/nsenter"
) )
func TestParseFindMnt(t *testing.T) { func TestParseFindMnt(t *testing.T) {
@ -72,120 +75,635 @@ func TestParseFindMnt(t *testing.T) {
} }
} }
func TestGetPidOnHost(t *testing.T) { func TestCheckDeviceInode(t *testing.T) {
tempDir, err := ioutil.TempDir("", "get_pid_on_host_tests") testDir, err := ioutil.TempDir("", "nsenter-mounter-device-")
if err != nil { if err != nil {
t.Fatalf(err.Error()) t.Fatalf("Cannot create temporary directory: %s", err)
} }
defer os.RemoveAll(tempDir) defer os.RemoveAll(testDir)
tests := []struct { tests := []struct {
name string name string
procFile string srcPath string
expectedPid int dstPath string
expectError bool expectError string
}{ }{
{ {
name: "valid status file", name: "the same file",
procFile: `Name: cat srcPath: filepath.Join(testDir, "1"),
Umask: 0002 dstPath: filepath.Join(testDir, "1"),
State: R (running) expectError: "",
Tgid: 15041
Ngid: 0
Pid: 15041
PPid: 22699
TracerPid: 0
Uid: 1000 1000 1000 1000
Gid: 1000 1000 1000 1000
FDSize: 256
Groups: 10 135 156 157 158 973 984 1000 1001
NStgid: 15041
NSpid: 15041
NSpgid: 15041
NSsid: 22699
VmPeak: 115016 kB
VmSize: 115016 kB
VmLck: 0 kB
VmPin: 0 kB
VmHWM: 816 kB
VmRSS: 816 kB
RssAnon: 64 kB
RssFile: 752 kB
RssShmem: 0 kB
VmData: 312 kB
VmStk: 136 kB
VmExe: 32 kB
VmLib: 2060 kB
VmPTE: 44 kB
VmPMD: 12 kB
VmSwap: 0 kB
HugetlbPages: 0 kB
Threads: 1
SigQ: 2/60752
SigPnd: 0000000000000000
ShdPnd: 0000000000000000
SigBlk: 0000000000000000
SigIgn: 0000000000000000
SigCgt: 0000000000000000
CapInh: 0000000000000000
CapPrm: 0000000000000000
CapEff: 0000000000000000
CapBnd: 0000003fffffffff
CapAmb: 0000000000000000
NoNewPrivs: 0
Seccomp: 0
Cpus_allowed: ff
Cpus_allowed_list: 0-7
Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001
Mems_allowed_list: 0
voluntary_ctxt_switches: 0
nonvoluntary_ctxt_switches: 0
`,
expectedPid: 15041,
}, },
{ {
name: "no Pid:", name: "different file on the same FS",
procFile: `Name: cat srcPath: filepath.Join(testDir, "2.1"),
Umask: 0002 dstPath: filepath.Join(testDir, "2.2"),
State: R (running) expectError: "different inode",
Tgid: 15041
Ngid: 0
PPid: 22699
`,
expectedPid: 0,
expectError: true,
}, },
{ {
name: "invalid Pid:", name: "different file on different device",
procFile: `Name: cat srcPath: filepath.Join(testDir, "3"),
Umask: 0002 // /proc is always on a different "device" than /tmp (or $TEMP)
State: R (running) dstPath: "/proc/self/status",
Tgid: 15041 expectError: "different device",
Ngid: 0
Pid: invalid
PPid: 22699
`,
expectedPid: 0,
expectError: true,
}, },
} }
for i, test := range tests { for _, test := range tests {
filename := path.Join(tempDir, strconv.Itoa(i)) if err := ioutil.WriteFile(test.srcPath, []byte{}, 0644); err != nil {
err := ioutil.WriteFile(filename, []byte(test.procFile), 0666) t.Errorf("Test %q: cannot create srcPath %s: %s", test.name, test.srcPath, err)
if err != nil { continue
t.Fatalf(err.Error())
} }
mounter := NsenterMounter{}
pid, err := mounter.getPidOnHost(filename) // Don't create dst if it exists
if _, err := os.Stat(test.dstPath); os.IsNotExist(err) {
if err := ioutil.WriteFile(test.dstPath, []byte{}, 0644); err != nil {
t.Errorf("Test %q: cannot create dstPath %s: %s", test.name, test.dstPath, err)
continue
}
} else if err != nil {
t.Errorf("Test %q: cannot check existence of dstPath %s: %s", test.name, test.dstPath, err)
continue
}
fd, err := unix.Open(test.srcPath, unix.O_CREAT, 0644)
if err != nil {
t.Errorf("Test %q: cannot open srcPath %s: %s", test.name, test.srcPath, err)
continue
}
err = checkDeviceInode(fd, test.dstPath)
if test.expectError == "" && err != nil {
t.Errorf("Test %q: expected no error, got %s", test.name, err)
}
if test.expectError != "" {
if err == nil {
t.Errorf("Test %q: expected error, got none", test.name)
} else {
if !strings.Contains(err.Error(), test.expectError) {
t.Errorf("Test %q: expected error %q, got %q", test.name, test.expectError, err)
}
}
}
}
}
func newFakeNsenterMounter(tmpdir string, t *testing.T) (mounter *NsenterMounter, rootfsPath string, varlibPath string, err error) {
rootfsPath = filepath.Join(tmpdir, "rootfs")
if err := os.Mkdir(rootfsPath, 0755); err != nil {
return nil, "", "", err
}
ne, err := nsenter.NewFakeNsenter(rootfsPath)
if err != nil {
return nil, "", "", err
}
varlibPath = filepath.Join(tmpdir, "/var/lib/kubelet")
if err := os.MkdirAll(varlibPath, 0755); err != nil {
return nil, "", "", err
}
return NewNsenterMounter(varlibPath, ne), rootfsPath, varlibPath, nil
}
func TestNsenterExistsFile(t *testing.T) {
tests := []struct {
name string
prepare func(base, rootfs string) (string, error)
expectedOutput bool
expectError bool
}{
{
name: "simple existing file",
prepare: func(base, rootfs string) (string, error) {
// On the host: /base/file
path := filepath.Join(base, "file")
if err := ioutil.WriteFile(path, []byte{}, 0644); err != nil {
return "", err
}
// In kubelet: /rootfs/base/file
if _, err := writeRootfsFile(rootfs, path, 0644); err != nil {
return "", err
}
return path, nil
},
expectedOutput: true,
},
{
name: "simple non-existing file",
prepare: func(base, rootfs string) (string, error) {
path := filepath.Join(base, "file")
return path, nil
},
expectedOutput: false,
},
{
name: "simple non-accessible file",
prepare: func(base, rootfs string) (string, error) {
// On the host:
// create /base/dir/file, then make the dir inaccessible
dir := filepath.Join(base, "dir")
if err := os.MkdirAll(dir, 0755); err != nil {
return "", err
}
path := filepath.Join(dir, "file")
if err := ioutil.WriteFile(path, []byte{}, 0); err != nil {
return "", err
}
if err := os.Chmod(dir, 0644); err != nil {
return "", err
}
// In kubelet: do the same with /rootfs/base/dir/file
rootfsPath, err := writeRootfsFile(rootfs, path, 0777)
if err != nil {
return "", err
}
rootfsDir := filepath.Dir(rootfsPath)
if err := os.Chmod(rootfsDir, 0644); err != nil {
return "", err
}
return path, nil
},
expectedOutput: false,
expectError: true,
},
{
name: "relative symlink to existing file",
prepare: func(base, rootfs string) (string, error) {
// On the host: /base/link -> file
file := filepath.Join(base, "file")
if err := ioutil.WriteFile(file, []byte{}, 0); err != nil {
return "", err
}
path := filepath.Join(base, "link")
if err := os.Symlink("file", path); err != nil {
return "", err
}
// In kubelet: /rootfs/base/file
if _, err := writeRootfsFile(rootfs, file, 0644); err != nil {
return "", err
}
return path, nil
},
expectedOutput: true,
},
{
name: "absolute symlink to existing file",
prepare: func(base, rootfs string) (string, error) {
// On the host: /base/link -> /base/file
file := filepath.Join(base, "file")
if err := ioutil.WriteFile(file, []byte{}, 0); err != nil {
return "", err
}
path := filepath.Join(base, "link")
if err := os.Symlink(file, path); err != nil {
return "", err
}
// In kubelet: /rootfs/base/file
if _, err := writeRootfsFile(rootfs, file, 0644); err != nil {
return "", err
}
return path, nil
},
expectedOutput: true,
},
{
name: "relative symlink to non-existing file",
prepare: func(base, rootfs string) (string, error) {
path := filepath.Join(base, "link")
if err := os.Symlink("file", path); err != nil {
return "", err
}
return path, nil
},
expectedOutput: false,
},
{
name: "absolute symlink to non-existing file",
prepare: func(base, rootfs string) (string, error) {
file := filepath.Join(base, "file")
path := filepath.Join(base, "link")
if err := os.Symlink(file, path); err != nil {
return "", err
}
return path, nil
},
expectedOutput: false,
},
{
name: "symlink loop",
prepare: func(base, rootfs string) (string, error) {
path := filepath.Join(base, "link")
if err := os.Symlink(path, path); err != nil {
return "", err
}
return path, nil
},
expectedOutput: false,
// TODO: realpath -m is not able to detect symlink loop. Should we care?
expectError: false,
},
}
for _, test := range tests {
tmpdir, err := ioutil.TempDir("", "nsenter-exists-file")
if err != nil {
t.Error(err)
continue
}
defer os.RemoveAll(tmpdir)
testBase := filepath.Join(tmpdir, "base")
if err := os.Mkdir(testBase, 0755); err != nil {
t.Error(err)
continue
}
mounter, rootfs, _, err := newFakeNsenterMounter(tmpdir, t)
if err != nil {
t.Error(err)
continue
}
path, err := test.prepare(testBase, rootfs)
if err != nil {
t.Error(err)
continue
}
out, err := mounter.ExistsPath(path)
if err != nil && !test.expectError { if err != nil && !test.expectError {
t.Errorf("Test %q: unexpected error: %s", test.name, err) t.Errorf("Test %q: unexpected error: %s", test.name, err)
} }
if err == nil && test.expectError { if err == nil && test.expectError {
t.Errorf("Test %q: expected error, got none", test.name) t.Errorf("Test %q: expected error, got none", test.name)
} }
if pid != test.expectedPid {
t.Errorf("Test %q: expected pid %d, got %d", test.name, test.expectedPid, pid) if out != test.expectedOutput {
t.Errorf("Test %q: expected return value %v, got %v", test.name, test.expectedOutput, out)
}
}
}
func TestNsenterGetMode(t *testing.T) {
tests := []struct {
name string
prepare func(base, rootfs string) (string, error)
expectedMode os.FileMode
expectError bool
}{
{
name: "simple file",
prepare: func(base, rootfs string) (string, error) {
// On the host: /base/file
path := filepath.Join(base, "file")
if err := ioutil.WriteFile(path, []byte{}, 0644); err != nil {
return "", err
}
// Prepare a different file as /rootfs/base/file (="the host
// visible from container") to check that NsEnterMounter calls
// stat on this file and not on /base/file.
// Visible from kubelet: /rootfs/base/file
if _, err := writeRootfsFile(rootfs, path, 0777); err != nil {
return "", err
}
return path, nil
},
expectedMode: 0777,
},
{
name: "non-existing file",
prepare: func(base, rootfs string) (string, error) {
path := filepath.Join(base, "file")
return path, nil
},
expectedMode: 0,
expectError: true,
},
{
name: "absolute symlink to existing file",
prepare: func(base, rootfs string) (string, error) {
// On the host: /base/link -> /base/file
file := filepath.Join(base, "file")
if err := ioutil.WriteFile(file, []byte{}, 0644); err != nil {
return "", err
}
path := filepath.Join(base, "link")
if err := os.Symlink(file, path); err != nil {
return "", err
}
// Visible from kubelet:
// /rootfs/base/file
if _, err := writeRootfsFile(rootfs, file, 0747); err != nil {
return "", err
}
return path, nil
},
expectedMode: 0747,
},
{
name: "relative symlink to existing file",
prepare: func(base, rootfs string) (string, error) {
// On the host: /base/link -> file
file := filepath.Join(base, "file")
if err := ioutil.WriteFile(file, []byte{}, 0741); err != nil {
return "", err
}
path := filepath.Join(base, "link")
if err := os.Symlink("file", path); err != nil {
return "", err
}
// Visible from kubelet:
// /rootfs/base/file
if _, err := writeRootfsFile(rootfs, file, 0647); err != nil {
return "", err
}
return path, nil
},
expectedMode: 0647,
},
}
for _, test := range tests {
tmpdir, err := ioutil.TempDir("", "nsenter-get-mode-")
if err != nil {
t.Error(err)
continue
}
defer os.RemoveAll(tmpdir)
testBase := filepath.Join(tmpdir, "base")
if err := os.Mkdir(testBase, 0755); err != nil {
t.Error(err)
continue
}
mounter, rootfs, _, err := newFakeNsenterMounter(tmpdir, t)
if err != nil {
t.Error(err)
continue
}
path, err := test.prepare(testBase, rootfs)
if err != nil {
t.Error(err)
continue
}
mode, err := mounter.GetMode(path)
if err != nil && !test.expectError {
t.Errorf("Test %q: unexpected error: %s", test.name, err)
}
if err == nil && test.expectError {
t.Errorf("Test %q: expected error, got none", test.name)
}
if mode != test.expectedMode {
t.Errorf("Test %q: expected return value %v, got %v", test.name, test.expectedMode, mode)
}
}
}
func writeRootfsFile(rootfs, path string, mode os.FileMode) (string, error) {
fullPath := filepath.Join(rootfs, path)
dir := filepath.Dir(fullPath)
if err := os.MkdirAll(dir, 0755); err != nil {
return "", err
}
if err := ioutil.WriteFile(fullPath, []byte{}, mode); err != nil {
return "", err
}
// Use chmod, io.WriteFile is affected by umask
if err := os.Chmod(fullPath, mode); err != nil {
return "", err
}
return fullPath, nil
}
func TestNsenterSafeMakeDir(t *testing.T) {
tests := []struct {
name string
prepare func(base, rootfs, varlib string) (expectedDir string, err error)
subdir string
expectError bool
// If true, "base" directory for SafeMakeDir will be /var/lib/kubelet
baseIsVarLib bool
}{
{
name: "simple directory",
// evaluated in base
subdir: "some/subdirectory/structure",
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// expected to be created in /roots/
expectedDir = filepath.Join(rootfs, base, "some/subdirectory/structure")
return expectedDir, nil
},
},
{
name: "simple existing directory",
// evaluated in base
subdir: "some/subdirectory/structure",
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// On the host: directory exists
hostPath := filepath.Join(base, "some/subdirectory/structure")
if err := os.MkdirAll(hostPath, 0755); err != nil {
return "", err
}
// In rootfs: directory exists
kubeletPath := filepath.Join(rootfs, hostPath)
if err := os.MkdirAll(kubeletPath, 0755); err != nil {
return "", err
}
// expected to be created in /roots/
expectedDir = kubeletPath
return expectedDir, nil
},
},
{
name: "absolute symlink into safe place",
// evaluated in base
subdir: "some/subdirectory/structure",
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// On the host: /base/other/subdirectory exists, /base/some is link to /base/other
hostPath := filepath.Join(base, "other/subdirectory")
if err := os.MkdirAll(hostPath, 0755); err != nil {
return "", err
}
somePath := filepath.Join(base, "some")
otherPath := filepath.Join(base, "other")
if err := os.Symlink(otherPath, somePath); err != nil {
return "", err
}
// In rootfs: /base/other/subdirectory exists
kubeletPath := filepath.Join(rootfs, hostPath)
if err := os.MkdirAll(kubeletPath, 0755); err != nil {
return "", err
}
// expected 'structure' to be created
expectedDir = filepath.Join(rootfs, hostPath, "structure")
return expectedDir, nil
},
},
{
name: "relative symlink into safe place",
// evaluated in base
subdir: "some/subdirectory/structure",
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// On the host: /base/other/subdirectory exists, /base/some is link to other
hostPath := filepath.Join(base, "other/subdirectory")
if err := os.MkdirAll(hostPath, 0755); err != nil {
return "", err
}
somePath := filepath.Join(base, "some")
if err := os.Symlink("other", somePath); err != nil {
return "", err
}
// In rootfs: /base/other/subdirectory exists
kubeletPath := filepath.Join(rootfs, hostPath)
if err := os.MkdirAll(kubeletPath, 0755); err != nil {
return "", err
}
// expected 'structure' to be created
expectedDir = filepath.Join(rootfs, hostPath, "structure")
return expectedDir, nil
},
},
{
name: "symlink into unsafe place",
// evaluated in base
subdir: "some/subdirectory/structure",
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// On the host: /base/some is link to /bin/other
somePath := filepath.Join(base, "some")
if err := os.Symlink("/bin", somePath); err != nil {
return "", err
}
return "", nil
},
expectError: true,
},
{
name: "simple directory in /var/lib/kubelet",
// evaluated in varlib
subdir: "some/subdirectory/structure",
baseIsVarLib: true,
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// expected to be created in /base/var/lib/kubelet, not in /rootfs!
expectedDir = filepath.Join(varlib, "some/subdirectory/structure")
return expectedDir, nil
},
},
{
name: "safe symlink in /var/lib/kubelet",
// evaluated in varlib
subdir: "some/subdirectory/structure",
baseIsVarLib: true,
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// On the host: /varlib/kubelet/other/subdirectory exists, /varlib/some is link to other
hostPath := filepath.Join(varlib, "other/subdirectory")
if err := os.MkdirAll(hostPath, 0755); err != nil {
return "", err
}
somePath := filepath.Join(varlib, "some")
if err := os.Symlink("other", somePath); err != nil {
return "", err
}
// expected to be created in /base/var/lib/kubelet, not in /rootfs!
expectedDir = filepath.Join(varlib, "other/subdirectory/structure")
return expectedDir, nil
},
},
{
name: "unsafe symlink in /var/lib/kubelet",
// evaluated in varlib
subdir: "some/subdirectory/structure",
baseIsVarLib: true,
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// On the host: /varlib/some is link to /bin
somePath := filepath.Join(varlib, "some")
if err := os.Symlink("/bin", somePath); err != nil {
return "", err
}
return "", nil
},
expectError: true,
},
}
for _, test := range tests {
tmpdir, err := ioutil.TempDir("", "nsenter-get-mode-")
if err != nil {
t.Error(err)
continue
}
defer os.RemoveAll(tmpdir)
mounter, rootfs, varlib, err := newFakeNsenterMounter(tmpdir, t)
if err != nil {
t.Error(err)
continue
}
// Prepare base directory for the test
testBase := filepath.Join(tmpdir, "base")
if err := os.Mkdir(testBase, 0755); err != nil {
t.Error(err)
continue
}
// Prepare base directory also in /rootfs
rootfsBase := filepath.Join(rootfs, testBase)
if err := os.MkdirAll(rootfsBase, 0755); err != nil {
t.Error(err)
continue
}
expectedDir := ""
if test.prepare != nil {
expectedDir, err = test.prepare(testBase, rootfs, varlib)
if err != nil {
t.Error(err)
continue
}
}
if test.baseIsVarLib {
// use /var/lib/kubelet as the test base so we can test creating
// subdirs there directly in /var/lib/kubenet and not in
// /rootfs/var/lib/kubelet
testBase = varlib
}
err = mounter.SafeMakeDir(test.subdir, testBase, 0755)
if err != nil && !test.expectError {
t.Errorf("Test %q: unexpected error: %s", test.name, err)
}
if test.expectError {
if err == nil {
t.Errorf("Test %q: expected error, got none", test.name)
} else {
if !strings.Contains(err.Error(), "is outside of allowed base") {
t.Errorf("Test %q: expected error to contain \"is outside of allowed base\", got this one instead: %s", test.name, err)
}
}
}
if expectedDir != "" {
_, err := os.Stat(expectedDir)
if err != nil {
t.Errorf("Test %q: expected %q to exist, got error: %s", test.name, expectedDir, err)
}
} }
} }
} }

14
pkg/util/mount/nsenter_mount_unsupported.go
View File

@ -21,12 +21,14 @@ package mount
import ( import (
"errors" "errors"
"os" "os"
"k8s.io/kubernetes/pkg/util/nsenter"
) )
type NsenterMounter struct{} type NsenterMounter struct{}
func NewNsenterMounter() (*NsenterMounter, error) { func NewNsenterMounter(rootDir string, ne *nsenter.Nsenter) *NsenterMounter {
return &NsenterMounter{}, nil return &NsenterMounter{}
} }
var _ = Interface(&NsenterMounter{}) var _ = Interface(&NsenterMounter{})
@ -83,8 +85,8 @@ func (*NsenterMounter) MakeFile(pathname string) error {
return nil return nil
} }
func (*NsenterMounter) ExistsPath(pathname string) bool { func (*NsenterMounter) ExistsPath(pathname string) (bool, error) {
return true return true, errors.New("not implemented")
} }
func (*NsenterMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error { func (*NsenterMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
@ -110,3 +112,7 @@ func (*NsenterMounter) GetFSGroup(pathname string) (int64, error) {
func (*NsenterMounter) GetSELinuxSupport(pathname string) (bool, error) { func (*NsenterMounter) GetSELinuxSupport(pathname string) (bool, error) {
return false, errors.New("not implemented") return false, errors.New("not implemented")
} }
func (*NsenterMounter) GetMode(pathname string) (os.FileMode, error) {
return 0, errors.New("not implemented")
}

19
pkg/util/nsenter/BUILD
View File

@ -1,4 +1,4 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library") load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library( go_library(
name = "go_default_library", name = "go_default_library",
@ -92,3 +92,20 @@ filegroup(
tags = ["automanaged"], tags = ["automanaged"],
visibility = ["//visibility:public"], visibility = ["//visibility:public"],
) )
go_test(
name = "go_default_test",
srcs = select({
"@io_bazel_rules_go//go/platform:linux": [
"nsenter_test.go",
],
"//conditions:default": [],
}),
embed = [":go_default_library"],
deps = select({
"@io_bazel_rules_go//go/platform:linux": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"//conditions:default": [],
}),
)

149
pkg/util/nsenter/nsenter.go
View File

@ -19,6 +19,8 @@ limitations under the License.
package nsenter package nsenter
import ( import (
"context"
"errors"
"fmt" "fmt"
"os" "os"
"path/filepath" "path/filepath"
@ -30,9 +32,11 @@ import (
) )
const ( const (
hostRootFsPath = "/rootfs" // DefaultHostRootFsPath is path to host's filesystem mounted into container
// hostProcMountNsPath is the default mount namespace for rootfs // with kubelet.
hostProcMountNsPath = "/rootfs/proc/1/ns/mnt" DefaultHostRootFsPath = "/rootfs"
// mountNsPath is the default mount namespace of the host
mountNsPath = "/proc/1/ns/mnt"
// nsenterPath is the default nsenter command // nsenterPath is the default nsenter command
nsenterPath = "nsenter" nsenterPath = "nsenter"
) )
@ -65,30 +69,46 @@ const (
type Nsenter struct { type Nsenter struct {
// a map of commands to their paths on the host filesystem // a map of commands to their paths on the host filesystem
paths map[string]string paths map[string]string
// Path to the host filesystem, typically "/rootfs". Used only for testing.
hostRootFsPath string
// Exec implementation, used only for testing
executor exec.Interface
} }
// NewNsenter constructs a new instance of Nsenter // NewNsenter constructs a new instance of Nsenter
func NewNsenter() (*Nsenter, error) { func NewNsenter(hostRootFsPath string, executor exec.Interface) (*Nsenter, error) {
ne := &Nsenter{ ne := &Nsenter{
paths: map[string]string{ hostRootFsPath: hostRootFsPath,
"mount": "", executor: executor,
"findmnt": "", }
"umount": "", if err := ne.initPaths(); err != nil {
"systemd-run": "", return nil, err
"stat": "", }
"touch": "", return ne, nil
"mkdir": "", }
"ls": "",
"sh": "", func (ne *Nsenter) initPaths() error {
"chmod": "", ne.paths = map[string]string{}
}, binaries := []string{
"mount",
"findmnt",
"umount",
"systemd-run",
"stat",
"touch",
"mkdir",
"sh",
"chmod",
"realpath",
} }
// search for the required commands in other locations besides /usr/bin // search for the required commands in other locations besides /usr/bin
for binary := range ne.paths { for _, binary := range binaries {
// check for binary under the following directories // check for binary under the following directories
for _, path := range []string{"/", "/bin", "/usr/sbin", "/usr/bin"} { for _, path := range []string{"/", "/bin", "/usr/sbin", "/usr/bin"} {
binPath := filepath.Join(path, binary) binPath := filepath.Join(path, binary)
if _, err := os.Stat(filepath.Join(hostRootFsPath, binPath)); err != nil { if _, err := os.Stat(filepath.Join(ne.hostRootFsPath, binPath)); err != nil {
continue continue
} }
ne.paths[binary] = binPath ne.paths[binary] = binPath
@ -96,19 +116,19 @@ func NewNsenter() (*Nsenter, error) {
} }
// systemd-run is optional, bailout if we don't find any of the other binaries // systemd-run is optional, bailout if we don't find any of the other binaries
if ne.paths[binary] == "" && binary != "systemd-run" { if ne.paths[binary] == "" && binary != "systemd-run" {
return nil, fmt.Errorf("unable to find %v", binary) return fmt.Errorf("unable to find %v", binary)
} }
} }
return ne, nil return nil
} }
// Exec executes nsenter commands in hostProcMountNsPath mount namespace // Exec executes nsenter commands in hostProcMountNsPath mount namespace
func (ne *Nsenter) Exec(cmd string, args []string) exec.Cmd { func (ne *Nsenter) Exec(cmd string, args []string) exec.Cmd {
hostProcMountNsPath := filepath.Join(ne.hostRootFsPath, mountNsPath)
fullArgs := append([]string{fmt.Sprintf("--mount=%s", hostProcMountNsPath), "--"}, fullArgs := append([]string{fmt.Sprintf("--mount=%s", hostProcMountNsPath), "--"},
append([]string{ne.AbsHostPath(cmd)}, args...)...) append([]string{ne.AbsHostPath(cmd)}, args...)...)
glog.V(5).Infof("Running nsenter command: %v %v", nsenterPath, fullArgs) glog.V(5).Infof("Running nsenter command: %v %v", nsenterPath, fullArgs)
exec := exec.New() return ne.executor.Command(nsenterPath, fullArgs...)
return exec.Command(nsenterPath, fullArgs...)
} }
// AbsHostPath returns the absolute runnable path for a specified command // AbsHostPath returns the absolute runnable path for a specified command
@ -128,8 +148,26 @@ func (ne *Nsenter) SupportsSystemd() (string, bool) {
// EvalSymlinks returns the path name on the host after evaluating symlinks on the // EvalSymlinks returns the path name on the host after evaluating symlinks on the
// host. // host.
func (ne *Nsenter) EvalSymlinks(pathname string) (string, error) { // mustExist makes EvalSymlinks to return error when the path does not
args := []string{"-m", pathname} // exist. When it's false, it evaluates symlinks of the existing part and
// blindly adds the non-existing part:
// pathname: /mnt/volume/non/existing/directory
// /mnt/volume exists
// non/existing/directory does not exist
// -> It resolves symlinks in /mnt/volume to say /mnt/foo and returns
// /mnt/foo/non/existing/directory.
//
// BEWARE! EvalSymlinks is not able to detect symlink looks with mustExist=false!
// If /tmp/link is symlink to /tmp/link, EvalSymlinks(/tmp/link/foo) returns /tmp/link/foo.
func (ne *Nsenter) EvalSymlinks(pathname string, mustExist bool) (string, error) {
var args []string
if mustExist {
// "realpath -e: all components of the path must exist"
args = []string{"-e", pathname}
} else {
// "realpath -m: no path components need exist or be a directory"
args = []string{"-m", pathname}
}
outBytes, err := ne.Exec("realpath", args).CombinedOutput() outBytes, err := ne.Exec("realpath", args).CombinedOutput()
if err != nil { if err != nil {
glog.Infof("failed to resolve symbolic links on %s: %v", pathname, err) glog.Infof("failed to resolve symbolic links on %s: %v", pathname, err)
@ -139,11 +177,60 @@ func (ne *Nsenter) EvalSymlinks(pathname string) (string, error) {
} }
// KubeletPath returns the path name that can be accessed by containerized // KubeletPath returns the path name that can be accessed by containerized
// kubelet, after evaluating symlinks on the host. // kubelet. It is recommended to resolve symlinks on the host by EvalSymlinks
func (ne *Nsenter) KubeletPath(pathname string) (string, error) { // before calling this function
hostpath, err := ne.EvalSymlinks(pathname) func (ne *Nsenter) KubeletPath(pathname string) string {
if err != nil { return filepath.Join(ne.hostRootFsPath, pathname)
return "", err
}
return filepath.Join(hostRootFsPath, hostpath), nil
} }
// NewFakeNsenter returns a Nsenter that does not run "nsenter --mount=... --",
// but runs everything in the same mount namespace as the unit test binary.
// rootfsPath is supposed to be a symlink, e.g. /tmp/xyz/rootfs -> /.
// This fake Nsenter is enough for most operations, e.g. to resolve symlinks,
// but it's not enough to call /bin/mount - unit tests don't run as root.
func NewFakeNsenter(rootfsPath string) (*Nsenter, error) {
executor := &fakeExec{
rootfsPath: rootfsPath,
}
// prepare /rootfs/bin, usr/bin and usr/sbin
bin := filepath.Join(rootfsPath, "bin")
if err := os.Symlink("/bin", bin); err != nil {
return nil, err
}
usr := filepath.Join(rootfsPath, "usr")
if err := os.Mkdir(usr, 0755); err != nil {
return nil, err
}
usrbin := filepath.Join(usr, "bin")
if err := os.Symlink("/usr/bin", usrbin); err != nil {
return nil, err
}
usrsbin := filepath.Join(usr, "sbin")
if err := os.Symlink("/usr/sbin", usrsbin); err != nil {
return nil, err
}
return NewNsenter(rootfsPath, executor)
}
type fakeExec struct {
rootfsPath string
}
func (f fakeExec) Command(cmd string, args ...string) exec.Cmd {
// This will intentionaly panic if Nsenter does not provide enough arguments.
realCmd := args[2]
realArgs := args[3:]
return exec.New().Command(realCmd, realArgs...)
}
func (fakeExec) LookPath(file string) (string, error) {
return "", errors.New("not implemented")
}
func (fakeExec) CommandContext(ctx context.Context, cmd string, args ...string) exec.Cmd {
return nil
}
var _ exec.Interface = fakeExec{}

311
pkg/util/nsenter/nsenter_test.go Normal file
View File

@ -0,0 +1,311 @@
// +build linux
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package nsenter
import (
"io/ioutil"
"os"
"path/filepath"
"testing"
"k8s.io/utils/exec"
)
func TestExec(t *testing.T) {
tests := []struct {
name string
command string
args []string
expectedOutput string
expectError bool
}{
{
name: "simple command",
command: "echo",
args: []string{"hello", "world"},
expectedOutput: "hello world\n",
},
{
name: "nozero exit code",
command: "false",
expectError: true,
},
}
executor := fakeExec{
rootfsPath: "/rootfs",
}
for _, test := range tests {
ns := Nsenter{
hostRootFsPath: "/rootfs",
executor: executor,
}
cmd := ns.Exec(test.command, test.args)
outBytes, err := cmd.CombinedOutput()
out := string(outBytes)
if err != nil && !test.expectError {
t.Errorf("Test %q: unexpected error: %s", test.name, err)
}
if err == nil && test.expectError {
t.Errorf("Test %q: expected error, got none", test.name)
}
if test.expectedOutput != out {
t.Errorf("test %q: expected output %q, got %q", test.name, test.expectedOutput, out)
}
}
}
func TestKubeletPath(t *testing.T) {
tests := []struct {
rootfs string
hostpath string
expectedKubeletPath string
}{
{
// simple join
"/rootfs",
"/some/path",
"/rootfs/some/path",
},
{
// squash slashes
"/rootfs/",
"//some/path",
"/rootfs/some/path",
},
}
for _, test := range tests {
ns := Nsenter{
hostRootFsPath: test.rootfs,
}
out := ns.KubeletPath(test.hostpath)
if out != test.expectedKubeletPath {
t.Errorf("Expected path %q, got %q", test.expectedKubeletPath, out)
}
}
}
func TestEvalSymlinks(t *testing.T) {
tests := []struct {
name string
mustExist bool
prepare func(tmpdir string) (src string, expectedDst string, err error)
expectError bool
}{
{
name: "simple file /src",
mustExist: true,
prepare: func(tmpdir string) (src string, expectedDst string, err error) {
src = filepath.Join(tmpdir, "src")
err = ioutil.WriteFile(src, []byte{}, 0644)
return src, src, err
},
},
{
name: "non-existing file /src",
mustExist: true,
prepare: func(tmpdir string) (src string, expectedDst string, err error) {
src = filepath.Join(tmpdir, "src")
return src, "", nil
},
expectError: true,
},
{
name: "non-existing file /src/ with mustExist=false",
mustExist: false,
prepare: func(tmpdir string) (src string, expectedDst string, err error) {
src = filepath.Join(tmpdir, "src")
return src, src, nil
},
},
{
name: "non-existing file /existing/path/src with mustExist=false with existing directories",
mustExist: false,
prepare: func(tmpdir string) (src string, expectedDst string, err error) {
src = filepath.Join(tmpdir, "existing/path")
if err := os.MkdirAll(src, 0755); err != nil {
return "", "", err
}
src = filepath.Join(src, "src")
return src, src, nil
},
},
{
name: "simple symlink /src -> /dst",
mustExist: false,
prepare: func(tmpdir string) (src string, expectedDst string, err error) {
dst := filepath.Join(tmpdir, "dst")
if err = ioutil.WriteFile(dst, []byte{}, 0644); err != nil {
return "", "", err
}
src = filepath.Join(tmpdir, "src")
err = os.Symlink(dst, src)
return src, dst, err
},
},
{
name: "dangling symlink /src -> /non-existing-path",
mustExist: true,
prepare: func(tmpdir string) (src string, expectedDst string, err error) {
dst := filepath.Join(tmpdir, "non-existing-path")
src = filepath.Join(tmpdir, "src")
err = os.Symlink(dst, src)
return src, "", err
},
expectError: true,
},
{
name: "dangling symlink /src -> /non-existing-path with mustExist=false",
mustExist: false,
prepare: func(tmpdir string) (src string, expectedDst string, err error) {
dst := filepath.Join(tmpdir, "non-existing-path")
src = filepath.Join(tmpdir, "src")
err = os.Symlink(dst, src)
return src, dst, err
},
},
{
name: "symlink to directory /src/file, where /src is link to /dst",
mustExist: true,
prepare: func(tmpdir string) (src string, expectedDst string, err error) {
dst := filepath.Join(tmpdir, "dst")
if err = os.Mkdir(dst, 0755); err != nil {
return "", "", err
}
dstFile := filepath.Join(dst, "file")
if err = ioutil.WriteFile(dstFile, []byte{}, 0644); err != nil {
return "", "", err
}
src = filepath.Join(tmpdir, "src")
if err = os.Symlink(dst, src); err != nil {
return "", "", err
}
srcFile := filepath.Join(src, "file")
return srcFile, dstFile, nil
},
},
{
name: "symlink to non-existing directory: /src/file, where /src is link to /dst and dst does not exist",
mustExist: true,
prepare: func(tmpdir string) (src string, expectedDst string, err error) {
dst := filepath.Join(tmpdir, "dst")
src = filepath.Join(tmpdir, "src")
if err = os.Symlink(dst, src); err != nil {
return "", "", err
}
srcFile := filepath.Join(src, "file")
return srcFile, "", nil
},
expectError: true,
},
{
name: "symlink to non-existing directory: /src/file, where /src is link to /dst and dst does not exist with mustExist=false",
mustExist: false,
prepare: func(tmpdir string) (src string, expectedDst string, err error) {
dst := filepath.Join(tmpdir, "dst")
dstFile := filepath.Join(dst, "file")
src = filepath.Join(tmpdir, "src")
if err = os.Symlink(dst, src); err != nil {
return "", "", err
}
srcFile := filepath.Join(src, "file")
return srcFile, dstFile, nil
},
},
}
for _, test := range tests {
ns := Nsenter{
hostRootFsPath: "/rootfs",
executor: fakeExec{
rootfsPath: "/rootfs",
},
}
tmpdir, err := ioutil.TempDir("", "nsenter-hostpath-")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
src, expectedDst, err := test.prepare(tmpdir)
if err != nil {
t.Error(err)
continue
}
dst, err := ns.EvalSymlinks(src, test.mustExist)
if err != nil && !test.expectError {
t.Errorf("Test %q: unexpected error: %s", test.name, err)
}
if err == nil && test.expectError {
t.Errorf("Test %q: expected error, got none", test.name)
}
if dst != expectedDst {
t.Errorf("Test %q: expected destination %q, got %q", test.name, expectedDst, dst)
}
}
}
func TestNewNsenter(t *testing.T) {
// Create a symlink /tmp/xyz/rootfs -> / and use it as rootfs path
// It should resolve all binaries correctly, the test runs on Linux
tmpdir, err := ioutil.TempDir("", "nsenter-hostpath-")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
rootfs := filepath.Join(tmpdir, "rootfs")
if err = os.Symlink("/", rootfs); err != nil {
t.Fatal(err)
}
_, err = NewNsenter(rootfs, exec.New())
if err != nil {
t.Errorf("Error: %s", err)
}
}
func TestNewNsenterError(t *testing.T) {
// Create empty dir /tmp/xyz/rootfs and use it as rootfs path
// It should resolve all binaries correctly, the test runs on Linux
tmpdir, err := ioutil.TempDir("", "nsenter-hostpath-")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
rootfs := filepath.Join(tmpdir, "rootfs")
if err = os.MkdirAll(rootfs, 0755); err != nil {
t.Fatal(err)
}
_, err = NewNsenter(rootfs, exec.New())
if err == nil {
t.Errorf("Expected error, got none")
}
}

8
pkg/util/nsenter/nsenter_unsupported.go
View File

@ -22,6 +22,12 @@ import (
"k8s.io/utils/exec" "k8s.io/utils/exec"
) )
const (
// DefaultHostRootFsPath is path to host's filesystem mounted into container
// with kubelet.
DefaultHostRootFsPath = "/rootfs"
)
// Nsenter is part of experimental support for running the kubelet // Nsenter is part of experimental support for running the kubelet
// in a container. // in a container.
type Nsenter struct { type Nsenter struct {
@ -30,7 +36,7 @@ type Nsenter struct {
} }
// NewNsenter constructs a new instance of Nsenter // NewNsenter constructs a new instance of Nsenter
func NewNsenter() (*Nsenter, error) { func NewNsenter(hostRootFsPath string, executor exec.Interface) (*Nsenter, error) {
return &Nsenter{}, nil return &Nsenter{}, nil
} }

8
pkg/util/removeall/removeall_test.go
View File

@ -75,8 +75,8 @@ func (mounter *fakeMounter) MakeFile(pathname string) error {
return nil return nil
} }
func (mounter *fakeMounter) ExistsPath(pathname string) bool { func (mounter *fakeMounter) ExistsPath(pathname string) (bool, error) {
return true return true, errors.New("not implemented")
} }
func (mounter *fakeMounter) PrepareSafeSubpath(subPath mount.Subpath) (newHostPath string, cleanupAction func(), err error) { func (mounter *fakeMounter) PrepareSafeSubpath(subPath mount.Subpath) (newHostPath string, cleanupAction func(), err error) {
@ -103,6 +103,10 @@ func (mounter *fakeMounter) GetSELinuxSupport(pathname string) (bool, error) {
return false, errors.New("not implemented") return false, errors.New("not implemented")
} }
func (mounter *fakeMounter) GetMode(pathname string) (os.FileMode, error) {
return 0, errors.New("not implemented")
}
func (mounter *fakeMounter) IsLikelyNotMountPoint(file string) (bool, error) { func (mounter *fakeMounter) IsLikelyNotMountPoint(file string) (bool, error) {
name := path.Base(file) name := path.Base(file)
if strings.HasPrefix(name, "mount") { if strings.HasPrefix(name, "mount") {

3
pkg/volume/host_path/host_path.go
View File

@ -350,7 +350,8 @@ type fileTypeChecker struct {
} }
func (ftc *fileTypeChecker) Exists() bool { func (ftc *fileTypeChecker) Exists() bool {
return ftc.mounter.ExistsPath(ftc.path) exists, err := ftc.mounter.ExistsPath(ftc.path)
return exists && err == nil
} }
func (ftc *fileTypeChecker) IsFile() bool { func (ftc *fileTypeChecker) IsFile() bool {

8
pkg/volume/host_path/host_path_test.go
View File

@ -369,8 +369,8 @@ func (fftc *fakeFileTypeChecker) MakeDir(pathname string) error {
return nil return nil
} }
func (fftc *fakeFileTypeChecker) ExistsPath(pathname string) bool { func (fftc *fakeFileTypeChecker) ExistsPath(pathname string) (bool, error) {
return true return true, nil
} }
func (fftc *fakeFileTypeChecker) GetFileType(_ string) (utilmount.FileType, error) { func (fftc *fakeFileTypeChecker) GetFileType(_ string) (utilmount.FileType, error) {
@ -401,6 +401,10 @@ func (fftc *fakeFileTypeChecker) GetSELinuxSupport(pathname string) (bool, error
return false, errors.New("not implemented") return false, errors.New("not implemented")
} }
func (fftc *fakeFileTypeChecker) GetMode(pathname string) (os.FileMode, error) {
return 0, errors.New("not implemented")
}
func setUp() error { func setUp() error {
err := os.MkdirAll("/tmp/ExistingFolder", os.FileMode(0755)) err := os.MkdirAll("/tmp/ExistingFolder", os.FileMode(0755))
if err != nil { if err != nil {