From 46b0b3491f3163118e4cba9e0b799765588f4103 Mon Sep 17 00:00:00 2001 From: Di Xu Date: Fri, 1 Sep 2017 09:53:16 +0800 Subject: [PATCH] refactor nsenter to new pkg/util --- pkg/util/mount/mount_linux.go | 6 +- pkg/util/mount/nsenter_mount.go | 136 +++++------------------- pkg/util/nsenter/nsenter.go | 124 +++++++++++++++++++++ pkg/util/nsenter/nsenter_unsupported.go | 50 +++++++++ 4 files changed, 203 insertions(+), 113 deletions(-) create mode 100644 pkg/util/nsenter/nsenter.go create mode 100644 pkg/util/nsenter/nsenter_unsupported.go diff --git a/pkg/util/mount/mount_linux.go b/pkg/util/mount/mount_linux.go index 2845861cd31..0303eb7dc08 100644 --- a/pkg/util/mount/mount_linux.go +++ b/pkg/util/mount/mount_linux.go @@ -350,9 +350,7 @@ func parseProcMounts(content []byte) ([]MountPoint, error) { } func (mounter *Mounter) MakeRShared(path string) error { - mountCmd := defaultMountCommand - mountArgs := []string{} - return doMakeRShared(path, procMountInfoPath, mountCmd, mountArgs) + return doMakeRShared(path, procMountInfoPath) } // formatAndMount uses unix utils to format and mount the given disk @@ -523,7 +521,7 @@ func parseMountInfo(filename string) ([]mountInfo, error) { // path is shared and bind-mounts it as rshared if needed. mountCmd and // mountArgs are expected to contain mount-like command, doMakeRShared will add // '--bind ' and '--make-rshared ' to mountArgs. -func doMakeRShared(path string, mountInfoFilename string, mountCmd string, mountArgs []string) error { +func doMakeRShared(path string, mountInfoFilename string) error { shared, err := isShared(path, mountInfoFilename) if err != nil { return err diff --git a/pkg/util/mount/nsenter_mount.go b/pkg/util/mount/nsenter_mount.go index 24a095a6c39..f1f69d24d32 100644 --- a/pkg/util/mount/nsenter_mount.go +++ b/pkg/util/mount/nsenter_mount.go @@ -25,78 +25,30 @@ import ( "strings" "github.com/golang/glog" - "k8s.io/utils/exec" + "k8s.io/kubernetes/pkg/util/nsenter" ) -// NsenterMounter is part of experimental support for running the kubelet -// in a container. Currently, all docker containers receive their own mount -// namespaces. NsenterMounter works by executing nsenter to run commands in +const ( + // hostProcMountsPath is the default mount path for rootfs + hostProcMountsPath = "/rootfs/proc/1/mounts" + // hostProcMountinfoPath is the default mount info path for rootfs + hostProcMountinfoPath = "/rootfs/proc/1/mountinfo" +) + +// Currently, all docker containers receive their own mount namespaces. +// NsenterMounter works by executing nsenter to run commands in // the host's mount namespace. -// -// NsenterMounter requires: -// -// 1. Docker >= 1.6 due to the dependency on the slave propagation mode -// of the bind-mount of the kubelet root directory in the container. -// Docker 1.5 used a private propagation mode for bind-mounts, so mounts -// performed in the host's mount namespace do not propagate out to the -// bind-mount in this docker version. -// 2. The host's root filesystem must be available at /rootfs -// 3. The nsenter binary must be on the Kubelet process' PATH in the container's -// filesystem. -// 4. The Kubelet process must have CAP_SYS_ADMIN (required by nsenter); at -// the present, this effectively means that the kubelet is running in a -// privileged container. -// 5. The volume path used by the Kubelet must be the same inside and outside -// the container and be writable by the container (to initialize volume) -// contents. TODO: remove this requirement. -// 6. The host image must have mount, findmnt, and umount binaries in /bin, -// /usr/sbin, or /usr/bin -// 7. The host image should have systemd-run in /bin, /usr/sbin, or /usr/bin -// For more information about mount propagation modes, see: -// https://www.kernel.org/doc/Documentation/filesystems/sharedsubtree.txt type NsenterMounter struct { - // a map of commands to their paths on the host filesystem - paths map[string]string + ne *nsenter.Nsenter } func NewNsenterMounter() *NsenterMounter { - m := &NsenterMounter{ - paths: map[string]string{ - "mount": "", - "findmnt": "", - "umount": "", - "systemd-run": "", - }, - } - // search for the mount command in other locations besides /usr/bin - for binary := range m.paths { - // default to root - m.paths[binary] = filepath.Join("/", binary) - for _, path := range []string{"/bin", "/usr/sbin", "/usr/bin"} { - binPath := filepath.Join(path, binary) - if _, err := os.Stat(filepath.Join(hostRootFsPath, binPath)); err != nil { - continue - } - m.paths[binary] = binPath - break - } - // TODO: error, so that the kubelet can stop if the mounts don't exist - // (don't forget that systemd-run is optional) - } - return m + return &NsenterMounter{ne: nsenter.NewNsenter()} } // NsenterMounter implements mount.Interface var _ = Interface(&NsenterMounter{}) -const ( - hostRootFsPath = "/rootfs" - hostProcMountsPath = "/rootfs/proc/1/mounts" - hostProcMountinfoPath = "/rootfs/proc/1/mountinfo" - hostMountNamespacePath = "/rootfs/proc/1/ns/mnt" - nsenterPath = "nsenter" -) - // Mount runs mount(8) in the host's root mount namespace. Aside from this // aspect, Mount has the same semantics as the mounter returned by mount.New() func (n *NsenterMounter) Mount(source string, target string, fstype string, options []string) error { @@ -116,26 +68,22 @@ func (n *NsenterMounter) Mount(source string, target string, fstype string, opti // doNsenterMount nsenters the host's mount namespace and performs the // requested mount. func (n *NsenterMounter) doNsenterMount(source, target, fstype string, options []string) error { - glog.V(5).Infof("nsenter Mounting %s %s %s %v", source, target, fstype, options) - args := n.makeNsenterArgs(source, target, fstype, options) - - glog.V(5).Infof("Mount command: %v %v", nsenterPath, args) - exec := exec.New() - outputBytes, err := exec.Command(nsenterPath, args...).CombinedOutput() + glog.V(5).Infof("nsenter mount %s %s %s %v", source, target, fstype, options) + cmd, args := n.makeNsenterArgs(source, target, fstype, options) + outputBytes, err := n.ne.Exec(cmd, args).CombinedOutput() if len(outputBytes) != 0 { glog.V(5).Infof("Output of mounting %s to %s: %v", source, target, string(outputBytes)) } - return err } // makeNsenterArgs makes a list of argument to nsenter in order to do the // requested mount. -func (n *NsenterMounter) makeNsenterArgs(source, target, fstype string, options []string) []string { - mountCmd := n.absHostPath("mount") +func (n *NsenterMounter) makeNsenterArgs(source, target, fstype string, options []string) (string, []string) { + mountCmd := n.ne.AbsHostPath("mount") mountArgs := makeMountArgs(source, target, fstype, options) - if systemdRunPath, hasSystemd := n.paths["systemd-run"]; hasSystemd { + if systemdRunPath, hasSystemd := n.ne.SupportsSystemd(); hasSystemd { // Complete command line: // nsenter --mount=/rootfs/proc/1/ns/mnt -- /bin/systemd-run --description=... --scope -- /bin/mount -t // Expected flow is: @@ -165,34 +113,20 @@ func (n *NsenterMounter) makeNsenterArgs(source, target, fstype string, options // No code here, mountCmd and mountArgs use /bin/mount } - nsenterArgs := []string{ - "--mount=" + hostMountNamespacePath, - "--", - mountCmd, - } - nsenterArgs = append(nsenterArgs, mountArgs...) - - return nsenterArgs + return mountCmd, mountArgs } // Unmount runs umount(8) in the host's mount namespace. func (n *NsenterMounter) Unmount(target string) error { - args := []string{ - "--mount=" + hostMountNamespacePath, - "--", - n.absHostPath("umount"), - target, - } + args := []string{target} // No need to execute systemd-run here, it's enough that unmount is executed // in the host's mount namespace. It will finish appropriate fuse daemon(s) // running in any scope. - glog.V(5).Infof("Unmount command: %v %v", nsenterPath, args) - exec := exec.New() - outputBytes, err := exec.Command(nsenterPath, args...).CombinedOutput() + glog.V(5).Infof("nsenter unmount args: %v", args) + outputBytes, err := n.ne.Exec("umount", args).CombinedOutput() if len(outputBytes) != 0 { glog.V(5).Infof("Output of unmounting %s: %v", target, string(outputBytes)) } - return err } @@ -207,7 +141,7 @@ func (m *NsenterMounter) IsNotMountPoint(dir string) (bool, error) { func (*NsenterMounter) IsMountPointMatch(mp MountPoint, dir string) bool { deletedDir := fmt.Sprintf("%s\\040(deleted)", dir) - return ((mp.Path == dir) || (mp.Path == deletedDir)) + return (mp.Path == dir) || (mp.Path == deletedDir) } // IsLikelyNotMountPoint determines whether a path is a mountpoint by calling findmnt @@ -227,11 +161,9 @@ func (n *NsenterMounter) IsLikelyNotMountPoint(file string) (bool, error) { // the first of multiple possible mountpoints using --first-only. // Also add fstype output to make sure that the output of target file will give the full path // TODO: Need more refactoring for this function. Track the solution with issue #26996 - args := []string{"--mount=" + hostMountNamespacePath, "--", n.absHostPath("findmnt"), "-o", "target,fstype", "--noheadings", "--first-only", "--target", file} - glog.V(5).Infof("findmnt command: %v %v", nsenterPath, args) - - exec := exec.New() - out, err := exec.Command(nsenterPath, args...).CombinedOutput() + args := []string{"-o", "target,fstype", "--noheadings", "--first-only", "--target", file} + glog.V(5).Infof("nsenter findmnt args: %v", args) + out, err := n.ne.Exec("findmnt", args).CombinedOutput() if err != nil { glog.V(2).Infof("Failed findmnt command for path %s: %v", file, err) // Different operating systems behave differently for paths which are not mount points. @@ -285,20 +217,6 @@ func (n *NsenterMounter) GetDeviceNameFromMount(mountPath, pluginDir string) (st return getDeviceNameFromMount(n, mountPath, pluginDir) } -func (n *NsenterMounter) absHostPath(command string) string { - path, ok := n.paths[command] - if !ok { - return command - } - return path -} - func (n *NsenterMounter) MakeRShared(path string) error { - nsenterCmd := nsenterPath - nsenterArgs := []string{ - "--mount=" + hostMountNamespacePath, - "--", - n.absHostPath("mount"), - } - return doMakeRShared(path, hostProcMountinfoPath, nsenterCmd, nsenterArgs) + return doMakeRShared(path, hostProcMountinfoPath) } diff --git a/pkg/util/nsenter/nsenter.go b/pkg/util/nsenter/nsenter.go new file mode 100644 index 00000000000..32fbc08487e --- /dev/null +++ b/pkg/util/nsenter/nsenter.go @@ -0,0 +1,124 @@ +// +build linux + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nsenter + +import ( + "fmt" + "os" + "path/filepath" + + "k8s.io/utils/exec" + + "github.com/golang/glog" +) + +const ( + hostRootFsPath = "/rootfs" + // hostProcMountNsPath is the default mount namespace for rootfs + hostProcMountNsPath = "/rootfs/proc/1/ns/mnt" + // nsenterPath is the default nsenter command + nsenterPath = "nsenter" +) + +// Nsenter is part of experimental support for running the kubelet +// in a container. +// +// Nsenter requires: +// +// 1. Docker >= 1.6 due to the dependency on the slave propagation mode +// of the bind-mount of the kubelet root directory in the container. +// Docker 1.5 used a private propagation mode for bind-mounts, so mounts +// performed in the host's mount namespace do not propagate out to the +// bind-mount in this docker version. +// 2. The host's root filesystem must be available at /rootfs +// 3. The nsenter binary must be on the Kubelet process' PATH in the container's +// filesystem. +// 4. The Kubelet process must have CAP_SYS_ADMIN (required by nsenter); at +// the present, this effectively means that the kubelet is running in a +// privileged container. +// 5. The volume path used by the Kubelet must be the same inside and outside +// the container and be writable by the container (to initialize volume) +// contents. TODO: remove this requirement. +// 6. The host image must have "mount", "findmnt", "umount", "stat", "touch", +// "mkdir", "ls", "sh" and "chmod" binaries in /bin, /usr/sbin, or /usr/bin +// 7. The host image should have systemd-run in /bin, /usr/sbin, or /usr/bin +// For more information about mount propagation modes, see: +// https://www.kernel.org/doc/Documentation/filesystems/sharedsubtree.txt +type Nsenter struct { + // a map of commands to their paths on the host filesystem + paths map[string]string +} + +// NewNsenter constructs a new instance of Nsenter +func NewNsenter() *Nsenter { + ne := &Nsenter{ + paths: map[string]string{ + "mount": "", + "findmnt": "", + "umount": "", + "systemd-run": "", + "stat": "", + "touch": "", + "mkdir": "", + "ls": "", + "sh": "", + "chmod": "", + }, + } + // search for the required commands in other locations besides /usr/bin + for binary := range ne.paths { + // default to root + ne.paths[binary] = filepath.Join("/", binary) + for _, path := range []string{"/bin", "/usr/sbin", "/usr/bin"} { + binPath := filepath.Join(path, binary) + if _, err := os.Stat(filepath.Join(hostRootFsPath, binPath)); err != nil { + continue + } + ne.paths[binary] = binPath + break + } + // TODO: error, so that the kubelet can stop if the paths don't exist + // (don't forget that systemd-run is optional) + } + return ne +} + +// Exec executes nsenter commands in hostProcMountNsPath mount namespace +func (ne *Nsenter) Exec(cmd string, args []string) exec.Cmd { + fullArgs := append([]string{fmt.Sprintf("--mount=%s", hostProcMountNsPath), "--"}, + append([]string{ne.AbsHostPath(cmd)}, args...)...) + glog.V(5).Infof("Running nsenter command: %v %v", nsenterPath, fullArgs) + exec := exec.New() + return exec.Command(nsenterPath, fullArgs...) +} + +// AbsHostPath returns the absolute runnable path for a specified command +func (ne *Nsenter) AbsHostPath(command string) string { + path, ok := ne.paths[command] + if !ok { + return command + } + return path +} + +// SupportsSystemd checks whether command systemd-run exists +func (ne *Nsenter) SupportsSystemd() (string, bool) { + systemdRunPath, hasSystemd := ne.paths["systemd-run"] + return systemdRunPath, hasSystemd +} diff --git a/pkg/util/nsenter/nsenter_unsupported.go b/pkg/util/nsenter/nsenter_unsupported.go new file mode 100644 index 00000000000..17b3b5722d3 --- /dev/null +++ b/pkg/util/nsenter/nsenter_unsupported.go @@ -0,0 +1,50 @@ +// +build !linux + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nsenter + +import ( + "k8s.io/utils/exec" +) + +// Nsenter is part of experimental support for running the kubelet +// in a container. +type Nsenter struct { + // a map of commands to their paths on the host filesystem + Paths map[string]string +} + +// NewNsenter constructs a new instance of Nsenter +func NewNsenter() *Nsenter { + return &Nsenter{} +} + +// Exec executes nsenter commands in hostProcMountNsPath mount namespace +func (ne *Nsenter) Exec(args ...string) exec.Cmd { + return nil +} + +// AbsHostPath returns the absolute runnable path for a specified command +func (ne *Nsenter) AbsHostPath(command string) string { + return "" +} + +// SupportsSystemd checks whether command systemd-run exists +func (ne *Nsenter) SupportsSystemd() (string, bool) { + return "", false +}